From 8dbd8bd7ea3d9aabc49adbf22cb87b6f635414cd Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Mon, 4 Mar 2024 16:47:25 +0100
Subject: [PATCH 01/33] WIP: sqlpage.run_sql

---
 src/webserver/database/sql_pseudofunctions.rs | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/webserver/database/sql_pseudofunctions.rs b/src/webserver/database/sql_pseudofunctions.rs
index bf34eca1..bf90de8b 100644
--- a/src/webserver/database/sql_pseudofunctions.rs
+++ b/src/webserver/database/sql_pseudofunctions.rs
@@ -38,6 +38,7 @@ pub(super) enum StmtParam {
     UploadedFileMimeType(String),
     ReadFileAsText(Box<StmtParam>),
     ReadFileAsDataUrl(Box<StmtParam>),
+    RunSql(Box<StmtParam>),
     Path,
     Protocol,
 }
@@ -107,6 +108,7 @@ pub(super) fn func_call_to_param(func_name: &str, arguments: &mut [FunctionArg])
         "read_file_as_data_url" => StmtParam::ReadFileAsDataUrl(Box::new(
             extract_variable_argument("read_file_as_data_url", arguments),
         )),
+        "run_sql" => StmtParam::RunSql(Box::new(extract_variable_argument("run_sql", arguments))),
         unknown_name => StmtParam::Error(format!(
             "Unknown function {unknown_name}({})",
             FormatArguments(arguments)
@@ -126,6 +128,7 @@ pub(super) async fn extract_req_param<'a>(
         StmtParam::UrlEncode(inner) => url_encode(inner, request)?,
         StmtParam::ReadFileAsText(inner) => read_file_as_text(inner, request).await?,
         StmtParam::ReadFileAsDataUrl(inner) => read_file_as_data_url(inner, request).await?,
+        StmtParam::RunSql(inner) => run_sql(inner, request).await?,
         _ => extract_req_param_non_nested(param, request)?,
     })
 }
@@ -248,6 +251,17 @@ async fn read_file_as_data_url<'a>(
     Ok(Some(Cow::Owned(data_url)))
 }
 
+async fn run_sql<'a>(
+    param0: &StmtParam,
+    request: &'a RequestInfo,
+) -> Result<Option<Cow<'a, str>>, anyhow::Error> {
+    let Some(sql_file) = extract_req_param_non_nested(param0, request)? else {
+        log::debug!("run_sql: first argument is NULL, returning NULL");
+        return Ok(None);
+    };
+    todo!("run_sql: {sql_file:?}");
+}
+
 fn mime_from_upload<'a>(param0: &StmtParam, request: &'a RequestInfo) -> Option<&'a Mime> {
     if let StmtParam::UploadedFilePath(name) | StmtParam::UploadedFileMimeType(name) = param0 {
         request.uploaded_files.get(name)?.content_type.as_ref()
@@ -308,8 +322,9 @@ pub(super) fn extract_req_param_non_nested<'a>(
             .map(|x| Cow::Borrowed(x.as_ref())),
         StmtParam::ReadFileAsText(_) => bail!("Nested read_file_as_text() function not allowed",),
         StmtParam::ReadFileAsDataUrl(_) => {
-            bail!("Nested read_file_as_data_url() function not allowed",)
+            bail!("Nested read_file_as_data_url() function not allowed")
         }
+        StmtParam::RunSql(_) => bail!("Nested run_sql() function not allowed"),
     })
 }
 

From 75d7c460405e3e425138faa19eaa064877311990 Mon Sep 17 00:00:00 2001
From: olivierauverlot <olivier.auverlot@gmail.com>
Date: Mon, 4 Mar 2024 16:08:17 +0100
Subject: [PATCH 02/33] A unique identifier for the input, which can then be
 used to select and manage the field with Javascript code. (#250)

Co-authored-by: Olivier Auverlot <oauv@duck.com>
---
 .../sqlpage/migrations/01_documentation.sql              | 3 ++-
 sqlpage/templates/form.handlebars                        | 9 ++++++---
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/examples/official-site/sqlpage/migrations/01_documentation.sql b/examples/official-site/sqlpage/migrations/01_documentation.sql
index a3876da0..5ce9d6b9 100644
--- a/examples/official-site/sqlpage/migrations/01_documentation.sql
+++ b/examples/official-site/sqlpage/migrations/01_documentation.sql
@@ -237,7 +237,8 @@ INSERT INTO parameter(component, name, description, type, top_level, optional) S
     ('class', 'A CSS class to apply to the form element.', 'TEXT', FALSE, TRUE),
     ('prefix_icon','Icon to display on the left side of the input field, on the same line.','ICON',FALSE,TRUE),
     ('prefix','Text to display on the left side of the input field, on the same line.','TEXT',FALSE,TRUE),
-    ('suffix','Short text to display after th input, on the same line. Useful to add units or a currency symbol to an input.','TEXT',FALSE,TRUE)
+    ('suffix','Short text to display after th input, on the same line. Useful to add units or a currency symbol to an input.','TEXT',FALSE,TRUE),
+    ('id','A unique identifier for the input, which can then be used to select and manage the field with Javascript code. Usefull for advanced using as setting client side event listeners, interactive control of input field (disabled, visibility, read only, e.g.) and AJAX requests.','TEXT',FALSE,TRUE)
 ) x;
 INSERT INTO example(component, description, properties) VALUES
     (
diff --git a/sqlpage/templates/form.handlebars b/sqlpage/templates/form.handlebars
index 74db9fd9..db1fcdf5 100644
--- a/sqlpage/templates/form.handlebars
+++ b/sqlpage/templates/form.handlebars
@@ -16,7 +16,7 @@
             {{#if (or (eq type "radio") (eq type "checkbox"))}}
                 <div class="form-selectgroup form-selectgroup-boxes d-flex flex-column my-1 col-md-{{default width 12}}">
                     <label class="form-selectgroup-item flex-fill">
-                        <input type="{{type}}" name="{{name}}" value="{{value}}" {{#if required}}required{{/if}} {{#if checked}}checked{{/if}} class="form-selectgroup-input">
+                        <input type="{{type}}" {{#if id}}id="{{id}}" {{/if}}name="{{name}}" value="{{value}}" {{#if required}}required{{/if}} {{#if checked}}checked{{/if}} class="form-selectgroup-input">
                         <div class="form-selectgroup-label d-flex align-items-center p-3">
                             <div class="me-3">
                                 <span class="form-selectgroup-check"></span>
@@ -35,7 +35,7 @@
                 </div>
             {{else}}
             {{~#if (eq type "hidden")}}
-                <input type="hidden" name="{{name}}" value="{{value}}">
+                <input type="hidden" name="{{name}}" {{#if id}}id="{{id}}" {{/if}}value="{{value}}">
             {{else}}
                 <label class="form-label mb-2 col-md-{{default width 12}}">
                     {{default label name}}
@@ -48,7 +48,8 @@
                             class="form-control"
                             placeholder="{{placeholder}}"
                             rows="{{default rows 3}}"
-                            {{#if value}}value="{{value}}" {{/if~}}
+                            {{#if id}}id="{{id}}" {{/if~}}
+                            {{~#if value}}value="{{value}}" {{/if~}}
                             {{~#if minlength}}minlength="{{minlength}}" {{/if~}}
                             {{~#if maxlength}}maxlength="{{maxlength}}" {{/if~}}
                             {{~#if required}}required="required" {{/if~}}
@@ -58,6 +59,7 @@
                         </textarea>
                     {{else}}{{#if (eq type 'select')}}
                         <select name="{{name}}" class="form-select"
+                        {{~#if id}} id="{{id}}" {{/if~}}
                         {{~#if required}} required="required" {{/if~}}
                         {{~#if autofocus}} autofocus {{/if~}}
                         {{~#if multiple}} multiple {{/if~}}
@@ -71,6 +73,7 @@
                             {{#if prefix_icon}}<span class="input-group-text">{{icon_img prefix_icon}}</span>{{/if}}
                             {{#if prefix}}<span class="input-group-text">{{prefix}}</span>{{/if}}
                             <input name="{{name}}" class="form-control {{class}}" 
+                                {{~#if id}} id="{{id}}" {{/if~}}
                                 {{~#if type}} type="{{type}}" {{/if~}}
                                 {{~#if placeholder}} placeholder="{{placeholder}}" {{/if~}}
                                 {{~#if value}} value="{{value}}" {{/if~}}

From 74527bc28b01d12b1b214a5761867736c7222c6c Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Mon, 4 Mar 2024 16:52:28 +0100
Subject: [PATCH 03/33] update changelog

---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f588a338..7481051b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # CHANGELOG.md
 
+## 0.20.0 (unreleased)
+
+ - Add an `id` attribute to form fields in the [form](https://sql.ophir.dev/documentation.sql?component=form#component) component. This allows you to easily reference form fields in custom javascript code.
+
 ## 0.19.1 (2024-02-28)
   - **SECURITY**: fixes users being able to re-run migrations by visiting `/sqlpage/migrations/NNNN_name.sql` pages. If you are using sqlpage migrations, your migrations are not idempotent, and you use the default SQLPAGE_WEB_ROOT (`./`) and `SQLPAGE_CONFIGURATION_DIRECTORY` (`./sqlpage/`), you should upgrade to this version as soon as possible. If you are using a custom `SQLPAGE_WEB_ROOT` or `SQLPAGE_CONFIGURATION_DIRECTORY` or your migrations are idempotent, you can upgrade at your convenience.
  - Better error messages on invalid database connection strings. SQLPage now displays a more precise and useful message when an error occurs instead of a "panic" message.

From f865b1f4ef2fecb35c56d658e9300382f6a03707 Mon Sep 17 00:00:00 2001
From: olivierauverlot <olivier.auverlot@gmail.com>
Date: Tue, 5 Mar 2024 19:42:44 +0100
Subject: [PATCH 04/33] New RSS component (#252)

* Produces a data flow in the RSS format

* add documentation and rss reference

* more complete rss component, with support for podcasts

* podcast changes

* documentation

* format dates as rfc2822

* extract template helpers

* extract more helpers

* remove unnecassary error handling

* fmt

* two-param helpers

* rfc2822_date_helper

* test rfc date formatter

* accept dates with or without time

* remove unneeded line breaks

---------

Co-authored-by: Olivier Auverlot <oauv@duck.com>
Co-authored-by: lovasoa <pere.jobs@gmail.com>
---
 examples/official-site/blog.sql               |   1 +
 examples/official-site/index.sql              |   3 +-
 examples/official-site/rss.sql                |  21 +
 .../sqlpage/migrations/01_documentation.sql   |   1 +
 .../sqlpage/migrations/37_rss.sql             | 273 +++++++++++++
 sqlpage/templates/rss.handlebars              |  42 ++
 sqlpage/templates/shell.handlebars            |   3 +
 src/lib.rs                                    |   1 +
 src/template_helpers.rs                       | 383 ++++++++++++++++++
 src/templates.rs                              | 235 +----------
 10 files changed, 730 insertions(+), 233 deletions(-)
 create mode 100644 examples/official-site/rss.sql
 create mode 100644 examples/official-site/sqlpage/migrations/37_rss.sql
 create mode 100644 sqlpage/templates/rss.handlebars
 create mode 100644 src/template_helpers.rs

diff --git a/examples/official-site/blog.sql b/examples/official-site/blog.sql
index a3789946..5936fdd3 100644
--- a/examples/official-site/blog.sql
+++ b/examples/official-site/blog.sql
@@ -10,6 +10,7 @@ select 'shell' as component,
     'Poppins' as font,
     'https://cdn.jsdelivr.net/npm/prismjs@1/components/prism-core.min.js' as javascript,
     'https://cdn.jsdelivr.net/npm/prismjs@1/plugins/autoloader/prism-autoloader.min.js' as javascript,
+    './rss.sql' as rss,
     '/prism-tabler-theme.css' as css;
     
 SELECT 'text' AS component,
diff --git a/examples/official-site/index.sql b/examples/official-site/index.sql
index 7273b43e..b2498fb8 100644
--- a/examples/official-site/index.sql
+++ b/examples/official-site/index.sql
@@ -9,7 +9,8 @@ select 'shell' as component,
     'blog' as menu_item,
     'documentation' as menu_item,
     19 as font_size,
-    'Poppins' as font;
+    'Poppins' as font,
+    './rss.sql' as rss;
 
 SELECT 'hero' as component,
     'SQLPage' as title,
diff --git a/examples/official-site/rss.sql b/examples/official-site/rss.sql
new file mode 100644
index 00000000..3bbe4153
--- /dev/null
+++ b/examples/official-site/rss.sql
@@ -0,0 +1,21 @@
+select 'http_header' as component,
+    'application/rss+xml' as "Content-Type";
+select 'shell-empty' as component;
+select 'rss' as component,
+    'SQLPage blog' as title,
+    'https://sql.ophir.dev/blog.sql' as link,
+    'latest news about SQLpage' as description,
+    'en' as language,
+    'https://sql.ophir.dev/rss.sql' as self_link,
+    'Technology' as category,
+    '2de3f968-9928-5ec6-9653-6fc6fe382cfd' as guid;
+SELECT title,
+    description,
+    CASE
+        WHEN external_url IS NOT NULL THEN external_url
+        ELSE 'https://sql.ophir.dev/blog.sql?post=' || title
+    END AS link,
+    created_at AS date,
+    false AS explicit
+FROM blog_posts
+ORDER BY created_at DESC;
\ No newline at end of file
diff --git a/examples/official-site/sqlpage/migrations/01_documentation.sql b/examples/official-site/sqlpage/migrations/01_documentation.sql
index 5ce9d6b9..650622b9 100644
--- a/examples/official-site/sqlpage/migrations/01_documentation.sql
+++ b/examples/official-site/sqlpage/migrations/01_documentation.sql
@@ -723,6 +723,7 @@ INSERT INTO parameter(component, name, description, type, top_level, optional) S
     ('link', 'The target of the link in the top navigation bar.', 'URL', TRUE, TRUE),
     ('css', 'The URL of a CSS file to load and apply to the page.', 'URL', TRUE, TRUE),
     ('javascript', 'The URL of a Javascript file to load and execute on the page.', 'URL', TRUE, TRUE),
+    ('rss', 'The URL of an RSS feed to display in the top navigation bar. You can use the rss component to generate the field.', 'URL', TRUE, TRUE),
     ('image', 'The URL of an image to display next to the page title.', 'URL', TRUE, TRUE),
     ('icon', 'Name of an icon (from tabler-icons.io) to display next to the title in the navigation bar.', 'ICON', TRUE, TRUE),
     ('menu_item', 'Adds a menu item in the navigation bar at the top of the page. The menu item will have the specified name, and will link to as .sql file of the same name. A dropdown can be generated by passing a json object with a `title` and `submenu` properties.', 'TEXT', TRUE, TRUE),
diff --git a/examples/official-site/sqlpage/migrations/37_rss.sql b/examples/official-site/sqlpage/migrations/37_rss.sql
new file mode 100644
index 00000000..0af9fa69
--- /dev/null
+++ b/examples/official-site/sqlpage/migrations/37_rss.sql
@@ -0,0 +1,273 @@
+-- Documentation for the RSS component
+INSERT INTO component (name, description, icon, introduced_in_version) VALUES (
+    'rss',
+    'Produces a data flow in the RSS format.
+Can be used to generate a podcast feed.
+To use this component, you must first return an HTTP header with the "application/rss+xml" content type (see http_header component). Next, you must use the shell-empty component to avoid that SQLPage generates HTML code.',
+    'rss',
+    '0.20.0'
+);
+
+INSERT INTO parameter (component,name,description,type,top_level,optional) VALUES (
+    'rss',
+    'title',
+    'Defines the title of the channel.',
+    'TEXT',
+    TRUE,
+    FALSE
+),(
+    'rss',
+    'link',
+    'Defines the hyperlink to the channel.',
+    'URL',
+    TRUE,
+    FALSE
+),(
+    'rss',
+    'description',
+    'Describes the channel.',
+    'TEXT',
+    TRUE,
+    FALSE
+),(
+    'rss',
+    'language',
+    'Defines the language of the channel, specified in the ISO 639 format. For example, "en" for English, "fr" for French.',
+    'TEXT',
+    TRUE,
+    TRUE
+),(
+    'rss',
+    'category',
+    'Defines the category of the channel. The value should be a string representing the category (e.g., "News", "Technology", etc.).',
+    'TEXT',
+    TRUE,
+    TRUE
+),(
+    'rss',
+    'explicit',
+    'Indicates whether the channel contains explicit content. The value can be either TRUE or FALSE.',
+    'BOOLEAN',
+    TRUE,
+    TRUE
+),(
+    'rss',
+    'image_url',
+    'Provides a URL linking to the artwork for the channel.',
+    'URL',
+    TRUE,
+    TRUE
+),(
+    'rss',
+    'author',
+    'Defines the group, person, or people responsible for creating the channel.',
+    'TEXT',
+    TRUE,
+    TRUE
+),(
+    'rss',
+    'copyright',
+    'Provides the copyright details for the channel.',
+    'TEXT',
+    TRUE,
+    TRUE
+),(
+    'rss',
+    'self_link',
+    'URL of the RSS feed.',
+    'URL',
+    TRUE,
+    TRUE
+),(
+    'rss',
+    'funding_url',
+    'Specifies the donation/funding links for the channel. The content of the tag is the recommended string to be used with the link.',
+    'URL',
+    TRUE,
+    TRUE
+),(
+    'rss',
+    'type',
+    'Specifies the channel as either episodic or serial. The value can be either "episodic" or "serial".',
+    'TEXT',
+    TRUE,
+    TRUE
+),(
+    'rss',
+    'complete',
+    'Specifies that a channel is complete and will not post any more items in the future.',
+    'BOOLEAN',
+    TRUE,
+    TRUE
+),(
+    'rss',
+    'locked',
+    'Tells podcast hosting platforms whether they are allowed to import this feed.',
+    'BOOLEAN',
+    TRUE,
+    TRUE
+),(
+    'rss',
+    'guid',
+    'The globally unique identifier (GUID) for a channel. The value is a UUIDv5.',
+    'TEXT',
+    TRUE,
+    TRUE
+),(
+    'rss',
+    'title',
+    'Defines the title of the feed item (episode name, blog post title, etc.).',
+    'TEXT',
+    FALSE,
+    FALSE
+),(
+    'rss',
+    'link',
+    'Defines the hyperlink to the item (blog post URL, etc.).',
+    'URL',
+    FALSE,
+    FALSE
+),(
+    'rss',
+    'description',
+    'Describes the item',
+    'TEXT',
+    FALSE,
+    FALSE
+),(
+    'rss',
+    'date',
+    'Indicates when the item was published (RFC-822 date-time).',
+    'TEXT',
+    FALSE,
+    TRUE
+),(
+    'rss',
+    'enclosure_url',
+    'For podcast episodes, provides a URL linking to the audio/video episode content, in mp3, m4a, m4v, or mp4 format.',
+    'URL',
+    FALSE,
+    TRUE
+),(
+    'rss',
+    'enclosure_length',
+    'The length in bytes of the audio/video episode content.',
+    'INTEGER',
+    FALSE,
+    TRUE
+),(
+    'rss',
+    'enclosure_type',
+    'The MIME media type of the audio/video episode content (e.g., "audio/mpeg", "audio/m4a", "video/m4v", "video/mp4").',
+    'TEXT',
+    FALSE,
+    TRUE
+),(
+    'rss',
+    'guid',
+    'The globally unique identifier (GUID) for an item.',
+    'TEXT',
+    FALSE,
+    TRUE
+),(
+    'rss',
+    'episode',
+    'The chronological number that is associated with an item.',
+    'INTEGER',
+    FALSE,
+    TRUE
+),(
+    'rss',
+    'season',
+    'The chronological number associated with an item''s season.',
+    'INTEGER',
+    FALSE,
+    TRUE
+),(
+    'rss',
+    'episode_type',
+    'Defines the type of content for a specific item. The value can be either "full", "trailer", or "bonus".',
+    'TEXT',
+    FALSE,
+    TRUE
+),(
+    'rss',
+    'block',
+    'Prevents a specific item from appearing in podcast listening applications.',
+    'BOOLEAN',
+    FALSE,
+    TRUE
+),(
+    'rss',
+    'explicit',
+    'Indicates whether the item contains explicit content. The value can be either TRUE or FALSE.',
+    'BOOLEAN',
+    FALSE,
+    TRUE
+),(
+    'rss',
+    'image_url',
+    'Provides a URL linking to the artwork for the item.',
+    'URL',
+    FALSE,
+    TRUE
+),(
+    'rss',
+    'duration',
+    'The duration of an item in seconds.',
+    'INTEGER',
+    FALSE,
+    TRUE
+),(
+    'rss',
+    'transcript_url',
+    'A link to a transcript or closed captions file for the item.',
+    'URL',
+    FALSE,
+    TRUE
+),(
+    'rss',
+    'transcript_type',
+    'The type of the transcript or closed captions file for the item (e.g., "text/plain", "text/html", "text/vtt", "application/json", "application/x-subrip").',
+    'TEXT',
+    FALSE,
+    TRUE
+);
+
+-- Insert example(s) for the component
+INSERT INTO example (component, description)
+VALUES (
+        'rss',
+        '
+### An RSS channel about SQLPage latest news.
+
+```sql
+select ''http_header'' as component, ''application/rss+xml'' as content_type;
+select ''shell-empty'' as component;
+select
+  ''rss'' as component,
+  ''SQLPage blog'' as title,
+  ''https://sql.ophir.dev/blog.sql'' as link,
+  ''latest news about SQLpage'' as description,
+  ''en'' as language,
+  ''Technology'' as category,
+  FALSE as explicit,
+  ''https://sql.ophir.dev/favicon.ico'' as image_url,
+  ''Ophir Lojkine'' as author,
+  ''https://github.com/sponsors/lovasoa'' as funding_url,
+  ''episodic'' as type;
+select
+  ''Hello everyone !'' as title,
+  ''https://sql.ophir.dev/blog.sql?post=Come%20see%20me%20build%20twitter%20live%20on%20stage%20in%20Prague'' as link,
+  ''If some of you european SQLPagers are around Prague this december, I will be giving a talk about SQLPage at pgconf.eu on December 14th.'' as description,
+  ''http://127.0.0.1:8080/sqlpage_introduction_video.webm'' as enclosure_url,
+  123456789 as enclosure_length,
+  ''video/webm'' as enclosure_type,
+  ''2023-12-04'' as date;
+```
+
+Once you have your rss feed ready, you can submit it to podcast directories like
+[Apple Podcasts](https://podcastsconnect.apple.com/my-podcasts),
+[Spotify](https://podcasters.spotify.com/),
+[Google Podcasts](https://podcastsmanager.google.com/)...
+');
\ No newline at end of file
diff --git a/sqlpage/templates/rss.handlebars b/sqlpage/templates/rss.handlebars
new file mode 100644
index 00000000..779e911a
--- /dev/null
+++ b/sqlpage/templates/rss.handlebars
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="utf-8"?>
+<rss version="2.0"
+    xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd"
+    xmlns:podcast="https://podcastindex.org/namespace/1.0"
+    xmlns:atom="http://www.w3.org/2005/Atom"
+    xmlns:content="http://purl.org/rss/1.0/modules/content/">
+    <channel>
+        <title>{{title}}</title>
+        <link>{{link}}</link>
+        <description>{{description}}</description>
+        {{~#if language}}<language>{{language}}</language>{{/if}}
+        {{~#if category}}<itunes:category text="{{category}}">{{sub_category}}</itunes:category>{{/if}}
+        <itunes:explicit>{{#if explicit}}true{{else}}false{{/if}}</itunes:explicit>
+        {{~#if image_url}}<itunes:image href="{{image_url}}" />{{/if}}
+        {{~#if author}}<itunes:author>{{author}}</itunes:author>{{/if}}
+        {{~#if copyright}}<copyright>{{copyright}}</copyright>{{/if}}
+        {{~#if funding_url}}<podcast:funding url="{{funding_url}}">{{funding_text}}</podcast:funding>{{/if}}
+        {{~#if type}}<itunes:type>{{type}}</itunes:type>{{/if}}
+        {{~#if complete}}<itunes:complete>yes</itunes:complete>{{/if}}
+        {{~#if locked}}<podcast:locked>yes</podcast:locked>{{/if}}
+        {{~#if guid}}<podcast:guid>{{guid}}</podcast:guid>{{/if}}
+        {{~#if self_link}}<atom:link href="{{self_link}}" rel="self" type="application/rss+xml" />{{/if}}
+        {{#each_row}}
+        <item>
+            <title>{{title}}</title>
+            <link>{{link}}</link>
+            <description>{{description}}</description>
+            {{~#if date}}<pubDate>{{rfc2822_date date}}</pubDate>{{/if}}
+            {{~#if enclosure_url}}<enclosure length="{{enclosure_length}}" type="{{enclosure_type}}" url="{{enclosure_url}}" />{{/if}}
+            {{~#if guid}}<guid isPermaLink="false">{{guid}}</guid>{{/if}}
+            {{~#if episode}}<itunes:episode>{{episode}}</itunes:episode>{{/if}}
+            {{~#if season}}<itunes:season>{{season}}</itunes:season>{{/if}}
+            {{~#if episode_type}}<itunes:episodeType>{{episode_type}}</itunes:episodeType>{{/if}}
+            {{~#if block}}<itunes:block>yes</itunes:block>{{/if}}
+            {{~#if (not (eq explicit NULL))}}<itunes:explicit>{{#if explicit}}true{{else}}false{{/if}}</itunes:explicit>{{/if}}
+            {{~#if image_url}}<itunes:image href="{{image_url}}" />{{/if}}
+            {{~#if duration}}<itunes:duration>{{duration}}</itunes:duration>{{/if}}
+            {{~#if transcript_url}}<podcast:transcript url="{{transcript_url}}" type="{{transcript_type}}" />{{/if}}
+        </item>
+        {{/each_row}}
+    </channel>
+</rss>
diff --git a/sqlpage/templates/shell.handlebars b/sqlpage/templates/shell.handlebars
index 72d0c4b5..788cc12f 100644
--- a/sqlpage/templates/shell.handlebars
+++ b/sqlpage/templates/shell.handlebars
@@ -34,6 +34,9 @@
     {{#if refresh}}
         <meta http-equiv="refresh" content="{{refresh}}">
     {{/if}}
+    {{#if rss}}
+        <link rel="alternate" type="application/rss+xml" title="{{title}}" href="{{rss}}">
+    {{/if}}
     <meta name="generator" content="SQLPage"/>
 </head>
 
diff --git a/src/lib.rs b/src/lib.rs
index e20c2195..38845be9 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -7,6 +7,7 @@ pub mod app_config;
 pub mod file_cache;
 pub mod filesystem;
 pub mod render;
+pub mod template_helpers;
 pub mod templates;
 pub mod utils;
 pub mod webserver;
diff --git a/src/template_helpers.rs b/src/template_helpers.rs
new file mode 100644
index 00000000..6d356382
--- /dev/null
+++ b/src/template_helpers.rs
@@ -0,0 +1,383 @@
+use std::borrow::Cow;
+
+use anyhow::Context as _;
+use handlebars::{
+    handlebars_helper, Context, Handlebars, PathAndJson, RenderError, RenderErrorReason,
+    Renderable, ScopedJson,
+};
+use serde_json::Value as JsonValue;
+
+use crate::utils::static_filename;
+
+/// Simple json to json helper
+type H = fn(&JsonValue) -> JsonValue;
+/// Simple json to json helper with error handling
+type EH = fn(&JsonValue) -> anyhow::Result<JsonValue>;
+/// Helper that takes two arguments
+type HH = fn(&JsonValue, &JsonValue) -> JsonValue;
+
+pub fn register_all_helpers(h: &mut Handlebars<'_>) {
+    register_helper(h, "stringify", stringify_helper as H);
+    register_helper(h, "parse_json", parse_json_helper as EH);
+    register_helper(h, "default", default_helper as HH);
+    register_helper(h, "entries", entries_helper as H);
+    // delay helper: store a piece of information in memory that can be output later with flush_delayed
+    h.register_helper("delay", Box::new(delay_helper));
+    h.register_helper("flush_delayed", Box::new(flush_delayed_helper));
+    register_helper(h, "plus", plus_helper as HH);
+    register_helper(h, "plus", minus_helper as HH);
+    h.register_helper("sum", Box::new(sum_helper));
+    register_helper(h, "starts_with", starts_with_helper as HH);
+
+    // to_array: convert a value to a single-element array. If the value is already an array, return it as-is.
+    register_helper(h, "to_array", to_array_helper as H);
+
+    // array_contains: check if an array contains an element. If the first argument is not an array, it is compared to the second argument.
+    handlebars_helper!(array_contains: |array: Json, element: Json| match array {
+        JsonValue::Array(arr) => arr.contains(element),
+        other => other == element
+    });
+    h.register_helper("array_contains", Box::new(array_contains));
+
+    // static_path helper: generate a path to a static file. Replaces sqpage.js by sqlpage.<hash>.js
+    register_helper(h, "static_path", static_path_helper as EH);
+
+    // icon helper: generate an image with the specified icon
+    h.register_helper("icon_img", Box::new(icon_img_helper));
+    register_helper(h, "markdown", markdown_helper as EH);
+    register_helper(h, "buildinfo", buildinfo_helper as EH);
+    register_helper(h, "typeof", typeof_helper as H);
+    register_helper(h, "rfc2822_date", rfc2822_date_helper as EH);
+}
+
+fn stringify_helper(v: &JsonValue) -> JsonValue {
+    v.to_string().into()
+}
+
+fn parse_json_helper(v: &JsonValue) -> Result<JsonValue, anyhow::Error> {
+    Ok(match v {
+        serde_json::value::Value::String(s) => serde_json::from_str(s)?,
+        other => other.clone(),
+    })
+}
+
+fn default_helper(v: &JsonValue, default: &JsonValue) -> JsonValue {
+    if v.is_null() {
+        default.clone()
+    } else {
+        v.clone()
+    }
+}
+
+fn plus_helper(a: &JsonValue, b: &JsonValue) -> JsonValue {
+    if let (Some(a), Some(b)) = (a.as_i64(), b.as_i64()) {
+        (a + b).into()
+    } else if let (Some(a), Some(b)) = (a.as_f64(), b.as_f64()) {
+        (a + b).into()
+    } else {
+        JsonValue::Null
+    }
+}
+
+fn minus_helper(a: &JsonValue, b: &JsonValue) -> JsonValue {
+    if let (Some(a), Some(b)) = (a.as_i64(), b.as_i64()) {
+        (a - b).into()
+    } else if let (Some(a), Some(b)) = (a.as_f64(), b.as_f64()) {
+        (a - b).into()
+    } else {
+        JsonValue::Null
+    }
+}
+
+fn starts_with_helper(a: &JsonValue, b: &JsonValue) -> JsonValue {
+    if let (Some(a), Some(b)) = (a.as_str(), b.as_str()) {
+        a.starts_with(b)
+    } else if let (Some(arr1), Some(arr2)) = (a.as_array(), b.as_array()) {
+        arr1.starts_with(arr2)
+    } else {
+        false
+    }
+    .into()
+}
+fn entries_helper(v: &JsonValue) -> JsonValue {
+    match v {
+        serde_json::value::Value::Object(map) => map
+            .into_iter()
+            .map(|(k, v)| serde_json::json!({"key": k, "value": v}))
+            .collect(),
+        serde_json::value::Value::Array(values) => values
+            .iter()
+            .enumerate()
+            .map(|(k, v)| serde_json::json!({"key": k, "value": v}))
+            .collect(),
+        _ => vec![],
+    }
+    .into()
+}
+
+fn to_array_helper(v: &JsonValue) -> JsonValue {
+    match v {
+        JsonValue::Array(arr) => arr.clone(),
+        JsonValue::Null => vec![],
+        JsonValue::String(s) if s.starts_with('[') => {
+            if let Ok(JsonValue::Array(r)) = serde_json::from_str(s) {
+                r
+            } else {
+                vec![JsonValue::String(s.clone())]
+            }
+        }
+        other => vec![other.clone()],
+    }
+    .into()
+}
+
+fn static_path_helper(v: &JsonValue) -> anyhow::Result<JsonValue> {
+    match v.as_str().with_context(|| "static_path: not a string")? {
+        "sqlpage.js" => Ok(static_filename!("sqlpage.js").into()),
+        "sqlpage.css" => Ok(static_filename!("sqlpage.css").into()),
+        "apexcharts.js" => Ok(static_filename!("apexcharts.js").into()),
+        other => Err(anyhow::anyhow!("unknown static file: {other:?}")),
+    }
+}
+
+fn typeof_helper(v: &JsonValue) -> JsonValue {
+    match v {
+        JsonValue::Null => "null",
+        JsonValue::Bool(_) => "boolean",
+        JsonValue::Number(_) => "number",
+        JsonValue::String(_) => "string",
+        JsonValue::Array(_) => "array",
+        JsonValue::Object(_) => "object",
+    }
+    .into()
+}
+
+fn markdown_helper(x: &JsonValue) -> anyhow::Result<JsonValue> {
+    let as_str = match x {
+        JsonValue::String(s) => Cow::Borrowed(s),
+        JsonValue::Array(arr) => Cow::Owned(
+            arr.iter()
+                .map(|v| v.as_str().unwrap_or_default())
+                .collect::<Vec<_>>()
+                .join("\n"),
+        ),
+        JsonValue::Null => Cow::Owned(String::new()),
+        other => Cow::Owned(other.to_string()),
+    };
+    markdown::to_html_with_options(&as_str, &markdown::Options::gfm())
+        .map(JsonValue::String)
+        .map_err(|e| anyhow::anyhow!("markdown error: {e}"))
+}
+
+fn buildinfo_helper(x: &JsonValue) -> anyhow::Result<JsonValue> {
+    match x {
+        JsonValue::String(s) if s == "CARGO_PKG_NAME" => Ok(env!("CARGO_PKG_NAME").into()),
+        JsonValue::String(s) if s == "CARGO_PKG_VERSION" => Ok(env!("CARGO_PKG_VERSION").into()),
+        other => Err(anyhow::anyhow!("unknown buildinfo key: {other:?}")),
+    }
+}
+
+// rfc2822_date: take an ISO date and convert it to an RFC 2822 date
+fn rfc2822_date_helper(v: &JsonValue) -> anyhow::Result<JsonValue> {
+    let date: chrono::DateTime<chrono::FixedOffset> = match v {
+        JsonValue::String(s) => {
+            // we accept both dates with and without time
+            chrono::DateTime::parse_from_rfc3339(s)
+                .or_else(|_| {
+                    chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d")
+                        .map(|d| d.and_hms_opt(0, 0, 0).unwrap().and_utc().fixed_offset())
+                })
+                .with_context(|| format!("invalid date: {s}"))?
+        }
+        JsonValue::Number(n) => {
+            chrono::DateTime::from_timestamp(n.as_i64().with_context(|| "not a timestamp")?, 0)
+                .with_context(|| "invalid timestamp")?
+                .into()
+        }
+        other => anyhow::bail!("expected a date, got {other:?}"),
+    };
+    // format: Thu, 01 Jan 1970 00:00:00 +0000
+    Ok(date.format("%a, %d %b %Y %T %z").to_string().into())
+}
+
+fn with_each_block<'a, 'reg, 'rc>(
+    rc: &'a mut handlebars::RenderContext<'reg, 'rc>,
+    mut action: impl FnMut(&mut handlebars::BlockContext<'rc>, bool) -> Result<(), RenderError>,
+) -> Result<(), RenderError> {
+    let mut blks = Vec::new();
+    while let Some(mut top) = rc.block_mut().map(std::mem::take) {
+        rc.pop_block();
+        action(&mut top, rc.block().is_none())?;
+        blks.push(top);
+    }
+    while let Some(blk) = blks.pop() {
+        rc.push_block(blk);
+    }
+    Ok(())
+}
+
+pub(crate) const DELAYED_CONTENTS: &str = "_delayed_contents";
+
+fn delay_helper<'reg, 'rc>(
+    h: &handlebars::Helper<'rc>,
+    r: &'reg Handlebars<'reg>,
+    ctx: &'rc Context,
+    rc: &mut handlebars::RenderContext<'reg, 'rc>,
+    _out: &mut dyn handlebars::Output,
+) -> handlebars::HelperResult {
+    let inner = h
+        .template()
+        .ok_or(RenderErrorReason::BlockContentRequired)?;
+    let mut str_out = handlebars::StringOutput::new();
+    inner.render(r, ctx, rc, &mut str_out)?;
+    let mut delayed_render = str_out.into_string()?;
+    with_each_block(rc, |block, is_last| {
+        if is_last {
+            let old_delayed_render = block
+                .get_local_var(DELAYED_CONTENTS)
+                .and_then(JsonValue::as_str)
+                .unwrap_or_default();
+            delayed_render += old_delayed_render;
+            let contents = JsonValue::String(std::mem::take(&mut delayed_render));
+            block.set_local_var(DELAYED_CONTENTS, contents);
+        }
+        Ok(())
+    })?;
+    Ok(())
+}
+
+fn flush_delayed_helper<'reg, 'rc>(
+    _h: &handlebars::Helper<'rc>,
+    _r: &'reg Handlebars<'reg>,
+    _ctx: &'rc Context,
+    rc: &mut handlebars::RenderContext<'reg, 'rc>,
+    writer: &mut dyn handlebars::Output,
+) -> handlebars::HelperResult {
+    with_each_block(rc, |block_context, _last| {
+        let delayed = block_context
+            .get_local_var(DELAYED_CONTENTS)
+            .and_then(JsonValue::as_str)
+            .filter(|s| !s.is_empty());
+        if let Some(contents) = delayed {
+            writer.write(contents)?;
+            block_context.set_local_var(DELAYED_CONTENTS, JsonValue::Null);
+        }
+        Ok(())
+    })
+}
+
+fn sum_helper<'reg, 'rc>(
+    helper: &handlebars::Helper<'rc>,
+    _r: &'reg Handlebars<'reg>,
+    _ctx: &'rc Context,
+    _rc: &mut handlebars::RenderContext<'reg, 'rc>,
+    writer: &mut dyn handlebars::Output,
+) -> handlebars::HelperResult {
+    let mut sum = 0f64;
+    for v in helper.params() {
+        sum += v
+            .value()
+            .as_f64()
+            .ok_or(RenderErrorReason::InvalidParamType("number"))?;
+    }
+    write!(writer, "{sum}")?;
+    Ok(())
+}
+
+fn icon_img_helper<'reg, 'rc>(
+    helper: &handlebars::Helper<'rc>,
+    _r: &'reg Handlebars<'reg>,
+    _ctx: &'rc Context,
+    _rc: &mut handlebars::RenderContext<'reg, 'rc>,
+    writer: &mut dyn handlebars::Output,
+) -> handlebars::HelperResult {
+    let null = handlebars::JsonValue::Null;
+    let params = [0, 1].map(|i| helper.params().get(i).map_or(&null, PathAndJson::value));
+    let name = match params[0] {
+        JsonValue::String(s) => s,
+        other => {
+            log::debug!("icon_img: {other:?} is not an icon name, not rendering anything");
+            return Ok(());
+        }
+    };
+    let size = params[1].as_u64().unwrap_or(24);
+    write!(
+        writer,
+        "<svg width={size} height={size}><use href=\"/{}#tabler-{name}\" /></svg>",
+        static_filename!("tabler-icons.svg")
+    )?;
+    Ok(())
+}
+
+trait CanHelp: Send + Sync + 'static {
+    fn call(&self, v: &[PathAndJson]) -> Result<JsonValue, String>;
+}
+
+impl CanHelp for H {
+    fn call(&self, args: &[PathAndJson]) -> Result<JsonValue, String> {
+        match args {
+            [v] => Ok(self(v.value())),
+            _ => Err("expected one argument".to_string()),
+        }
+    }
+}
+
+impl CanHelp for EH {
+    fn call(&self, args: &[PathAndJson]) -> Result<JsonValue, String> {
+        match args {
+            [v] => self(v.value()).map_err(|e| e.to_string()),
+            _ => Err("expected one argument".to_string()),
+        }
+    }
+}
+
+impl CanHelp for HH {
+    fn call(&self, args: &[PathAndJson]) -> Result<JsonValue, String> {
+        match args {
+            [a, b] => Ok(self(a.value(), b.value())),
+            _ => Err("expected two arguments".to_string()),
+        }
+    }
+}
+
+struct JFun<F: CanHelp> {
+    name: &'static str,
+    fun: F,
+}
+impl<F: CanHelp> handlebars::HelperDef for JFun<F> {
+    fn call_inner<'reg: 'rc, 'rc>(
+        &self,
+        helper: &handlebars::Helper<'rc>,
+        _r: &'reg Handlebars<'reg>,
+        _: &'rc Context,
+        _rc: &mut handlebars::RenderContext<'reg, 'rc>,
+    ) -> Result<handlebars::ScopedJson<'rc>, RenderError> {
+        let result = self
+            .fun
+            .call(helper.params().as_slice())
+            .map_err(|s| RenderErrorReason::Other(format!("{}: {}", self.name, s)))?;
+        Ok(ScopedJson::Derived(result))
+    }
+}
+
+fn register_helper(h: &mut Handlebars, name: &'static str, fun: impl CanHelp) {
+    h.register_helper(name, Box::new(JFun { name, fun }));
+}
+
+#[test]
+fn test_rfc2822_date() {
+    assert_eq!(
+        rfc2822_date_helper(&JsonValue::String("1970-01-02T03:04:05+02:00".into()))
+            .unwrap()
+            .as_str()
+            .unwrap(),
+        "Fri, 02 Jan 1970 03:04:05 +0200"
+    );
+    assert_eq!(
+        rfc2822_date_helper(&JsonValue::String("1970-01-02".into()))
+            .unwrap()
+            .as_str()
+            .unwrap(),
+        "Fri, 02 Jan 1970 00:00:00 +0000"
+    );
+}
diff --git a/src/templates.rs b/src/templates.rs
index 877b9314..3bd74d61 100644
--- a/src/templates.rs
+++ b/src/templates.rs
@@ -1,19 +1,12 @@
 use crate::file_cache::AsyncFromStrWithState;
-use crate::utils::static_filename;
+use crate::template_helpers::register_all_helpers;
 use crate::{AppState, FileCache, TEMPLATES_DIR};
 use async_trait::async_trait;
-use handlebars::{
-    handlebars_helper, template::TemplateElement, Context, Handlebars, JsonValue, RenderError,
-    Renderable, Template,
-};
-use handlebars::{PathAndJson, RenderErrorReason};
+use handlebars::{template::TemplateElement, Handlebars, Template};
 use include_dir::{include_dir, Dir};
-use std::borrow::Cow;
 use std::path::PathBuf;
 use std::sync::Arc;
 
-pub(crate) const DELAYED_CONTENTS: &str = "_delayed_contents";
-
 pub struct SplitTemplate {
     pub before_list: Template,
     pub list_content: Template,
@@ -79,234 +72,12 @@ pub struct AllTemplates {
     split_templates: FileCache<SplitTemplate>,
 }
 
-fn with_each_block<'a, 'reg, 'rc>(
-    rc: &'a mut handlebars::RenderContext<'reg, 'rc>,
-    mut action: impl FnMut(&mut handlebars::BlockContext<'rc>, bool) -> Result<(), RenderError>,
-) -> Result<(), RenderError> {
-    let mut blks = Vec::new();
-    while let Some(mut top) = rc.block_mut().map(std::mem::take) {
-        rc.pop_block();
-        action(&mut top, rc.block().is_none())?;
-        blks.push(top);
-    }
-    while let Some(blk) = blks.pop() {
-        rc.push_block(blk);
-    }
-    Ok(())
-}
-
-fn delay_helper<'reg, 'rc>(
-    h: &handlebars::Helper<'rc>,
-    r: &'reg Handlebars<'reg>,
-    ctx: &'rc Context,
-    rc: &mut handlebars::RenderContext<'reg, 'rc>,
-    _out: &mut dyn handlebars::Output,
-) -> handlebars::HelperResult {
-    let inner = h
-        .template()
-        .ok_or(RenderErrorReason::BlockContentRequired)?;
-    let mut str_out = handlebars::StringOutput::new();
-    inner.render(r, ctx, rc, &mut str_out)?;
-    let mut delayed_render = str_out.into_string()?;
-    with_each_block(rc, |block, is_last| {
-        if is_last {
-            let old_delayed_render = block
-                .get_local_var(DELAYED_CONTENTS)
-                .and_then(JsonValue::as_str)
-                .unwrap_or_default();
-            delayed_render += old_delayed_render;
-            let contents = JsonValue::String(std::mem::take(&mut delayed_render));
-            block.set_local_var(DELAYED_CONTENTS, contents);
-        }
-        Ok(())
-    })?;
-    Ok(())
-}
-
-fn flush_delayed_helper<'reg, 'rc>(
-    _h: &handlebars::Helper<'rc>,
-    _r: &'reg Handlebars<'reg>,
-    _ctx: &'rc Context,
-    rc: &mut handlebars::RenderContext<'reg, 'rc>,
-    writer: &mut dyn handlebars::Output,
-) -> handlebars::HelperResult {
-    with_each_block(rc, |block_context, _last| {
-        let delayed = block_context
-            .get_local_var(DELAYED_CONTENTS)
-            .and_then(JsonValue::as_str)
-            .filter(|s| !s.is_empty());
-        if let Some(contents) = delayed {
-            writer.write(contents)?;
-            block_context.set_local_var(DELAYED_CONTENTS, JsonValue::Null);
-        }
-        Ok(())
-    })
-}
-
-fn sum_helper<'reg, 'rc>(
-    helper: &handlebars::Helper<'rc>,
-    _r: &'reg Handlebars<'reg>,
-    _ctx: &'rc Context,
-    _rc: &mut handlebars::RenderContext<'reg, 'rc>,
-    writer: &mut dyn handlebars::Output,
-) -> handlebars::HelperResult {
-    let mut sum = 0f64;
-    for v in helper.params() {
-        sum += v
-            .value()
-            .as_f64()
-            .ok_or(RenderErrorReason::InvalidParamType("number"))?;
-    }
-    write!(writer, "{sum}")?;
-    Ok(())
-}
-
-fn icon_img_helper<'reg, 'rc>(
-    helper: &handlebars::Helper<'rc>,
-    _r: &'reg Handlebars<'reg>,
-    _ctx: &'rc Context,
-    _rc: &mut handlebars::RenderContext<'reg, 'rc>,
-    writer: &mut dyn handlebars::Output,
-) -> handlebars::HelperResult {
-    let null = handlebars::JsonValue::Null;
-    let params = [0, 1].map(|i| helper.params().get(i).map_or(&null, PathAndJson::value));
-    let name = match params[0] {
-        JsonValue::String(s) => s,
-        other => {
-            log::debug!("icon_img: {other:?} is not an icon name, not rendering anything");
-            return Ok(());
-        }
-    };
-    let size = params[1].as_u64().unwrap_or(24);
-    write!(
-        writer,
-        "<svg width={size} height={size}><use href=\"/{}#tabler-{name}\" /></svg>",
-        static_filename!("tabler-icons.svg")
-    )?;
-    Ok(())
-}
-
 const STATIC_TEMPLATES: Dir = include_dir!("$CARGO_MANIFEST_DIR/sqlpage/templates");
 
 impl AllTemplates {
     pub fn init() -> anyhow::Result<Self> {
         let mut handlebars = Handlebars::new();
-
-        handlebars_helper!(stringify: |v: Json| v.to_string());
-        handlebars.register_helper("stringify", Box::new(stringify));
-
-        handlebars_helper!(parse_json: |v: Json| match v {
-            obj @ serde_json::value::Value::String(s) =>
-                serde_json::from_str(s)
-                .unwrap_or_else(|_| {
-                    log::warn!("Failed to parse JSON string: {}", s);
-                    obj.clone()
-                }),
-            other => other.clone()
-        });
-        handlebars.register_helper("parse_json", Box::new(parse_json));
-
-        handlebars_helper!(default: |a: Json, b:Json| if a.is_null() {b} else {a}.clone());
-        handlebars.register_helper("default", Box::new(default));
-
-        handlebars_helper!(entries: |v: Json | match v {
-            serde_json::value::Value::Object(map) =>
-                map.into_iter()
-                    .map(|(k, v)| serde_json::json!({"key": k, "value": v}))
-                    .collect(),
-            serde_json::value::Value::Array(values) =>
-                values.iter()
-                    .enumerate()
-                    .map(|(k, v)| serde_json::json!({"key": k, "value": v}))
-                    .collect(),
-            _ => vec![]
-        });
-
-        handlebars.register_helper("entries", Box::new(entries));
-
-        // delay helper: store a piece of information in memory that can be output later with flush_delayed
-        handlebars.register_helper("delay", Box::new(delay_helper));
-        handlebars.register_helper("flush_delayed", Box::new(flush_delayed_helper));
-
-        handlebars_helper!(plus: |a: Json, b:Json| a.as_i64().unwrap_or_default() + b.as_i64().unwrap_or_default());
-        handlebars.register_helper("plus", Box::new(plus));
-
-        handlebars_helper!(minus: |a: Json, b:Json| a.as_i64().unwrap_or_default() - b.as_i64().unwrap_or_default());
-        handlebars.register_helper("minus", Box::new(minus));
-
-        handlebars.register_helper("sum", Box::new(sum_helper));
-
-        handlebars_helper!(starts_with: |s: str, prefix:str| s.starts_with(prefix));
-        handlebars.register_helper("starts_with", Box::new(starts_with));
-
-        // to_array: convert a value to a single-element array. If the value is already an array, return it as-is.
-        handlebars_helper!(to_array: |x: Json| match x {
-            JsonValue::Array(arr) => arr.clone(),
-            JsonValue::Null => vec![],
-            JsonValue::String(s) if s.starts_with('[')  => {
-                if let Ok(JsonValue::Array(r)) = serde_json::from_str(s) {
-                    r
-                } else {
-                    vec![JsonValue::String(s.clone())]
-                }
-            }
-            other => vec![other.clone()]
-        });
-        handlebars.register_helper("to_array", Box::new(to_array));
-
-        // array_contains: check if an array contains an element. If the first argument is not an array, it is compared to the second argument.
-        handlebars_helper!(array_contains: |array: Json, element: Json| match array {
-            JsonValue::Array(arr) => arr.contains(element),
-            other => other == element
-        });
-        handlebars.register_helper("array_contains", Box::new(array_contains));
-
-        // static_path helper: generate a path to a static file. Replaces sqpage.js by sqlpage.<hash>.js
-        handlebars_helper!(static_path: |x: str| match x {
-            "sqlpage.js" => static_filename!("sqlpage.js"),
-            "sqlpage.css" => static_filename!("sqlpage.css"),
-            "apexcharts.js" => static_filename!("apexcharts.js"),
-            unknown => {
-                log::error!("Unknown static path: {}", unknown);
-                "!!unknown static path!!"
-            }
-        });
-        handlebars.register_helper("static_path", Box::new(static_path));
-
-        // icon helper: generate an image with the specified icon
-        handlebars.register_helper("icon_img", Box::new(icon_img_helper));
-
-        handlebars_helper!(markdown_helper: |x: Json| {
-            let as_str = match x {
-                JsonValue::String(s) => Cow::Borrowed(s),
-                JsonValue::Array(arr) => Cow::Owned(arr.iter().map(|v|v.as_str().unwrap_or_default()).collect::<Vec<_>>().join("\n")),
-                JsonValue::Null => Cow::Owned(String::new()),
-                other => Cow::Owned(other.to_string())
-            };
-            markdown::to_html_with_options(&as_str, &markdown::Options::gfm())
-            .unwrap_or_else(|s|s)
-        });
-        handlebars.register_helper("markdown", Box::new(markdown_helper));
-
-        handlebars_helper!(buildinfo_helper: |x: str|
-            match x {
-                "CARGO_PKG_NAME" => env!("CARGO_PKG_NAME"),
-                "CARGO_PKG_VERSION" => env!("CARGO_PKG_VERSION"),
-                _ => "!!unknown buildinfo key!!"
-            }
-        );
-        handlebars.register_helper("buildinfo", Box::new(buildinfo_helper));
-
-        handlebars_helper!(typeof_helper: |x: Json| match x {
-            JsonValue::Null => "null",
-            JsonValue::Bool(_) => "boolean",
-            JsonValue::Number(_) => "number",
-            JsonValue::String(_) => "string",
-            JsonValue::Array(_) => "array",
-            JsonValue::Object(_) => "object",
-        });
-        handlebars.register_helper("typeof", Box::new(typeof_helper));
-
+        register_all_helpers(&mut handlebars);
         let mut this = Self {
             handlebars,
             split_templates: FileCache::new(),

From 7738895b6b8efe08c0fa2a31ed5c03ecdfe1c472 Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Tue, 5 Mar 2024 19:44:40 +0100
Subject: [PATCH 05/33] changelog: rss

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7481051b..495065be 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,7 @@
 ## 0.20.0 (unreleased)
 
  - Add an `id` attribute to form fields in the [form](https://sql.ophir.dev/documentation.sql?component=form#component) component. This allows you to easily reference form fields in custom javascript code.
+ - New [`rss`](https://sql.ophir.dev/documentation.sql?component=rss#component) component to create RSS feeds, including **podcast feeds**. You can now create and manage your podcast feed entirely in SQL, and distribute it to all podcast directories such as Apple Podcasts, Spotify, and Google Podcasts.
 
 ## 0.19.1 (2024-02-28)
   - **SECURITY**: fixes users being able to re-run migrations by visiting `/sqlpage/migrations/NNNN_name.sql` pages. If you are using sqlpage migrations, your migrations are not idempotent, and you use the default SQLPAGE_WEB_ROOT (`./`) and `SQLPAGE_CONFIGURATION_DIRECTORY` (`./sqlpage/`), you should upgrade to this version as soon as possible. If you are using a custom `SQLPAGE_WEB_ROOT` or `SQLPAGE_CONFIGURATION_DIRECTORY` or your migrations are idempotent, you can upgrade at your convenience.

From 45e1d4e93f090550146175f14ed60b2521a51090 Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Tue, 5 Mar 2024 19:52:00 +0100
Subject: [PATCH 06/33] improve the documentation for the DEBUG component

closes #251
---
 .../official-site/sqlpage/migrations/06_debug.sql | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/examples/official-site/sqlpage/migrations/06_debug.sql b/examples/official-site/sqlpage/migrations/06_debug.sql
index eaefff66..7df4263b 100644
--- a/examples/official-site/sqlpage/migrations/06_debug.sql
+++ b/examples/official-site/sqlpage/migrations/06_debug.sql
@@ -2,7 +2,9 @@
 INSERT INTO component (name, description, icon)
 VALUES (
         'debug',
-        'Display all the parameters passed to the component. Useful for debugging: just replace the name of the component you want to debug with ''debug''.',
+        'Visualize any set of values as JSON.
+Can be used to display all the parameters passed to the component.
+Useful for debugging: just replace the name of the component you want to debug with ''debug'', and see all the top-level and row-level parameters that are passed to it, and their types.',
         'bug'
     );
 -- Insert an example usage of the http_header component into the example table
@@ -10,5 +12,14 @@ INSERT INTO example (component, description, properties)
 VALUES (
         'debug',
         'At any time, if you are confused about what data you are passing to a component, just replace the component name with ''debug'' to see all the parameters that are passed to it.',
-        JSON('[{"component": "debug"}, {"x": "y", "z": 42}, {"a": "b", "c": null}]')
+        JSON('[{"component": "debug", "my_top_level_property": true}, {"x": "y", "z": 42}, {"a": "b", "c": null}]')
+    ),
+    (
+        'debug',
+        'Show the result of a SQLPage function:
+        
+```sql
+select ''debug'' as component, sqlpage.environment_variable(''HOME'');
+```',
+        NULL
     );
\ No newline at end of file

From 565dc84246c409d561d5648eb5f2dbc5c90c01e1 Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Thu, 7 Mar 2024 11:56:44 +0100
Subject: [PATCH 07/33] initial run_sql implementatttion

---
 src/file_cache.rs                             |  8 ++--
 src/webserver/database/execute_queries.rs     | 11 +++++
 src/webserver/database/sql_pseudofunctions.rs | 44 +++++++++++++++++--
 src/webserver/http.rs                         |  2 +-
 src/webserver/http_request_info.rs            | 18 ++++++++
 5 files changed, 75 insertions(+), 8 deletions(-)

diff --git a/src/file_cache.rs b/src/file_cache.rs
index 63992140..90bdcf4f 100644
--- a/src/file_cache.rs
+++ b/src/file_cache.rs
@@ -5,7 +5,7 @@ use anyhow::Context;
 use async_trait::async_trait;
 use chrono::{DateTime, TimeZone, Utc};
 use std::collections::HashMap;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::sync::atomic::{
     AtomicU64,
     Ordering::{Acquire, Release},
@@ -97,7 +97,7 @@ impl<T: AsyncFromStrWithState> FileCache<T> {
 
     /// Gets a file from the cache, or loads it from the file system if it's not there
     /// This is a privileged operation; it should not be used for user-provided paths
-    pub async fn get(&self, app_state: &AppState, path: &PathBuf) -> anyhow::Result<Arc<T>> {
+    pub async fn get(&self, app_state: &AppState, path: &Path) -> anyhow::Result<Arc<T>> {
         self.get_with_privilege(app_state, path, true).await
     }
 
@@ -107,7 +107,7 @@ impl<T: AsyncFromStrWithState> FileCache<T> {
     pub async fn get_with_privilege(
         &self,
         app_state: &AppState,
-        path: &PathBuf,
+        path: &Path,
         privileged: bool,
     ) -> anyhow::Result<Arc<T>> {
         log::trace!("Attempting to get from cache {:?}", path);
@@ -164,7 +164,7 @@ impl<T: AsyncFromStrWithState> FileCache<T> {
             Ok(value) => {
                 let new_val = Arc::clone(&value.content);
                 log::trace!("Writing to cache {:?}", path);
-                self.cache.write().await.insert(path.clone(), value);
+                self.cache.write().await.insert(PathBuf::from(path), value);
                 log::trace!("Done writing to cache {:?}", path);
                 log::trace!("{:?} loaded in cache", path);
                 Ok(new_val)
diff --git a/src/webserver/database/execute_queries.rs b/src/webserver/database/execute_queries.rs
index 928a7b32..d3dfbd35 100644
--- a/src/webserver/database/execute_queries.rs
+++ b/src/webserver/database/execute_queries.rs
@@ -3,6 +3,7 @@ use futures_util::stream::Stream;
 use futures_util::StreamExt;
 use std::borrow::Cow;
 use std::collections::HashMap;
+use std::pin::Pin;
 
 use super::csv_import::run_csv_import;
 use super::sql::{ParsedSqlFile, ParsedStatement, StmtWithParams};
@@ -83,6 +84,16 @@ pub fn stream_query_results<'a>(
     .map(|res| res.unwrap_or_else(DbItem::Error))
 }
 
+/// This function is used to create a pinned boxed stream of query results.
+/// This allows recursive calls.
+pub fn stream_query_results_boxed<'a>(
+    db: &'a Database,
+    sql_file: &'a ParsedSqlFile,
+    request: &'a mut RequestInfo,
+) -> Pin<Box<dyn Stream<Item = DbItem> + 'a>> {
+    Box::pin(stream_query_results(db, sql_file, request))
+}
+
 fn vars_and_name<'a>(
     request: &'a mut RequestInfo,
     variable: &StmtParam,
diff --git a/src/webserver/database/sql_pseudofunctions.rs b/src/webserver/database/sql_pseudofunctions.rs
index bf90de8b..f2908acd 100644
--- a/src/webserver/database/sql_pseudofunctions.rs
+++ b/src/webserver/database/sql_pseudofunctions.rs
@@ -5,8 +5,17 @@ use actix_web_httpauth::headers::authorization::Basic;
 use base64::Engine;
 use mime_guess::{mime::APPLICATION_OCTET_STREAM, Mime};
 use sqlparser::ast::FunctionArg;
+use tokio_stream::StreamExt;
 
-use crate::webserver::{http::SingleOrVec, http_request_info::RequestInfo, ErrorWithStatus};
+use crate::webserver::{
+    database::{
+        execute_queries::{stream_query_results, stream_query_results_boxed},
+        DbItem,
+    },
+    http::SingleOrVec,
+    http_request_info::RequestInfo,
+    ErrorWithStatus,
+};
 
 use super::sql::{
     extract_integer, extract_single_quoted_string, extract_single_quoted_string_optional,
@@ -255,11 +264,40 @@ async fn run_sql<'a>(
     param0: &StmtParam,
     request: &'a RequestInfo,
 ) -> Result<Option<Cow<'a, str>>, anyhow::Error> {
-    let Some(sql_file) = extract_req_param_non_nested(param0, request)? else {
+    let Some(sql_file_path) = extract_req_param_non_nested(param0, request)? else {
         log::debug!("run_sql: first argument is NULL, returning NULL");
         return Ok(None);
     };
-    todo!("run_sql: {sql_file:?}");
+    let sql_file = request
+        .app_state
+        .sql_file_cache
+        .get_with_privilege(
+            &request.app_state,
+            std::path::Path::new(sql_file_path.as_ref()),
+            true,
+        )
+        .await?;
+    let mut tmp_req = request.clone();
+    let mut results_stream = Box::pin(stream_query_results_boxed(
+        &request.app_state.db,
+        &sql_file,
+        &mut tmp_req,
+    ));
+    use serde::ser::{SerializeSeq, Serializer};
+    let mut json_results_bytes = Vec::new();
+    let mut json_encoder = serde_json::Serializer::new(&mut json_results_bytes);
+    let mut seq = json_encoder.serialize_seq(None)?;
+    while let Some(db_item) = results_stream.next().await {
+        match db_item {
+            DbItem::Row(row) => {
+                log::debug!("run_sql: row: {:?}", row);
+                seq.serialize_element(&row)?
+            }
+            DbItem::FinishedQuery => log::trace!("run_sql: Finished query"),
+            DbItem::Error(err) => return Err(err.into()),
+        }
+    }
+    Ok(Some(Cow::Owned(String::from_utf8(json_results_bytes)?)))
 }
 
 fn mime_from_upload<'a>(param0: &StmtParam, request: &'a RequestInfo) -> Option<&'a Mime> {
diff --git a/src/webserver/http.rs b/src/webserver/http.rs
index fa746a9e..71bbc83e 100644
--- a/src/webserver/http.rs
+++ b/src/webserver/http.rs
@@ -306,7 +306,7 @@ fn send_anyhow_error(
         .unwrap_or_else(|_| log::error!("could not send headers"));
 }
 
-#[derive(Debug, serde::Serialize, serde::Deserialize, PartialEq)]
+#[derive(Debug, serde::Serialize, serde::Deserialize, PartialEq, Clone)]
 #[serde(untagged)]
 pub enum SingleOrVec {
     Single(String),
diff --git a/src/webserver/http_request_info.rs b/src/webserver/http_request_info.rs
index b9a16768..480032f1 100644
--- a/src/webserver/http_request_info.rs
+++ b/src/webserver/http_request_info.rs
@@ -35,6 +35,24 @@ pub struct RequestInfo {
     pub app_state: Arc<AppState>,
 }
 
+impl Clone for RequestInfo {
+    fn clone(&self) -> Self {
+        Self {
+            path: self.path.clone(),
+            protocol: self.protocol.clone(),
+            get_variables: self.get_variables.clone(),
+            post_variables: self.post_variables.clone(),
+            // uploaded_files is not cloned, as it contains file handles
+            uploaded_files: HashMap::new(),
+            headers: self.headers.clone(),
+            client_ip: self.client_ip,
+            cookies: self.cookies.clone(),
+            basic_auth: self.basic_auth.clone(),
+            app_state: self.app_state.clone(),
+        }
+    }
+}
+
 pub(crate) async fn extract_request_info(
     req: &mut ServiceRequest,
     app_state: Arc<AppState>,

From 93c7cf28daa8c48ea6a1de151df907b431982f1c Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Thu, 7 Mar 2024 15:33:15 +0100
Subject: [PATCH 08/33] clippy

---
 src/webserver/database/sql_pseudofunctions.rs | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/src/webserver/database/sql_pseudofunctions.rs b/src/webserver/database/sql_pseudofunctions.rs
index f2908acd..1f4ced28 100644
--- a/src/webserver/database/sql_pseudofunctions.rs
+++ b/src/webserver/database/sql_pseudofunctions.rs
@@ -8,10 +8,7 @@ use sqlparser::ast::FunctionArg;
 use tokio_stream::StreamExt;
 
 use crate::webserver::{
-    database::{
-        execute_queries::{stream_query_results, stream_query_results_boxed},
-        DbItem,
-    },
+    database::{execute_queries::stream_query_results_boxed, DbItem},
     http::SingleOrVec,
     http_request_info::RequestInfo,
     ErrorWithStatus,
@@ -264,6 +261,7 @@ async fn run_sql<'a>(
     param0: &StmtParam,
     request: &'a RequestInfo,
 ) -> Result<Option<Cow<'a, str>>, anyhow::Error> {
+    use serde::ser::{SerializeSeq, Serializer};
     let Some(sql_file_path) = extract_req_param_non_nested(param0, request)? else {
         log::debug!("run_sql: first argument is NULL, returning NULL");
         return Ok(None);
@@ -283,7 +281,6 @@ async fn run_sql<'a>(
         &sql_file,
         &mut tmp_req,
     ));
-    use serde::ser::{SerializeSeq, Serializer};
     let mut json_results_bytes = Vec::new();
     let mut json_encoder = serde_json::Serializer::new(&mut json_results_bytes);
     let mut seq = json_encoder.serialize_seq(None)?;
@@ -291,10 +288,10 @@ async fn run_sql<'a>(
         match db_item {
             DbItem::Row(row) => {
                 log::debug!("run_sql: row: {:?}", row);
-                seq.serialize_element(&row)?
+                seq.serialize_element(&row)?;
             }
             DbItem::FinishedQuery => log::trace!("run_sql: Finished query"),
-            DbItem::Error(err) => return Err(err.into()),
+            DbItem::Error(err) => return Err(err),
         }
     }
     Ok(Some(Cow::Owned(String::from_utf8(json_results_bytes)?)))

From 4360010221b6f8290d0ab914e8c4ea4fefb06020 Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Sat, 9 Mar 2024 10:40:21 +0100
Subject: [PATCH 09/33] document run_sql

---
 .../sqlpage/migrations/38_run_sql.sql         | 47 +++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 examples/official-site/sqlpage/migrations/38_run_sql.sql

diff --git a/examples/official-site/sqlpage/migrations/38_run_sql.sql b/examples/official-site/sqlpage/migrations/38_run_sql.sql
new file mode 100644
index 00000000..d6646812
--- /dev/null
+++ b/examples/official-site/sqlpage/migrations/38_run_sql.sql
@@ -0,0 +1,47 @@
+INSERT INTO sqlpage_functions (
+        "name",
+        "introduced_in_version",
+        "icon",
+        "description_md"
+    )
+VALUES (
+        'run_sql',
+        '0.20.0',
+        'login',
+        'Executes another SQL file and returns its result as a JSON array.
+
+### Example
+    
+#### Include a common header in all your pages
+
+It is common to want to run the same SQL queries at the beginning of all your pages,
+to check if an user is logged in, render a header, etc.
+You can create a file called `common_header.sql`, and use the `dynamic` component with the `run_sql` function to include it in all your pages.
+
+```sql
+select ''dynamic'' as component, sqlpage.run_sql(''common_header.sql'') as contents;
+```
+
+#### Notes
+
+ - **recursion**: you can use `run_sql` to include a file that itself includes another file, and so on. However, be careful to avoid infinite loops. SQLPage will throw an error if it detects a loop.
+ - **security**: be careful when using `run_sql` to include files. Never use `run_sql` with a user-provided parameter. Never run a file uploaded by a user, or a file that is not under your control.
+ - **variables**: the included file will have access to the same variables (URL parameters, POST variables, etc.)
+   as the calling file. It will not have access to uploaded files.
+   If the included file changes the value of a variable or creates a new variable, the change will not be visible in the calling file.
+'
+    );
+INSERT INTO sqlpage_function_parameters (
+        "function",
+        "index",
+        "name",
+        "description_md",
+        "type"
+    )
+VALUES (
+        'run_sql',
+        1,
+        'file',
+        'Path to the SQL file to execute, relative to the WEB ROOT (the root folder of your website).',
+        'TEXT'
+    );

From b98cc4f0f31a2bdea57f68e3757f9394a35d42fd Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Fri, 8 Mar 2024 11:49:47 +0100
Subject: [PATCH 10/33] bugfix: fix missing helper

---
 src/template_helpers.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/template_helpers.rs b/src/template_helpers.rs
index 6d356382..7264142b 100644
--- a/src/template_helpers.rs
+++ b/src/template_helpers.rs
@@ -25,7 +25,7 @@ pub fn register_all_helpers(h: &mut Handlebars<'_>) {
     h.register_helper("delay", Box::new(delay_helper));
     h.register_helper("flush_delayed", Box::new(flush_delayed_helper));
     register_helper(h, "plus", plus_helper as HH);
-    register_helper(h, "plus", minus_helper as HH);
+    register_helper(h, "minus", minus_helper as HH);
     h.register_helper("sum", Box::new(sum_helper));
     register_helper(h, "starts_with", starts_with_helper as HH);
 

From 9cd126d3d9c840ca88c5e894e6c66a0605f6dd3e Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Fri, 8 Mar 2024 11:55:20 +0100
Subject: [PATCH 11/33] documentation

---
 CHANGELOG.md                                 | 1 +
 examples/official-site/custom_components.sql | 1 +
 2 files changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 495065be..6b49ac60 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,7 @@
 
  - Add an `id` attribute to form fields in the [form](https://sql.ophir.dev/documentation.sql?component=form#component) component. This allows you to easily reference form fields in custom javascript code.
  - New [`rss`](https://sql.ophir.dev/documentation.sql?component=rss#component) component to create RSS feeds, including **podcast feeds**. You can now create and manage your podcast feed entirely in SQL, and distribute it to all podcast directories such as Apple Podcasts, Spotify, and Google Podcasts.
+ - Better error handling in template rendering. Many template helpers now display a more precise error message when they fail to execute. This makes it easier to debug errors when you [develop your own custom components](https://sql.ophir.dev/custom_components.sql).
 
 ## 0.19.1 (2024-02-28)
   - **SECURITY**: fixes users being able to re-run migrations by visiting `/sqlpage/migrations/NNNN_name.sql` pages. If you are using sqlpage migrations, your migrations are not idempotent, and you use the default SQLPAGE_WEB_ROOT (`./`) and `SQLPAGE_CONFIGURATION_DIRECTORY` (`./sqlpage/`), you should upgrade to this version as soon as possible. If you are using a custom `SQLPAGE_WEB_ROOT` or `SQLPAGE_CONFIGURATION_DIRECTORY` or your migrations are idempotent, you can upgrade at your convenience.
diff --git a/examples/official-site/custom_components.sql b/examples/official-site/custom_components.sql
index 0319b21a..0e517816 100644
--- a/examples/official-site/custom_components.sql
+++ b/examples/official-site/custom_components.sql
@@ -133,6 +133,7 @@ and SQLPage adds a few more:
 - `markdown`: renders markdown text
 - `each_row`: iterates over the rows of a query result
 - `typeof`: returns the type of a value (`string`, `number`, `boolean`, `object`, `array`, `null`)
+- `rfc2822_date`: formats a date as a string in the [RFC 2822](https://tools.ietf.org/html/rfc2822#section-3.3) format, that is, `Thu, 21 Dec 2000 16:01:07 +0200`
 
 ### Attributes
 

From 17bf7cada6763287cb660ee22a0a6177b7d9515b Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Fri, 8 Mar 2024 12:05:01 +0100
Subject: [PATCH 12/33] update sql parser

---
 CHANGELOG.md |   4 ++
 Cargo.lock   | 133 +++++++++++++++++++++++++++++----------------------
 Cargo.toml   |   4 +-
 3 files changed, 81 insertions(+), 60 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6b49ac60..7ce7a8c9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,10 @@
  - Add an `id` attribute to form fields in the [form](https://sql.ophir.dev/documentation.sql?component=form#component) component. This allows you to easily reference form fields in custom javascript code.
  - New [`rss`](https://sql.ophir.dev/documentation.sql?component=rss#component) component to create RSS feeds, including **podcast feeds**. You can now create and manage your podcast feed entirely in SQL, and distribute it to all podcast directories such as Apple Podcasts, Spotify, and Google Podcasts.
  - Better error handling in template rendering. Many template helpers now display a more precise error message when they fail to execute. This makes it easier to debug errors when you [develop your own custom components](https://sql.ophir.dev/custom_components.sql).
+ - Updated SQL parser to [v0.44](https://github.com/sqlparser-rs/sqlparser-rs/blob/main/CHANGELOG.md#0440-2024-03-02)
+   - support [EXECUTE ... USING](https://www.postgresql.org/docs/current/plpgsql-statements.html#PLPGSQL-STATEMENTS-EXECUTING-DYN) in PostgreSQL
+   - support `INSERT INTO ... SELECT ... RETURNING`, which allows you to insert data into a table, and easily pass values from the inserted row to a SQLPage component. [postgres docs](https://www.postgresql.org/docs/current/dml-returning.html), [mysql docs](https://mariadb.com/kb/en/insertreturning/), [sqlite docs](https://sqlite.org/lang_returning.html)
+   - support [`UPDATE ... FROM`](https://www.sqlite.org/lang_update.html#update_from) in SQLite
 
 ## 0.19.1 (2024-02-28)
   - **SECURITY**: fixes users being able to re-run migrations by visiting `/sqlpage/migrations/NNNN_name.sql` pages. If you are using sqlpage migrations, your migrations are not idempotent, and you use the default SQLPAGE_WEB_ROOT (`./`) and `SQLPAGE_CONFIGURATION_DIRECTORY` (`./sqlpage/`), you should upgrade to this version as soon as possible. If you are using a custom `SQLPAGE_WEB_ROOT` or `SQLPAGE_CONFIGURATION_DIRECTORY` or your migrations are idempotent, you can upgrade at your convenience.
diff --git a/Cargo.lock b/Cargo.lock
index f6eece9d..846be477 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -41,7 +41,7 @@ dependencies = [
  "flate2",
  "futures-core",
  "h2",
- "http 0.2.11",
+ "http 0.2.12",
  "httparse",
  "httpdate",
  "itoa",
@@ -114,7 +114,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d22475596539443685426b6bdadb926ad0ecaefdfc5fb05e5e3441f15463c511"
 dependencies = [
  "bytestring",
- "http 0.2.11",
+ "http 0.2.12",
  "regex",
  "serde",
  "tracing",
@@ -169,8 +169,8 @@ dependencies = [
  "actix-service",
  "actix-utils",
  "futures-core",
- "http 0.2.11",
- "http 1.0.0",
+ "http 0.2.12",
+ "http 1.1.0",
  "impl-more",
  "pin-project-lite",
  "tokio",
@@ -283,9 +283,9 @@ checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
 
 [[package]]
 name = "ahash"
-version = "0.8.10"
+version = "0.8.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b79b82693f705137f8fb9b37871d99e4f9a7df12b917eed79c3d3954830a60b"
+checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
 dependencies = [
  "cfg-if",
  "getrandom",
@@ -465,7 +465,7 @@ checksum = "afc4f0600c43df768851edad95ad43119ebde70e2feec8e39b91f97c9b62029e"
 dependencies = [
  "anyhow",
  "futures",
- "http 0.2.11",
+ "http 0.2.12",
  "httparse",
  "log",
 ]
@@ -583,7 +583,7 @@ dependencies = [
  "futures",
  "futures-rustls",
  "gloo-net",
- "http 0.2.11",
+ "http 0.2.12",
  "js-sys",
  "lazy_static",
  "log",
@@ -606,7 +606,7 @@ dependencies = [
  "futures",
  "futures-lite 1.13.0",
  "generic_static",
- "http 0.2.11",
+ "http 0.2.12",
  "log",
  "rand",
  "ring 0.16.20",
@@ -656,7 +656,7 @@ dependencies = [
  "futures-core",
  "futures-util",
  "h2",
- "http 0.2.11",
+ "http 0.2.12",
  "itoa",
  "log",
  "mime",
@@ -781,9 +781,9 @@ dependencies = [
 
 [[package]]
 name = "bumpalo"
-version = "3.15.3"
+version = "3.15.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b"
+checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa"
 
 [[package]]
 name = "byteorder"
@@ -808,10 +808,11 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.88"
+version = "1.0.90"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02f341c093d19155a6e41631ce5971aac4e9a868262212153124c15fa22d1cdc"
+checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5"
 dependencies = [
+ "jobserver",
  "libc",
 ]
 
@@ -823,9 +824,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "chrono"
-version = "0.4.34"
+version = "0.4.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b"
+checksum = "8eaf5903dcbc0a39312feb77df2ff4c76387d591b9fc7b04a238dcf8bb62639a"
 dependencies = [
  "android-tzdata",
  "iana-time-zone",
@@ -878,9 +879,9 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
 
 [[package]]
 name = "const-random"
-version = "0.1.17"
+version = "0.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aaf16c9c2c612020bcfd042e170f6e32de9b9d75adb5277cdbbd2e2c8c8299a"
+checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359"
 dependencies = [
  "const-random-macro",
 ]
@@ -1206,9 +1207,9 @@ dependencies = [
 
 [[package]]
 name = "env_logger"
-version = "0.11.2"
+version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c012a26a7f605efc424dd53697843a72be7dc86ad2d01f7814337794a12231d"
+checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9"
 dependencies = [
  "anstream",
  "anstyle",
@@ -1556,7 +1557,7 @@ dependencies = [
  "futures-core",
  "futures-sink",
  "futures-util",
- "http 0.2.11",
+ "http 0.2.12",
  "indexmap",
  "slab",
  "tokio",
@@ -1647,9 +1648,9 @@ dependencies = [
 
 [[package]]
 name = "http"
-version = "0.2.11"
+version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb"
+checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1"
 dependencies = [
  "bytes",
  "fnv",
@@ -1658,9 +1659,9 @@ dependencies = [
 
 [[package]]
 name = "http"
-version = "1.0.0"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea"
+checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258"
 dependencies = [
  "bytes",
  "fnv",
@@ -1674,7 +1675,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"
 dependencies = [
  "bytes",
- "http 0.2.11",
+ "http 0.2.12",
  "pin-project-lite",
 ]
 
@@ -1706,7 +1707,7 @@ dependencies = [
  "futures-channel",
  "futures-core",
  "futures-util",
- "http 0.2.11",
+ "http 0.2.12",
  "http-body",
  "httparse",
  "httpdate",
@@ -1828,11 +1829,20 @@ version = "1.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c"
 
+[[package]]
+name = "jobserver"
+version = "0.1.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "js-sys"
-version = "0.3.68"
+version = "0.3.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee"
+checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d"
 dependencies = [
  "wasm-bindgen",
 ]
@@ -1874,7 +1884,7 @@ dependencies = [
  "async-stream",
  "bytes",
  "futures",
- "http 0.2.11",
+ "http 0.2.12",
  "hyper",
  "lambda_runtime_api_client",
  "serde",
@@ -1891,7 +1901,7 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7210012be904051520f0dc502140ba599bae3042b65b3737b87727f1aa88a7d6"
 dependencies = [
- "http 0.2.11",
+ "http 0.2.12",
  "hyper",
  "tokio",
  "tower-service",
@@ -2343,18 +2353,18 @@ dependencies = [
 
 [[package]]
 name = "pin-project"
-version = "1.1.4"
+version = "1.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0"
+checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3"
 dependencies = [
  "pin-project-internal",
 ]
 
 [[package]]
 name = "pin-project-internal"
-version = "1.1.4"
+version = "1.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690"
+checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2533,9 +2543,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.4.5"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd"
+checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -2715,7 +2725,7 @@ dependencies = [
  "chrono",
  "futures",
  "futures-rustls",
- "http 0.2.11",
+ "http 0.2.12",
  "log",
  "pem",
  "rcgen",
@@ -2957,7 +2967,7 @@ dependencies = [
 
 [[package]]
 name = "sqlpage"
-version = "0.19.1"
+version = "0.20.0"
 dependencies = [
  "actix-multipart",
  "actix-rt",
@@ -2997,9 +3007,9 @@ dependencies = [
 
 [[package]]
 name = "sqlparser"
-version = "0.43.1"
+version = "0.44.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f95c4bae5aba7cd30bd506f7140026ade63cff5afd778af8854026f9606bf5d4"
+checksum = "aaf9c7ff146298ffda83a200f8d5084f08dcee1edfc135fcc1d646a45d50ffd6"
 dependencies = [
  "log",
  "sqlparser_derive",
@@ -3601,11 +3611,17 @@ version = "0.11.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
+[[package]]
+name = "wasite"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b"
+
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.91"
+version = "0.2.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f"
+checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8"
 dependencies = [
  "cfg-if",
  "wasm-bindgen-macro",
@@ -3613,9 +3629,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.91"
+version = "0.2.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b"
+checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da"
 dependencies = [
  "bumpalo",
  "log",
@@ -3628,9 +3644,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.41"
+version = "0.4.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97"
+checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0"
 dependencies = [
  "cfg-if",
  "js-sys",
@@ -3640,9 +3656,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.91"
+version = "0.2.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed"
+checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -3650,9 +3666,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.91"
+version = "0.2.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66"
+checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3663,15 +3679,15 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.91"
+version = "0.2.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838"
+checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
 
 [[package]]
 name = "web-sys"
-version = "0.3.68"
+version = "0.3.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446"
+checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -3704,11 +3720,12 @@ checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1"
 
 [[package]]
 name = "whoami"
-version = "1.4.1"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50"
+checksum = "0fec781d48b41f8163426ed18e8fc2864c12937df9ce54c88ede7bd47270893e"
 dependencies = [
- "wasm-bindgen",
+ "redox_syscall",
+ "wasite",
  "web-sys",
 ]
 
diff --git a/Cargo.toml b/Cargo.toml
index fca3b66f..ee18214d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "sqlpage"
-version = "0.19.1"
+version = "0.20.0"
 edition = "2021"
 description = "A SQL-only web application framework. Takes .sql files and formats the query result using pre-made configurable professional-looking components."
 keywords = ["web", "sql", "framework"]
@@ -33,7 +33,7 @@ anyhow = "1"
 serde = "1"
 serde_json = { version = "1.0.82", features = ["preserve_order"] }
 lambda-web = { version = "0.2.1", features = ["actix4"], optional = true }
-sqlparser = { version = "0.43.1", features = ["visitor"] }
+sqlparser = { version = "0.44.0", features = ["visitor"] }
 async-stream = "0.3"
 async-trait = "0.1.61"
 async-recursion = "1.0.0"

From 34a8b5d630b4d33d2253640d21f77a69992d6e68 Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Fri, 8 Mar 2024 12:20:30 +0100
Subject: [PATCH 13/33] improve carousel documentation

it wasn't very clear that we had a component to display images

https://github.com/lovasoa/SQLpage/pull/257
---
 examples/official-site/sqlpage/migrations/34_carousel.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/official-site/sqlpage/migrations/34_carousel.sql b/examples/official-site/sqlpage/migrations/34_carousel.sql
index 744d1420..fca6644a 100644
--- a/examples/official-site/sqlpage/migrations/34_carousel.sql
+++ b/examples/official-site/sqlpage/migrations/34_carousel.sql
@@ -1,7 +1,7 @@
 INSERT INTO component (name, description, icon, introduced_in_version)
 VALUES (
         'carousel',
-        'A carousel is used to display multiple pieces of visual content without taking up too much space.',
+        'A carousel is used to display images. When used with multiple images, it will cycle through them automatically or with controls, creating a slideshow.',
         'carousel-horizontal',
         '0.18.3'
     );

From d12d03871c8225660ed04d16aeb9fed2fea85b56 Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Sat, 9 Mar 2024 15:32:22 +0100
Subject: [PATCH 14/33] better error messages

---
 src/webserver/database/sql_pseudofunctions.rs             | 3 ++-
 src/webserver/http.rs                                     | 2 +-
 tests/sql_test_files/error_too_many_nested_inclusions.sql | 1 +
 3 files changed, 4 insertions(+), 2 deletions(-)
 create mode 100644 tests/sql_test_files/error_too_many_nested_inclusions.sql

diff --git a/src/webserver/database/sql_pseudofunctions.rs b/src/webserver/database/sql_pseudofunctions.rs
index 1f4ced28..9ae6b0e5 100644
--- a/src/webserver/database/sql_pseudofunctions.rs
+++ b/src/webserver/database/sql_pseudofunctions.rs
@@ -274,7 +274,8 @@ async fn run_sql<'a>(
             std::path::Path::new(sql_file_path.as_ref()),
             true,
         )
-        .await?;
+        .await
+        .with_context(|| format!("run_sql: invalid path {sql_file_path:?}"))?;
     let mut tmp_req = request.clone();
     let mut results_stream = Box::pin(stream_query_results_boxed(
         &request.app_state.db,
diff --git a/src/webserver/http.rs b/src/webserver/http.rs
index 71bbc83e..eb2affc0 100644
--- a/src/webserver/http.rs
+++ b/src/webserver/http.rs
@@ -271,7 +271,7 @@ fn send_anyhow_error(
         body.push_str("Contact the administrator for more information. A detailed error message has been logged.");
     } else {
         use std::fmt::Write;
-        write!(body, "{e:#}").unwrap();
+        write!(body, "{e:?}").unwrap();
     }
     resp = resp.set_body(BoxBody::new(body));
     resp.headers_mut().insert(
diff --git a/tests/sql_test_files/error_too_many_nested_inclusions.sql b/tests/sql_test_files/error_too_many_nested_inclusions.sql
new file mode 100644
index 00000000..c27eef9e
--- /dev/null
+++ b/tests/sql_test_files/error_too_many_nested_inclusions.sql
@@ -0,0 +1 @@
+select sqlpage.run_sql(sqlpage.path());
\ No newline at end of file

From 1ce69d821ea2c20651d255c60fe418a4802b6721 Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Sat, 9 Mar 2024 16:14:56 +0100
Subject: [PATCH 15/33] better error messages

---
 CHANGELOG.md                                  |  1 +
 .../sqlpage/migrations/38_run_sql.sql         |  3 +-
 src/webserver/database/execute_queries.rs     | 47 ++++++++++++++-----
 src/webserver/database/sql.rs                 |  6 ++-
 .../error_too_many_nested_inclusions.sql      |  5 +-
 5 files changed, 46 insertions(+), 16 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7ce7a8c9..6144c677 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@
  - Add an `id` attribute to form fields in the [form](https://sql.ophir.dev/documentation.sql?component=form#component) component. This allows you to easily reference form fields in custom javascript code.
  - New [`rss`](https://sql.ophir.dev/documentation.sql?component=rss#component) component to create RSS feeds, including **podcast feeds**. You can now create and manage your podcast feed entirely in SQL, and distribute it to all podcast directories such as Apple Podcasts, Spotify, and Google Podcasts.
  - Better error handling in template rendering. Many template helpers now display a more precise error message when they fail to execute. This makes it easier to debug errors when you [develop your own custom components](https://sql.ophir.dev/custom_components.sql).
+ - better error messages when an error occurs when defining a variable with `SET`. SQLPage now displays the query that caused the error, and the name of the variable that was being defined.
  - Updated SQL parser to [v0.44](https://github.com/sqlparser-rs/sqlparser-rs/blob/main/CHANGELOG.md#0440-2024-03-02)
    - support [EXECUTE ... USING](https://www.postgresql.org/docs/current/plpgsql-statements.html#PLPGSQL-STATEMENTS-EXECUTING-DYN) in PostgreSQL
    - support `INSERT INTO ... SELECT ... RETURNING`, which allows you to insert data into a table, and easily pass values from the inserted row to a SQLPage component. [postgres docs](https://www.postgresql.org/docs/current/dml-returning.html), [mysql docs](https://mariadb.com/kb/en/insertreturning/), [sqlite docs](https://sqlite.org/lang_returning.html)
diff --git a/examples/official-site/sqlpage/migrations/38_run_sql.sql b/examples/official-site/sqlpage/migrations/38_run_sql.sql
index d6646812..2e55129d 100644
--- a/examples/official-site/sqlpage/migrations/38_run_sql.sql
+++ b/examples/official-site/sqlpage/migrations/38_run_sql.sql
@@ -42,6 +42,7 @@ VALUES (
         'run_sql',
         1,
         'file',
-        'Path to the SQL file to execute, relative to the WEB ROOT (the root folder of your website).',
+        'Path to the SQL file to execute, can be absolute, or relative to the web root (the root folder of your website sql files).
+        In-database files, from the sqlpage_files(path, contents, last_modified) table are supported.',
         'TEXT'
     );
diff --git a/src/webserver/database/execute_queries.rs b/src/webserver/database/execute_queries.rs
index d3dfbd35..11b73e63 100644
--- a/src/webserver/database/execute_queries.rs
+++ b/src/webserver/database/execute_queries.rs
@@ -1,4 +1,4 @@
-use anyhow::anyhow;
+use anyhow::{anyhow, Context};
 use futures_util::stream::Stream;
 use futures_util::StreamExt;
 use std::borrow::Cow;
@@ -61,18 +61,10 @@ pub fn stream_query_results<'a>(
                     }
                 },
                 ParsedStatement::SetVariable { variable, value} => {
-                    let query = bind_parameters(value, request).await?;
-                    let connection = take_connection(db, &mut connection_opt).await?;
-                    log::debug!("Executing query to set the {variable:?} variable: {:?}", query.sql);
-                    let value: Option<String> = connection.fetch_optional(query).await?.as_ref().and_then(row_to_string);
-                    let (vars, name) = vars_and_name(request, variable)?;
-                    if let Some(value) = value {
-                        log::debug!("Setting variable {name} to {value:?}");
-                        vars.insert(name.clone(), SingleOrVec::Single(value));
-                    } else {
-                        log::debug!("Removing variable {name}");
-                        vars.remove(&name);
-                    }
+                    execute_set_variable_query(db, &mut connection_opt, request, variable, value).await
+                    .with_context(||
+                        format!("Failed to set the {variable:?} variable to {value:?}")
+                    )?;
                 },
                 ParsedStatement::StaticSimpleSelect(value) => {
                     yield DbItem::Row(value.clone().into())
@@ -94,6 +86,35 @@ pub fn stream_query_results_boxed<'a>(
     Box::pin(stream_query_results(db, sql_file, request))
 }
 
+async fn execute_set_variable_query<'a>(
+    db: &'a Database,
+    connection_opt: &mut Option<PoolConnection<sqlx::Any>>,
+    request: &'a mut RequestInfo,
+    variable: &StmtParam,
+    statement: &StmtWithParams,
+) -> anyhow::Result<()> {
+    let query = bind_parameters(statement, request).await?;
+    let connection = take_connection(db, connection_opt).await?;
+    log::debug!(
+        "Executing query to set the {variable:?} variable: {:?}",
+        query.sql
+    );
+    let value: Option<String> = connection
+        .fetch_optional(query)
+        .await?
+        .as_ref()
+        .and_then(row_to_string);
+    let (vars, name) = vars_and_name(request, variable)?;
+    if let Some(value) = value {
+        log::debug!("Setting variable {name} to {value:?}");
+        vars.insert(name.clone(), SingleOrVec::Single(value));
+    } else {
+        log::debug!("Removing variable {name}");
+        vars.remove(&name);
+    }
+    Ok(())
+}
+
 fn vars_and_name<'a>(
     request: &'a mut RequestInfo,
     variable: &StmtParam,
diff --git a/src/webserver/database/sql.rs b/src/webserver/database/sql.rs
index d33fcc72..9f4f1f5e 100644
--- a/src/webserver/database/sql.rs
+++ b/src/webserver/database/sql.rs
@@ -72,6 +72,7 @@ fn parse_sql<'a>(
     dialect: &'a dyn Dialect,
     sql: &'a str,
 ) -> anyhow::Result<impl Iterator<Item = ParsedStatement> + 'a> {
+    log::trace!("Parsing SQL: {sql}");
     let tokens = Tokenizer::new(dialect, sql)
         .tokenize_with_location()
         .with_context(|| "SQLPage's SQL parser could not tokenize the sql file")?;
@@ -90,6 +91,7 @@ fn parse_single_statement(parser: &mut Parser<'_>, db_kind: AnyKind) -> Option<P
         Ok(stmt) => stmt,
         Err(err) => return Some(syntax_error(err, parser)),
     };
+    log::debug!("Parsed statement: {stmt}");
     while parser.consume_token(&SemiColon) {}
     if let Some(static_statement) = extract_static_simple_select(&stmt) {
         log::debug!("Optimised a static simple select to avoid a trivial database query: {stmt} optimized to {static_statement:?}");
@@ -106,8 +108,10 @@ fn parse_single_statement(parser: &mut Parser<'_>, db_kind: AnyKind) -> Option<P
     if let Some(csv_import) = extract_csv_copy_statement(&mut stmt) {
         return Some(ParsedStatement::CsvImport(csv_import));
     }
+    let query = stmt.to_string();
+    log::debug!("Final transformed statement: {stmt}");
     Some(ParsedStatement::StmtWithParams(StmtWithParams {
-        query: stmt.to_string(),
+        query,
         params,
     }))
 }
diff --git a/tests/sql_test_files/error_too_many_nested_inclusions.sql b/tests/sql_test_files/error_too_many_nested_inclusions.sql
index c27eef9e..a021bada 100644
--- a/tests/sql_test_files/error_too_many_nested_inclusions.sql
+++ b/tests/sql_test_files/error_too_many_nested_inclusions.sql
@@ -1 +1,4 @@
-select sqlpage.run_sql(sqlpage.path());
\ No newline at end of file
+select 'debug' as component,
+    sqlpage.run_sql(
+        'tests/sql_test_files/error_too_many_nested_inclusions.sql'
+    ) as contents;
\ No newline at end of file

From 1ce38ef02a2c022edb8908f05a95ad62e0a94c9b Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Sat, 9 Mar 2024 16:31:45 +0100
Subject: [PATCH 16/33] handle infinite loops in run_sql

---
 examples/official-site/sqlpage/migrations/38_run_sql.sql | 2 +-
 src/webserver/database/sql_pseudofunctions.rs            | 5 +++++
 src/webserver/http_request_info.rs                       | 3 +++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/examples/official-site/sqlpage/migrations/38_run_sql.sql b/examples/official-site/sqlpage/migrations/38_run_sql.sql
index 2e55129d..485355de 100644
--- a/examples/official-site/sqlpage/migrations/38_run_sql.sql
+++ b/examples/official-site/sqlpage/migrations/38_run_sql.sql
@@ -24,7 +24,7 @@ select ''dynamic'' as component, sqlpage.run_sql(''common_header.sql'') as conte
 
 #### Notes
 
- - **recursion**: you can use `run_sql` to include a file that itself includes another file, and so on. However, be careful to avoid infinite loops. SQLPage will throw an error if it detects a loop.
+ - **recursion**: you can use `run_sql` to include a file that itself includes another file, and so on. However, be careful to avoid infinite loops. SQLPage will throw an error if the inclusion depth is superior to 8.
  - **security**: be careful when using `run_sql` to include files. Never use `run_sql` with a user-provided parameter. Never run a file uploaded by a user, or a file that is not under your control.
  - **variables**: the included file will have access to the same variables (URL parameters, POST variables, etc.)
    as the calling file. It will not have access to uploaded files.
diff --git a/src/webserver/database/sql_pseudofunctions.rs b/src/webserver/database/sql_pseudofunctions.rs
index 9ae6b0e5..89e5daf1 100644
--- a/src/webserver/database/sql_pseudofunctions.rs
+++ b/src/webserver/database/sql_pseudofunctions.rs
@@ -277,6 +277,11 @@ async fn run_sql<'a>(
         .await
         .with_context(|| format!("run_sql: invalid path {sql_file_path:?}"))?;
     let mut tmp_req = request.clone();
+    if tmp_req.clone_depth > 8 {
+        bail!("Too many nested inclusions. run_sql can include a file that includes another file, but the depth is limited to 8 levels. \n\
+        This is to prevent infinite loops and stack overflows.\n\
+        Make sure that your SQL file does not try to run itself, directly or through a chain of other files.");
+    }
     let mut results_stream = Box::pin(stream_query_results_boxed(
         &request.app_state.db,
         &sql_file,
diff --git a/src/webserver/http_request_info.rs b/src/webserver/http_request_info.rs
index 480032f1..7cf3be49 100644
--- a/src/webserver/http_request_info.rs
+++ b/src/webserver/http_request_info.rs
@@ -33,6 +33,7 @@ pub struct RequestInfo {
     pub cookies: ParamMap,
     pub basic_auth: Option<Basic>,
     pub app_state: Arc<AppState>,
+    pub clone_depth: u8,
 }
 
 impl Clone for RequestInfo {
@@ -49,6 +50,7 @@ impl Clone for RequestInfo {
             cookies: self.cookies.clone(),
             basic_auth: self.basic_auth.clone(),
             app_state: self.app_state.clone(),
+            clone_depth: self.clone_depth + 1,
         }
     }
 }
@@ -94,6 +96,7 @@ pub(crate) async fn extract_request_info(
         basic_auth,
         app_state,
         protocol,
+        clone_depth: 0,
     }
 }
 

From ed6742a818a59a99f4af7bf68d7bc17735026540 Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Sat, 9 Mar 2024 16:33:20 +0100
Subject: [PATCH 17/33] fix double pinning

---
 src/webserver/database/sql_pseudofunctions.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/webserver/database/sql_pseudofunctions.rs b/src/webserver/database/sql_pseudofunctions.rs
index 89e5daf1..a830961e 100644
--- a/src/webserver/database/sql_pseudofunctions.rs
+++ b/src/webserver/database/sql_pseudofunctions.rs
@@ -282,11 +282,11 @@ async fn run_sql<'a>(
         This is to prevent infinite loops and stack overflows.\n\
         Make sure that your SQL file does not try to run itself, directly or through a chain of other files.");
     }
-    let mut results_stream = Box::pin(stream_query_results_boxed(
+    let mut results_stream = stream_query_results_boxed(
         &request.app_state.db,
         &sql_file,
         &mut tmp_req,
-    ));
+    );
     let mut json_results_bytes = Vec::new();
     let mut json_encoder = serde_json::Serializer::new(&mut json_results_bytes);
     let mut seq = json_encoder.serialize_seq(None)?;

From 34a558869ffb037e9ad9b9a701545afc8456ba4f Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Sat, 9 Mar 2024 16:34:50 +0100
Subject: [PATCH 18/33] better errors in run_sql

---
 src/webserver/database/sql_pseudofunctions.rs | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/webserver/database/sql_pseudofunctions.rs b/src/webserver/database/sql_pseudofunctions.rs
index a830961e..7cd47e8c 100644
--- a/src/webserver/database/sql_pseudofunctions.rs
+++ b/src/webserver/database/sql_pseudofunctions.rs
@@ -282,11 +282,8 @@ async fn run_sql<'a>(
         This is to prevent infinite loops and stack overflows.\n\
         Make sure that your SQL file does not try to run itself, directly or through a chain of other files.");
     }
-    let mut results_stream = stream_query_results_boxed(
-        &request.app_state.db,
-        &sql_file,
-        &mut tmp_req,
-    );
+    let mut results_stream =
+        stream_query_results_boxed(&request.app_state.db, &sql_file, &mut tmp_req);
     let mut json_results_bytes = Vec::new();
     let mut json_encoder = serde_json::Serializer::new(&mut json_results_bytes);
     let mut seq = json_encoder.serialize_seq(None)?;
@@ -297,7 +294,9 @@ async fn run_sql<'a>(
                 seq.serialize_element(&row)?;
             }
             DbItem::FinishedQuery => log::trace!("run_sql: Finished query"),
-            DbItem::Error(err) => return Err(err),
+            DbItem::Error(err) => {
+                return Err(err.context(format!("run_sql: SQL error in {sql_file_path:?}")))
+            }
         }
     }
     Ok(Some(Cow::Owned(String::from_utf8(json_results_bytes)?)))

From bb018d3a02b2141e6f5a3a9b2250f987fd0da3aa Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Sat, 9 Mar 2024 16:56:02 +0100
Subject: [PATCH 19/33] better error message

---
 src/webserver/database/sql_pseudofunctions.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/webserver/database/sql_pseudofunctions.rs b/src/webserver/database/sql_pseudofunctions.rs
index 7cd47e8c..d04eb11a 100644
--- a/src/webserver/database/sql_pseudofunctions.rs
+++ b/src/webserver/database/sql_pseudofunctions.rs
@@ -279,6 +279,7 @@ async fn run_sql<'a>(
     let mut tmp_req = request.clone();
     if tmp_req.clone_depth > 8 {
         bail!("Too many nested inclusions. run_sql can include a file that includes another file, but the depth is limited to 8 levels. \n\
+        Executing sqlpage.run_sql('{sql_file_path}') would exceed this limit. \n\
         This is to prevent infinite loops and stack overflows.\n\
         Make sure that your SQL file does not try to run itself, directly or through a chain of other files.");
     }
@@ -295,7 +296,7 @@ async fn run_sql<'a>(
             }
             DbItem::FinishedQuery => log::trace!("run_sql: Finished query"),
             DbItem::Error(err) => {
-                return Err(err.context(format!("run_sql: SQL error in {sql_file_path:?}")))
+                return Err(err.context(format!("run_sql: unable to run {sql_file_path:?}")))
             }
         }
     }

From 2770603b9c7bf9e576680840349e176b78cf3e5c Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Sat, 9 Mar 2024 17:04:57 +0100
Subject: [PATCH 20/33] fix run_sql documentation

---
 examples/official-site/sqlpage/migrations/38_run_sql.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/official-site/sqlpage/migrations/38_run_sql.sql b/examples/official-site/sqlpage/migrations/38_run_sql.sql
index 485355de..877c81a7 100644
--- a/examples/official-site/sqlpage/migrations/38_run_sql.sql
+++ b/examples/official-site/sqlpage/migrations/38_run_sql.sql
@@ -19,7 +19,7 @@ to check if an user is logged in, render a header, etc.
 You can create a file called `common_header.sql`, and use the `dynamic` component with the `run_sql` function to include it in all your pages.
 
 ```sql
-select ''dynamic'' as component, sqlpage.run_sql(''common_header.sql'') as contents;
+select ''dynamic'' as component, sqlpage.run_sql(''common_header.sql'') as properties;
 ```
 
 #### Notes

From 7d31033e6d96a8a7459a412b98ac26bb93f6bf90 Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Sat, 9 Mar 2024 17:05:06 +0100
Subject: [PATCH 21/33] add test for rn_sql

---
 tests/sql_test_files/it_works_dynamic_run_sql_include.sql | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 tests/sql_test_files/it_works_dynamic_run_sql_include.sql

diff --git a/tests/sql_test_files/it_works_dynamic_run_sql_include.sql b/tests/sql_test_files/it_works_dynamic_run_sql_include.sql
new file mode 100644
index 00000000..d4c3679f
--- /dev/null
+++ b/tests/sql_test_files/it_works_dynamic_run_sql_include.sql
@@ -0,0 +1 @@
+select 'dynamic' as component, sqlpage.run_sql('tests/sql_test_files/it_works_dynamic_shell.sql') as properties;

From 49bfbbdfa16c788b2cb800b068f2aaba36ae2605 Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Sat, 9 Mar 2024 17:05:21 +0100
Subject: [PATCH 22/33] better error message for the dynamic component

---
 src/render.rs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/render.rs b/src/render.rs
index 29fab508..f60ea7e9 100644
--- a/src/render.rs
+++ b/src/render.rs
@@ -404,9 +404,12 @@ impl<W: std::io::Write> RenderContext<W> {
             .get(properties_key)
             .with_context(|| format!("Missing '{properties_key}' key."))?;
         Ok(match properties_obj {
-            Value::String(s) => match serde_json::from_str::<JsonValue>(s)
-                .with_context(|| "parsing json properties")?
-            {
+            Value::String(s) => match serde_json::from_str::<JsonValue>(s).with_context(|| {
+                format!(
+                    "Unable to parse the 'properties' property of the dynamic component as JSON.\n\
+                Invalid json: {s}"
+                )
+            })? {
                 Value::Array(values) => values.into_iter().map(Cow::Owned).collect(),
                 obj @ Value::Object(_) => vec![Cow::Owned(obj)],
                 other => bail!(

From 1b2ca74a2faa80aeb3ce3f622df31d4718f95618 Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Sat, 9 Mar 2024 17:07:15 +0100
Subject: [PATCH 23/33] fix json encoding in run_sql

---
 src/webserver/database/sql_pseudofunctions.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/webserver/database/sql_pseudofunctions.rs b/src/webserver/database/sql_pseudofunctions.rs
index d04eb11a..68151e69 100644
--- a/src/webserver/database/sql_pseudofunctions.rs
+++ b/src/webserver/database/sql_pseudofunctions.rs
@@ -300,6 +300,7 @@ async fn run_sql<'a>(
             }
         }
     }
+    seq.end()?;
     Ok(Some(Cow::Owned(String::from_utf8(json_results_bytes)?)))
 }
 

From 96d1e49b9d4f6b80eae341978ed14116539d1a1d Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Sat, 9 Mar 2024 17:16:47 +0100
Subject: [PATCH 24/33] add a simple run_sql test

---
 tests/sql_test_files/it_works_run_sql.sql | 1 +
 tests/sql_test_files/it_works_simple.sql  | 2 ++
 2 files changed, 3 insertions(+)
 create mode 100644 tests/sql_test_files/it_works_run_sql.sql
 create mode 100644 tests/sql_test_files/it_works_simple.sql

diff --git a/tests/sql_test_files/it_works_run_sql.sql b/tests/sql_test_files/it_works_run_sql.sql
new file mode 100644
index 00000000..d7c830d8
--- /dev/null
+++ b/tests/sql_test_files/it_works_run_sql.sql
@@ -0,0 +1 @@
+select 'dynamic' as component, sqlpage.run_sql('tests/sql_test_files/it_works_simple.sql') as properties;
diff --git a/tests/sql_test_files/it_works_simple.sql b/tests/sql_test_files/it_works_simple.sql
new file mode 100644
index 00000000..cef377f5
--- /dev/null
+++ b/tests/sql_test_files/it_works_simple.sql
@@ -0,0 +1,2 @@
+select 'shell' as component, 'Hello world !' AS title;
+select 'text' as component, 'It works !' AS contents;
\ No newline at end of file

From 965aae27ac67be8e9fc3ac7668fdaa9ec126be7e Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Sun, 10 Mar 2024 11:58:44 +0100
Subject: [PATCH 25/33] implement dynamic component expantion at the
 execute_queries level

---
 src/webserver/database/execute_queries.rs | 51 +++++++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/src/webserver/database/execute_queries.rs b/src/webserver/database/execute_queries.rs
index 11b73e63..0309291f 100644
--- a/src/webserver/database/execute_queries.rs
+++ b/src/webserver/database/execute_queries.rs
@@ -16,6 +16,8 @@ use sqlx::any::{AnyArguments, AnyQueryResult, AnyRow, AnyStatement, AnyTypeInfo}
 use sqlx::pool::PoolConnection;
 use sqlx::{Any, AnyConnection, Arguments, Either, Executor, Statement};
 
+use serde_json::Value as JsonValue;
+
 use super::sql_pseudofunctions::StmtParam;
 use super::{highlight_sql_error, Database, DbItem};
 
@@ -33,6 +35,55 @@ impl Database {
     }
 }
 
+/// the raw query results can include (potentially nested) rows with a 'component' column that has the value 'dynamic'.
+/// in that case we need to parse the JSON in the 'properties' column, and emit a row for each value in the resulting json array.
+pub fn parse_dynamic_rows(mut row: JsonValue) -> Box<dyn Iterator<Item = DbItem>> {
+    if let Some(properties) = extract_dynamic_properties(&mut row) {
+        match dynamic_properties_to_iter(properties) {
+            Ok(iter) => Box::new(iter.map(DbItem::Row)),
+            Err(e) => Box::new(std::iter::once(DbItem::Error(e))),
+        }
+    } else {
+        Box::new(std::iter::once(DbItem::Row(row)))
+    }
+}
+
+/// if row.component == 'dynamic', return Some(row.properties), otherwise return None
+fn extract_dynamic_properties(data: &mut JsonValue) -> Option<JsonValue> {
+    let component = data.get("component").and_then(|v| v.as_str());
+    if component == Some("dynamic") {
+        let properties = data.get_mut("properties").map(|v| v.take());
+        Some(properties.unwrap_or_default())
+    } else {
+        None
+    }
+}
+
+fn dynamic_properties_to_iter(
+    mut data: JsonValue,
+) -> anyhow::Result<Box<dyn Iterator<Item = JsonValue>>> {
+    let properties_key = "properties";
+    let mut properties_obj = data
+        .get_mut(properties_key)
+        .with_context(|| format!("Missing '{properties_key}' key."))?
+        .take();
+    if let JsonValue::String(s) = properties_obj {
+        properties_obj = serde_json::from_str::<JsonValue>(&s).with_context(|| {
+            format!(
+                "Unable to parse the 'properties' property of the dynamic component as JSON.\n\
+                    Invalid json: {s}"
+            )
+        })?;
+    }
+    match properties_obj {
+        obj @ JsonValue::Object(_) => Ok(Box::new(std::iter::once(obj))),
+        JsonValue::Array(values) => Ok(Box::new(values.into_iter())),
+        other => anyhow::bail!(
+            "Dynamic component expected properties of type array or object, got {other} instead."
+        ),
+    }
+}
+
 pub fn stream_query_results<'a>(
     db: &'a Database,
     sql_file: &'a ParsedSqlFile,

From ff4789c3b76cdc13a92bcc518c6d3b757334db90 Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Sun, 10 Mar 2024 12:14:38 +0100
Subject: [PATCH 26/33] parse dynamic rows earlier

---
 src/webserver/database/execute_queries.rs | 24 +++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/src/webserver/database/execute_queries.rs b/src/webserver/database/execute_queries.rs
index 0309291f..96a49d84 100644
--- a/src/webserver/database/execute_queries.rs
+++ b/src/webserver/database/execute_queries.rs
@@ -37,14 +37,18 @@ impl Database {
 
 /// the raw query results can include (potentially nested) rows with a 'component' column that has the value 'dynamic'.
 /// in that case we need to parse the JSON in the 'properties' column, and emit a row for each value in the resulting json array.
-pub fn parse_dynamic_rows(mut row: JsonValue) -> Box<dyn Iterator<Item = DbItem>> {
-    if let Some(properties) = extract_dynamic_properties(&mut row) {
-        match dynamic_properties_to_iter(properties) {
-            Ok(iter) => Box::new(iter.map(DbItem::Row)),
-            Err(e) => Box::new(std::iter::once(DbItem::Error(e))),
+pub fn parse_dynamic_rows(db_item: DbItem) -> Box<dyn Iterator<Item = DbItem>> {
+    if let DbItem::Row(mut row) = db_item {
+        if let Some(properties) = extract_dynamic_properties(&mut row) {
+            match dynamic_properties_to_iter(properties) {
+                Ok(iter) => Box::new(iter.map(DbItem::Row)),
+                Err(e) => Box::new(std::iter::once(DbItem::Error(e))),
+            }
+        } else {
+            Box::new(std::iter::once(DbItem::Row(row)))
         }
     } else {
-        Box::new(std::iter::once(DbItem::Row(row)))
+        Box::new(std::iter::once(db_item))
     }
 }
 
@@ -105,7 +109,9 @@ pub fn stream_query_results<'a>(
                     let mut stream = connection.fetch_many(query);
                     while let Some(elem) = stream.next().await {
                         let is_err = elem.is_err();
-                        yield parse_single_sql_result(&stmt.query, elem);
+                        for i in parse_dynamic_rows(parse_single_sql_result(&stmt.query, elem)) {
+                            yield i;
+                        }
                         if is_err {
                             break;
                         }
@@ -118,7 +124,9 @@ pub fn stream_query_results<'a>(
                     )?;
                 },
                 ParsedStatement::StaticSimpleSelect(value) => {
-                    yield DbItem::Row(value.clone().into())
+                    for i in parse_dynamic_rows(DbItem::Row(value.clone().into())) {
+                        yield i;
+                    }
                 }
                 ParsedStatement::Error(e) => yield DbItem::Error(clone_anyhow_err(e)),
             }

From e1e203dc764c0abb67e60a38b9e6e90ccfec7608 Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Sun, 10 Mar 2024 12:19:14 +0100
Subject: [PATCH 27/33] fix properties extraction

---
 src/webserver/database/execute_queries.rs | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/src/webserver/database/execute_queries.rs b/src/webserver/database/execute_queries.rs
index 96a49d84..3baf77ae 100644
--- a/src/webserver/database/execute_queries.rs
+++ b/src/webserver/database/execute_queries.rs
@@ -64,13 +64,8 @@ fn extract_dynamic_properties(data: &mut JsonValue) -> Option<JsonValue> {
 }
 
 fn dynamic_properties_to_iter(
-    mut data: JsonValue,
+    mut properties_obj: JsonValue,
 ) -> anyhow::Result<Box<dyn Iterator<Item = JsonValue>>> {
-    let properties_key = "properties";
-    let mut properties_obj = data
-        .get_mut(properties_key)
-        .with_context(|| format!("Missing '{properties_key}' key."))?
-        .take();
     if let JsonValue::String(s) = properties_obj {
         properties_obj = serde_json::from_str::<JsonValue>(&s).with_context(|| {
             format!(

From 704a03301a042b5c4c0ca457f19ef41659c55b6c Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Sun, 10 Mar 2024 13:24:02 +0100
Subject: [PATCH 28/33] clippy

---
 src/webserver/database/execute_queries.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/webserver/database/execute_queries.rs b/src/webserver/database/execute_queries.rs
index 3baf77ae..08869210 100644
--- a/src/webserver/database/execute_queries.rs
+++ b/src/webserver/database/execute_queries.rs
@@ -56,7 +56,7 @@ pub fn parse_dynamic_rows(db_item: DbItem) -> Box<dyn Iterator<Item = DbItem>> {
 fn extract_dynamic_properties(data: &mut JsonValue) -> Option<JsonValue> {
     let component = data.get("component").and_then(|v| v.as_str());
     if component == Some("dynamic") {
-        let properties = data.get_mut("properties").map(|v| v.take());
+        let properties = data.get_mut("properties").map(JsonValue::take);
         Some(properties.unwrap_or_default())
     } else {
         None

From 1cdd2de83d41b343a2301727466bbe8801c8f7f4 Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Sun, 10 Mar 2024 14:06:33 +0100
Subject: [PATCH 29/33] extract and test dynamic component

---
 src/dynamic_component.rs                      |  65 +++++++++
 src/lib.rs                                    |   1 +
 src/webserver/database/execute_queries.rs     |  57 +-------
 ...ror_too_many_nested_dynamic_components.sql | 129 ++++++++++++++++++
 .../it_works_dynamic_nested.sql               |   8 ++
 5 files changed, 206 insertions(+), 54 deletions(-)
 create mode 100644 src/dynamic_component.rs
 create mode 100644 tests/sql_test_files/error_too_many_nested_dynamic_components.sql
 create mode 100644 tests/sql_test_files/it_works_dynamic_nested.sql

diff --git a/src/dynamic_component.rs b/src/dynamic_component.rs
new file mode 100644
index 00000000..f2a97fee
--- /dev/null
+++ b/src/dynamic_component.rs
@@ -0,0 +1,65 @@
+use anyhow::{self, Context as _};
+use serde_json::Value as JsonValue;
+
+use crate::webserver::database::DbItem;
+
+const MAX_RECURSION_DEPTH: u8 = 127;
+
+/// the raw query results can include (potentially nested) rows with a 'component' column that has the value 'dynamic'.
+/// in that case we need to parse the JSON in the 'properties' column, and emit a row for each value in the resulting json array.
+#[must_use] pub fn parse_dynamic_rows(db_item: DbItem) -> Box<dyn Iterator<Item = DbItem>> {
+    if let DbItem::Row(row) = db_item {
+        parse_dynamic_rows_json(row, 0)
+    } else {
+        Box::new(std::iter::once(db_item))
+    }
+}
+
+fn parse_dynamic_rows_json(mut row: JsonValue, depth: u8) -> Box<dyn Iterator<Item = DbItem>> {
+    if depth >= MAX_RECURSION_DEPTH {
+        return Box::new(std::iter::once(DbItem::Error(anyhow::anyhow!(
+            "Too many nested dynamic components: \n\
+            The 'dynamic' component can be used to render another 'dynamic' component, \
+            but the recursion cannot exceed {depth} layers."
+        ))));
+    }
+    if let Some(properties) = extract_dynamic_properties(&mut row) {
+        match dynamic_properties_to_iter(properties) {
+            Ok(iter) => Box::new(iter.flat_map(move |v| parse_dynamic_rows_json(v, depth + 1))),
+            Err(e) => Box::new(std::iter::once(DbItem::Error(e))),
+        }
+    } else {
+        Box::new(std::iter::once(DbItem::Row(row)))
+    }
+}
+
+/// if row.component == 'dynamic', return Some(row.properties), otherwise return None
+fn extract_dynamic_properties(data: &mut JsonValue) -> Option<JsonValue> {
+    let component = data.get("component").and_then(|v| v.as_str());
+    if component == Some("dynamic") {
+        let properties = data.get_mut("properties").map(JsonValue::take);
+        Some(properties.unwrap_or_default())
+    } else {
+        None
+    }
+}
+
+fn dynamic_properties_to_iter(
+    mut properties_obj: JsonValue,
+) -> anyhow::Result<Box<dyn Iterator<Item = JsonValue>>> {
+    if let JsonValue::String(s) = properties_obj {
+        properties_obj = serde_json::from_str::<JsonValue>(&s).with_context(|| {
+            format!(
+                "Unable to parse the 'properties' property of the dynamic component as JSON.\n\
+                    Invalid json: {s}"
+            )
+        })?;
+    }
+    match properties_obj {
+        obj @ JsonValue::Object(_) => Ok(Box::new(std::iter::once(obj))),
+        JsonValue::Array(values) => Ok(Box::new(values.into_iter())),
+        other => anyhow::bail!(
+            "Dynamic component expected properties of type array or object, got {other} instead."
+        ),
+    }
+}
diff --git a/src/lib.rs b/src/lib.rs
index 38845be9..9c3e5166 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -4,6 +4,7 @@
 extern crate core;
 
 pub mod app_config;
+pub mod dynamic_component;
 pub mod file_cache;
 pub mod filesystem;
 pub mod render;
diff --git a/src/webserver/database/execute_queries.rs b/src/webserver/database/execute_queries.rs
index 08869210..1a4df801 100644
--- a/src/webserver/database/execute_queries.rs
+++ b/src/webserver/database/execute_queries.rs
@@ -7,20 +7,18 @@ use std::pin::Pin;
 
 use super::csv_import::run_csv_import;
 use super::sql::{ParsedSqlFile, ParsedStatement, StmtWithParams};
+use crate::dynamic_component::parse_dynamic_rows;
 use crate::webserver::database::sql_pseudofunctions::extract_req_param;
 use crate::webserver::database::sql_to_json::row_to_string;
 use crate::webserver::http::SingleOrVec;
 use crate::webserver::http_request_info::RequestInfo;
 
+use super::sql_pseudofunctions::StmtParam;
+use super::{highlight_sql_error, Database, DbItem};
 use sqlx::any::{AnyArguments, AnyQueryResult, AnyRow, AnyStatement, AnyTypeInfo};
 use sqlx::pool::PoolConnection;
 use sqlx::{Any, AnyConnection, Arguments, Either, Executor, Statement};
 
-use serde_json::Value as JsonValue;
-
-use super::sql_pseudofunctions::StmtParam;
-use super::{highlight_sql_error, Database, DbItem};
-
 impl Database {
     pub(crate) async fn prepare_with(
         &self,
@@ -34,55 +32,6 @@ impl Database {
             .map_err(|e| highlight_sql_error("Failed to prepare SQL statement", query, e))
     }
 }
-
-/// the raw query results can include (potentially nested) rows with a 'component' column that has the value 'dynamic'.
-/// in that case we need to parse the JSON in the 'properties' column, and emit a row for each value in the resulting json array.
-pub fn parse_dynamic_rows(db_item: DbItem) -> Box<dyn Iterator<Item = DbItem>> {
-    if let DbItem::Row(mut row) = db_item {
-        if let Some(properties) = extract_dynamic_properties(&mut row) {
-            match dynamic_properties_to_iter(properties) {
-                Ok(iter) => Box::new(iter.map(DbItem::Row)),
-                Err(e) => Box::new(std::iter::once(DbItem::Error(e))),
-            }
-        } else {
-            Box::new(std::iter::once(DbItem::Row(row)))
-        }
-    } else {
-        Box::new(std::iter::once(db_item))
-    }
-}
-
-/// if row.component == 'dynamic', return Some(row.properties), otherwise return None
-fn extract_dynamic_properties(data: &mut JsonValue) -> Option<JsonValue> {
-    let component = data.get("component").and_then(|v| v.as_str());
-    if component == Some("dynamic") {
-        let properties = data.get_mut("properties").map(JsonValue::take);
-        Some(properties.unwrap_or_default())
-    } else {
-        None
-    }
-}
-
-fn dynamic_properties_to_iter(
-    mut properties_obj: JsonValue,
-) -> anyhow::Result<Box<dyn Iterator<Item = JsonValue>>> {
-    if let JsonValue::String(s) = properties_obj {
-        properties_obj = serde_json::from_str::<JsonValue>(&s).with_context(|| {
-            format!(
-                "Unable to parse the 'properties' property of the dynamic component as JSON.\n\
-                    Invalid json: {s}"
-            )
-        })?;
-    }
-    match properties_obj {
-        obj @ JsonValue::Object(_) => Ok(Box::new(std::iter::once(obj))),
-        JsonValue::Array(values) => Ok(Box::new(values.into_iter())),
-        other => anyhow::bail!(
-            "Dynamic component expected properties of type array or object, got {other} instead."
-        ),
-    }
-}
-
 pub fn stream_query_results<'a>(
     db: &'a Database,
     sql_file: &'a ParsedSqlFile,
diff --git a/tests/sql_test_files/error_too_many_nested_dynamic_components.sql b/tests/sql_test_files/error_too_many_nested_dynamic_components.sql
new file mode 100644
index 00000000..c0608b23
--- /dev/null
+++ b/tests/sql_test_files/error_too_many_nested_dynamic_components.sql
@@ -0,0 +1,129 @@
+select 'dynamic' as component,'{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"dynamic","properties":
+{"component":"text", "contents": "It works ! (but it shouldn''t)"}
+}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}
+' as properties;
\ No newline at end of file
diff --git a/tests/sql_test_files/it_works_dynamic_nested.sql b/tests/sql_test_files/it_works_dynamic_nested.sql
new file mode 100644
index 00000000..4c3c0fb8
--- /dev/null
+++ b/tests/sql_test_files/it_works_dynamic_nested.sql
@@ -0,0 +1,8 @@
+-- Checks that we can have a page with a single dynamic component containing multiple children
+select 'dynamic' as component,
+    '[
+        {"component":"dynamic", "properties": [
+            {"component":"text"},
+            {"contents":"It works !", "bold":true}
+        ]}
+    ]' as properties;
\ No newline at end of file

From c5d62a500cfe806cd9dc23de7cea4e3d48308a88 Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Sun, 10 Mar 2024 14:13:42 +0100
Subject: [PATCH 30/33] remove now redundant 'dynamic' handling from the render
 logic

---
 src/render.rs | 54 +--------------------------------------------------
 1 file changed, 1 insertion(+), 53 deletions(-)

diff --git a/src/render.rs b/src/render.rs
index f60ea7e9..5d733a20 100644
--- a/src/render.rs
+++ b/src/render.rs
@@ -274,15 +274,12 @@ pub struct RenderContext<W: std::io::Write> {
     pub writer: W,
     current_component: Option<SplitTemplateRenderer>,
     shell_renderer: SplitTemplateRenderer,
-    recursion_depth: usize,
     current_statement: usize,
 }
 
 const DEFAULT_COMPONENT: &str = "table";
 const PAGE_SHELL_COMPONENT: &str = "shell";
 const FRAGMENT_SHELL_COMPONENT: &str = "shell-empty";
-const DYNAMIC_COMPONENT: &str = "dynamic";
-const MAX_RECURSION_DEPTH: usize = 256;
 
 impl<W: std::io::Write> RenderContext<W> {
     pub async fn new(
@@ -293,12 +290,7 @@ impl<W: std::io::Write> RenderContext<W> {
     ) -> anyhow::Result<RenderContext<W>> {
         log::debug!("Creating the shell component for the page");
 
-        let mut initial_rows =
-            if get_object_str(&initial_row, "component") == Some(DYNAMIC_COMPONENT) {
-                Self::extract_dynamic_properties(&initial_row)?
-            } else {
-                vec![Cow::Borrowed(&initial_row)]
-            };
+        let mut initial_rows = vec![Cow::Borrowed(&initial_row)];
 
         if !initial_rows
             .first()
@@ -336,7 +328,6 @@ impl<W: std::io::Write> RenderContext<W> {
             writer,
             current_component: None,
             shell_renderer,
-            recursion_depth: 0,
             current_statement: 1,
         };
 
@@ -367,11 +358,6 @@ impl<W: std::io::Write> RenderContext<W> {
         let new_component = get_object_str(data, "component");
         let current_component = self.current_component().await?.name();
         match (current_component, new_component) {
-            (_current_component, Some(DYNAMIC_COMPONENT)) => {
-                self.render_dynamic(data).await.with_context(|| {
-                    format!("Unable to render dynamic component with properties {data}")
-                })?;
-            }
             (
                 _,
                 Some(
@@ -398,44 +384,6 @@ impl<W: std::io::Write> RenderContext<W> {
         Ok(())
     }
 
-    fn extract_dynamic_properties(data: &Value) -> anyhow::Result<Vec<Cow<'_, JsonValue>>> {
-        let properties_key = "properties";
-        let properties_obj = data
-            .get(properties_key)
-            .with_context(|| format!("Missing '{properties_key}' key."))?;
-        Ok(match properties_obj {
-            Value::String(s) => match serde_json::from_str::<JsonValue>(s).with_context(|| {
-                format!(
-                    "Unable to parse the 'properties' property of the dynamic component as JSON.\n\
-                Invalid json: {s}"
-                )
-            })? {
-                Value::Array(values) => values.into_iter().map(Cow::Owned).collect(),
-                obj @ Value::Object(_) => vec![Cow::Owned(obj)],
-                other => bail!(
-                    "Expected properties string to parse as array or object, got {other} instead."
-                ),
-            },
-            obj @ Value::Object(_) => vec![Cow::Borrowed(obj)],
-            Value::Array(values) => values.iter().map(Cow::Borrowed).collect(),
-            other => bail!("Expected properties of type array or object, got {other} instead."),
-        })
-    }
-
-    async fn render_dynamic(&mut self, data: &Value) -> anyhow::Result<()> {
-        anyhow::ensure!(
-            self.recursion_depth <= MAX_RECURSION_DEPTH,
-            "Maximum recursion depth exceeded in the dynamic component."
-        );
-        for dynamic_row_obj in Self::extract_dynamic_properties(data)? {
-            self.recursion_depth += 1;
-            let res = self.handle_row(&dynamic_row_obj).await;
-            self.recursion_depth -= 1;
-            res?;
-        }
-        Ok(())
-    }
-
     #[allow(clippy::unused_async)]
     pub async fn finish_query(&mut self) -> anyhow::Result<()> {
         log::debug!("-> Query {} finished", self.current_statement);

From 30d4abbc9838c1b963f5637a73954a3616a086a9 Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Sun, 10 Mar 2024 14:27:52 +0100
Subject: [PATCH 31/33] changelog

---
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6144c677..db483481 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,11 @@
 
 ## 0.20.0 (unreleased)
 
+ - **file inclusion**. This is a long awaited feature that allows you to include the contents of one file in another. This is useful to factorize common parts of your website, such as the header, or the authentication logic. There is a new [`sqlpage.run_sql`](https://sql.ophir.dev/functions.sql?function=run_sql#function) function that runs a given SQL file and returns its result as a JSON array. Combined with the existing [`dynamic`](https://sql.ophir.dev/documentation.sql?component=dynamic#component) component, this allows you to include the content of a file in another, like this:
+  ```sql
+  select 'dynamic' as component, sqlpage.run_sql('header.sql') as properties;
+  ```
+ - **more powerful *dynamic* component**: the dynamic component can now be used to generate the special *header* components too, such as the `redirect`, `cookie`, `authentication`, `http_header` and `json` components. The *shell* component used to be allowed in dynamic components, but only if they were not nested (a dynamic component inside another one). This limitation is now lifted. This is particularly useful in combination with the new file inclusion feature, to factorize common parts of your website.
  - Add an `id` attribute to form fields in the [form](https://sql.ophir.dev/documentation.sql?component=form#component) component. This allows you to easily reference form fields in custom javascript code.
  - New [`rss`](https://sql.ophir.dev/documentation.sql?component=rss#component) component to create RSS feeds, including **podcast feeds**. You can now create and manage your podcast feed entirely in SQL, and distribute it to all podcast directories such as Apple Podcasts, Spotify, and Google Podcasts.
  - Better error handling in template rendering. Many template helpers now display a more precise error message when they fail to execute. This makes it easier to debug errors when you [develop your own custom components](https://sql.ophir.dev/custom_components.sql).

From 63131646ec8fa0afee0a77a5948718e889ebf3fd Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Sun, 10 Mar 2024 20:24:09 +0100
Subject: [PATCH 32/33] Refactor dynamic component parsing

---
 src/dynamic_component.rs | 153 ++++++++++++++++++++++++++++++++-------
 1 file changed, 127 insertions(+), 26 deletions(-)

diff --git a/src/dynamic_component.rs b/src/dynamic_component.rs
index f2a97fee..63186428 100644
--- a/src/dynamic_component.rs
+++ b/src/dynamic_component.rs
@@ -3,37 +3,55 @@ use serde_json::Value as JsonValue;
 
 use crate::webserver::database::DbItem;
 
-const MAX_RECURSION_DEPTH: u8 = 127;
-
-/// the raw query results can include (potentially nested) rows with a 'component' column that has the value 'dynamic'.
-/// in that case we need to parse the JSON in the 'properties' column, and emit a row for each value in the resulting json array.
-#[must_use] pub fn parse_dynamic_rows(db_item: DbItem) -> Box<dyn Iterator<Item = DbItem>> {
-    if let DbItem::Row(row) = db_item {
-        parse_dynamic_rows_json(row, 0)
-    } else {
-        Box::new(std::iter::once(db_item))
+pub fn parse_dynamic_rows(row: DbItem) -> impl Iterator<Item = DbItem> {
+    DynamicComponentIterator {
+        stack: vec![],
+        db_item: Some(row),
     }
 }
 
-fn parse_dynamic_rows_json(mut row: JsonValue, depth: u8) -> Box<dyn Iterator<Item = DbItem>> {
-    if depth >= MAX_RECURSION_DEPTH {
-        return Box::new(std::iter::once(DbItem::Error(anyhow::anyhow!(
-            "Too many nested dynamic components: \n\
-            The 'dynamic' component can be used to render another 'dynamic' component, \
-            but the recursion cannot exceed {depth} layers."
-        ))));
+struct DynamicComponentIterator {
+    stack: Vec<anyhow::Result<JsonValue>>,
+    db_item: Option<DbItem>,
+}
+
+impl Iterator for DynamicComponentIterator {
+    type Item = DbItem;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if let Some(db_item) = self.db_item.take() {
+            if let DbItem::Row(mut row) = db_item {
+                if let Some(properties) = extract_dynamic_properties(&mut row) {
+                    self.stack = dynamic_properties_to_vec(properties);
+                } else {
+                    // Most common case: just a regular row. We allocated nothing.
+                    return Some(DbItem::Row(row));
+                }
+            } else {
+                return Some(db_item);
+            }
+        }
+        expand_dynamic_stack(&mut self.stack);
+        self.stack.pop().map(|result| match result {
+            Ok(row) => DbItem::Row(row),
+            Err(err) => DbItem::Error(err),
+        })
     }
-    if let Some(properties) = extract_dynamic_properties(&mut row) {
-        match dynamic_properties_to_iter(properties) {
-            Ok(iter) => Box::new(iter.flat_map(move |v| parse_dynamic_rows_json(v, depth + 1))),
-            Err(e) => Box::new(std::iter::once(DbItem::Error(e))),
+}
+
+fn expand_dynamic_stack(stack: &mut Vec<anyhow::Result<JsonValue>>) {
+    while let Some(Ok(mut next)) = stack.pop() {
+        if let Some(properties) = extract_dynamic_properties(&mut next) {
+            stack.extend(dynamic_properties_to_vec(properties));
+        } else {
+            stack.push(Ok(next));
+            return;
         }
-    } else {
-        Box::new(std::iter::once(DbItem::Row(row)))
     }
 }
 
 /// if row.component == 'dynamic', return Some(row.properties), otherwise return None
+#[inline]
 fn extract_dynamic_properties(data: &mut JsonValue) -> Option<JsonValue> {
     let component = data.get("component").and_then(|v| v.as_str());
     if component == Some("dynamic") {
@@ -44,9 +62,22 @@ fn extract_dynamic_properties(data: &mut JsonValue) -> Option<JsonValue> {
     }
 }
 
-fn dynamic_properties_to_iter(
+/// reverse the order of the vec returned by `dynamic_properties_to_result_vec`,
+/// and wrap each element in a Result
+fn dynamic_properties_to_vec(properties_obj: JsonValue) -> Vec<anyhow::Result<JsonValue>> {
+    dynamic_properties_to_result_vec(properties_obj).map_or_else(
+        |err| vec![Err(err)],
+        |vec| vec.into_iter().rev().map(Ok).collect::<Vec<_>>(),
+    )
+}
+
+/// if properties is a string, parse it as JSON and return a vec with the parsed value
+/// if properties is an array, return it as is
+/// if properties is an object, return it as a single element vec
+/// otherwise, return an error
+fn dynamic_properties_to_result_vec(
     mut properties_obj: JsonValue,
-) -> anyhow::Result<Box<dyn Iterator<Item = JsonValue>>> {
+) -> anyhow::Result<Vec<JsonValue>> {
     if let JsonValue::String(s) = properties_obj {
         properties_obj = serde_json::from_str::<JsonValue>(&s).with_context(|| {
             format!(
@@ -56,10 +87,80 @@ fn dynamic_properties_to_iter(
         })?;
     }
     match properties_obj {
-        obj @ JsonValue::Object(_) => Ok(Box::new(std::iter::once(obj))),
-        JsonValue::Array(values) => Ok(Box::new(values.into_iter())),
+        obj @ JsonValue::Object(_) => Ok(vec![obj]),
+        JsonValue::Array(values) => Ok(values),
         other => anyhow::bail!(
             "Dynamic component expected properties of type array or object, got {other} instead."
         ),
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_dynamic_properties_to_result_vec() {
+        let mut properties = JsonValue::String(r#"{"a": 1}"#.to_string());
+        assert_eq!(
+            dynamic_properties_to_result_vec(properties.clone()).unwrap(),
+            vec![JsonValue::Object(
+                serde_json::from_str(r#"{"a": 1}"#).unwrap()
+            )]
+        );
+
+        properties = JsonValue::Array(vec![JsonValue::String(r#"{"a": 1}"#.to_string())]);
+        assert_eq!(
+            dynamic_properties_to_result_vec(properties.clone()).unwrap(),
+            vec![JsonValue::String(r#"{"a": 1}"#.to_string())]
+        );
+
+        properties = JsonValue::Object(serde_json::from_str(r#"{"a": 1}"#).unwrap());
+        assert_eq!(
+            dynamic_properties_to_result_vec(properties.clone()).unwrap(),
+            vec![JsonValue::Object(
+                serde_json::from_str(r#"{"a": 1}"#).unwrap()
+            )]
+        );
+
+        properties = JsonValue::Null;
+        assert!(dynamic_properties_to_result_vec(properties).is_err());
+    }
+
+    #[test]
+    fn test_dynamic_properties_to_vec() {
+        let properties = JsonValue::String(r#"{"a": 1}"#.to_string());
+        assert_eq!(
+            dynamic_properties_to_vec(properties.clone())
+                .first()
+                .unwrap()
+                .as_ref()
+                .unwrap(),
+            &serde_json::json!({"a": 1})
+        );
+    }
+
+    #[test]
+    fn test_parse_dynamic_rows() {
+        let row = DbItem::Row(serde_json::json!({
+            "component": "dynamic",
+            "properties": [
+                {"a": 1},
+                {"component": "dynamic", "properties": {"nested": 2}},
+            ]
+        }));
+        let iter = parse_dynamic_rows(row)
+            .map(|item| match item {
+                DbItem::Row(row) => row,
+                x => panic!("Expected a row, got {x:?}"),
+            })
+            .collect::<Vec<_>>();
+        assert_eq!(
+            iter,
+            vec![
+                serde_json::json!({"a": 1}),
+                serde_json::json!({"nested": 2}),
+            ]
+        );
+    }
+}

From 6482d7f3ff25522cae1403d07350835e190f40d5 Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Mon, 11 Mar 2024 07:50:44 +0100
Subject: [PATCH 33/33] deeply nested dynamic components are now allowed

---
 CHANGELOG.md                                                    | 2 +-
 ...onents.sql => it_works_deeply_nested_dynamic_components.sql} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename tests/sql_test_files/{error_too_many_nested_dynamic_components.sql => it_works_deeply_nested_dynamic_components.sql} (100%)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index db483481..8b742317 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,7 +6,7 @@
   ```sql
   select 'dynamic' as component, sqlpage.run_sql('header.sql') as properties;
   ```
- - **more powerful *dynamic* component**: the dynamic component can now be used to generate the special *header* components too, such as the `redirect`, `cookie`, `authentication`, `http_header` and `json` components. The *shell* component used to be allowed in dynamic components, but only if they were not nested (a dynamic component inside another one). This limitation is now lifted. This is particularly useful in combination with the new file inclusion feature, to factorize common parts of your website.
+ - **more powerful *dynamic* component**: the [`dynamic`](https://sql.ophir.dev/documentation.sql?component=dynamic#component) component can now be used to generate the special *header* components too, such as the `redirect`, `cookie`, `authentication`, `http_header` and `json` components. The *shell* component used to be allowed in dynamic components, but only if they were not nested (a dynamic component inside another one). This limitation is now lifted. This is particularly useful in combination with the new file inclusion feature, to factorize common parts of your website. There used to be a limited to how deeply nested dynamic components could be, but this limitation is now lifted too.
  - Add an `id` attribute to form fields in the [form](https://sql.ophir.dev/documentation.sql?component=form#component) component. This allows you to easily reference form fields in custom javascript code.
  - New [`rss`](https://sql.ophir.dev/documentation.sql?component=rss#component) component to create RSS feeds, including **podcast feeds**. You can now create and manage your podcast feed entirely in SQL, and distribute it to all podcast directories such as Apple Podcasts, Spotify, and Google Podcasts.
  - Better error handling in template rendering. Many template helpers now display a more precise error message when they fail to execute. This makes it easier to debug errors when you [develop your own custom components](https://sql.ophir.dev/custom_components.sql).
diff --git a/tests/sql_test_files/error_too_many_nested_dynamic_components.sql b/tests/sql_test_files/it_works_deeply_nested_dynamic_components.sql
similarity index 100%
rename from tests/sql_test_files/error_too_many_nested_dynamic_components.sql
rename to tests/sql_test_files/it_works_deeply_nested_dynamic_components.sql