better csv ingestion, with tests and docs

Author: lovasoa

CHANGELOG.md

@@ -30,10 +30,17 @@ insert into files (content) values (sqlpage.read_file_as_data_url(sqlpage.upload
 returning 'text' as component, 'Uploaded new file with id: ' || id as contents;
 ```
 
+The maximum size of uploaded files is configurable with the [`max_uploaded_file_size`](./configuration.md) configuration parameter. By default, it is set to 5 MiB.
+
 #### Parsing CSV files
 
 SQLPage can also parse uploaded CSV files and insert them directly into a database table.
-SQLPage re-uses PostgreSQL's `COPY` statement to import the CSV file into the database, but makes it work with any database, by emulating the same behavior with simple `INSERT` statements.
+SQLPage re-uses PostgreSQL's [`COPY` syntax](https://www.postgresql.org/docs/current/sql-copy.html)
+to import the CSV file into the database.
+When connected to a PostgreSQL database, SQLPage will use the native `COPY` statement,
+for super fast and efficient on-database CSV parsing.
+But it will also work with any other database as well, by
+parsing the CSV locally and emulating the same behavior with simple `INSERT` statements.
 
 `user_file_upload.sql` :
 ```sql

configuration.md

@@ -16,7 +16,7 @@ on a [JSON](https://en.wikipedia.org/wiki/JSON) file placed in `sqlpage/sqlpage.
 | `sqlite_extensions`                        |                              | An array of SQLite extensions to load, such as `mod_spatialite`          |
 | `web_root`                                | `.`                      | The root directory of the web server, where the `index.sql` file is located.          |
 | `allow_exec` | false | Allow usage of the `sqlpage.exec` function. Do this only if all users with write access to sqlpage query files and to the optional `sqlpage_files` table on the database are trusted. |
-| `max_uploaded_file_size` | 10485760 | Maximum size of uploaded files in bytes. Defaults to 10 MiB. |
+| `max_uploaded_file_size` | 5242880 | Maximum size of uploaded files in bytes. Defaults to 5 MiB. |
 | `https_domain` | | Domain name to request a certificate for. Setting this parameter will automatically make SQLPage listen on port 443 and request an SSL certificate. The server will take a little bit longer to start the first time it has to request a certificate. |
 | `https_certificate_email` | contact@<https_domain> | The email address to use when requesting a certificate. |
 | `https_certificate_cache_dir` | ./sqlpage/https | A writeable directory where to cache the certificates, so that SQLPage can serve https traffic immediately when it restarts. |

examples/official-site/examples/handle_csv_upload.sql

@@ -0,0 +1,9 @@
+-- temporarily store the data in a table with text columns
+create temporary table if not exists product_tmp(name text, description text, price text);
+delete from product_tmp;
+
+-- copy the data from the CSV file into the temporary table
+copy product_tmp(name, description, price) from 'product_data_file';
+
+select 'table' as component;
+select * from product_tmp;

examples/official-site/sqlpage/migrations/01_documentation.sql

@@ -394,6 +394,58 @@ INSERT INTO uploaded_file(name, data) VALUES(:filename, sqlpage.uploaded_file_da
 ',
     json('[{"component":"form", "title": "Upload a picture", "validate": "Upload", "action": "examples/handle_picture_upload.sql"}, 
     {"name": "my_file", "type": "file", "accept": "image/png, image/jpeg",  "label": "Picture", "description": "Upload a nice picture", "required": true}
+    ]')),
+    ('form', '
+## Bulk data insertion
+
+You can use the `file` type to allow the user to upload a [CSV](https://en.wikipedia.org/wiki/Comma-separated_values) 
+file containing data to insert in a table.
+
+SQLPage can load data from a CSV file and insert it into a database table.
+SQLPage re-uses PostgreSQL''s [`COPY` syntax](https://www.postgresql.org/docs/current/sql-copy.html)
+to specify the format of the CSV file, but makes it work with all supported databases.
+
+> When connected to a PostgreSQL database, SQLPage will use the native `COPY` statement,
+> for super fast and efficient on-database CSV parsing.
+> But it will also work transparently with other databases, by
+> parsing the CSV locally and emulating the same behavior with simple `INSERT` statements.
+
+Here is how you could easily copy data from a CSV to a table in the database:
+
+```sql
+copy product(name, description) from ''product_data_input''
+with (header true, delimiter '','', quote ''"'');
+```
+
+If you want to pre-process the data before inserting it into the final table,
+you can use a temporary table to store the data, and then insert it into the final table:
+
+```sql
+-- temporarily store the data in a table with text columns
+create temporary table if not exists product_tmp(name text, description text, price text);
+delete from product_tmp;
+
+-- copy the data from the CSV file into the temporary table
+copy product_tmp(name, description, price) from ''product_data_input'';
+
+-- insert the data into the final table, converting the price column to an integer
+insert into product(name, description, price)
+select name, description, CAST(price AS integer) from product_tmp
+where price is not null and description is not null and length(description) > 10;
+```
+
+This will load the processed CSV into the product table, provided it has the following structure:
+
+```csv
+name,description,price
+"SQLPage","A tool to create websites using SQL",0
+"PostgreSQL","A powerful open-source relational database",0
+"SQLite","A lightweight relational database",0
+"MySQL","A popular open-source relational database",0
+```
+',
+    json('[{"component":"form", "title": "CSV import", "validate": "Load data", "action": "examples/handle_csv_upload.sql"}, 
+    {"name": "product_data_input", "type": "file", "accept": "text/csv",  "label": "Products", "description": "Upload a CSV with a name, description, and price columns", "required": true}
     ]'))
 ;
 

examples/official-site/sqlpage/sqlpage.json

@@ -0,0 +1,5 @@
+# The documentation site is fully static, so we don't need to persist any data.
+database_url": "sqlite::memory:"
+
+# We have a file upload example, and would like to limit the size of the uploaded files 
+max_uploaded_file_size: 256000

examples/official-site/sqlpage/sqlpage.yaml

@@ -163,7 +163,7 @@ fn default_web_root() -> PathBuf {
 }
 
 fn default_max_file_size() -> usize {
-    10 * 1024 * 1024
+    5 * 1024 * 1024
 }
 
 fn default_https_certificate_cache_dir() -> PathBuf {

src/app_config.rs

@@ -6,8 +6,8 @@ use sqlparser::ast::{
     CopyLegacyCsvOption, CopyLegacyOption, CopyOption, CopySource, CopyTarget, Statement,
 };
 use sqlx::{
-    any::{AnyArguments, AnyKind},
-    AnyConnection, Arguments, Executor,
+    any::{AnyArguments, AnyConnectionKind, AnyKind},
+    AnyConnection, Arguments, Executor, PgConnection,
 };
 use tokio::io::AsyncRead;
 
@@ -80,7 +80,7 @@ impl<'a> CopyCsvOption<'a> {
     }
 }
 
-pub fn extract_csv_copy_statement(stmt: &mut Statement) -> Option<CsvImport> {
+pub(super) fn extract_csv_copy_statement(stmt: &mut Statement) -> Option<CsvImport> {
     if let Statement::Copy {
         source: CopySource::Table {
             table_name,
@@ -137,7 +137,6 @@ pub fn extract_csv_copy_statement(stmt: &mut Statement) -> Option<CsvImport> {
             uploaded_file,
         })
     } else {
-        log::warn!("COPY statement not compatible with SQLPage: {stmt}");
         None
     }
 }
@@ -157,10 +156,39 @@ pub(super) async fn run_csv_import(
         .await
         .with_context(|| "opening csv")?;
     let buffered = tokio::io::BufReader::new(file);
-    run_csv_import_on_path(db, csv_import, buffered).await
+    // private_get_mut is not supposed to be used outside of sqlx, but it is the only way to
+    // access the underlying connection
+    match db.private_get_mut() {
+        AnyConnectionKind::Postgres(pg_connection) => {
+            run_csv_import_postgres(pg_connection, csv_import, buffered).await
+        }
+        _ => run_csv_import_insert(db, csv_import, buffered).await,
+    }
+    .with_context(|| {
+        format!(
+            "running CSV import from {} to {}",
+            csv_import.uploaded_file, csv_import.table_name
+        )
+    })
+}
+
+/// This function does not parse the CSV file, it only sends it to postgres.
+/// This is the fastest way to import a CSV file into postgres
+async fn run_csv_import_postgres(
+    db: &mut PgConnection,
+    csv_import: &CsvImport,
+    file: impl AsyncRead + Unpin + Send,
+) -> anyhow::Result<()> {
+    let mut copy_transact = db
+        .copy_in_raw(csv_import.query.as_str())
+        .await
+        .with_context(|| "running COPY IN")?;
+    copy_transact.read_from(file).await?;
+    copy_transact.finish().await?;
+    Ok(())
 }
 
-async fn run_csv_import_on_path(
+async fn run_csv_import_insert(
     db: &mut AnyConnection,
     csv_import: &CsvImport,
     file: impl AsyncRead + Unpin + Send,
@@ -287,13 +315,27 @@ async fn test_end_to_end() {
 
     let mut copy_stmt = sqlparser::parser::Parser::parse_sql(
         &sqlparser::dialect::GenericDialect {},
-        "COPY my_table (col1, col2) FROM 'my_file.csv' WITH (DELIMITER ';', HEADER)",
+        "COPY my_table (col1, col2) FROM 'my_file.csv' (DELIMITER ';', HEADER)",
     )
     .unwrap()
     .into_iter()
     .next()
     .unwrap();
     let csv_import = extract_csv_copy_statement(&mut copy_stmt).unwrap();
+    assert_eq!(
+        csv_import,
+        CsvImport {
+            query: "COPY my_table (col1, col2) FROM STDIN (DELIMITER ';', HEADER)".into(),
+            table_name: "my_table".into(),
+            columns: vec!["col1".into(), "col2".into()],
+            delimiter: Some(';'),
+            quote: None,
+            header: Some(true),
+            null_str: None,
+            escape: None,
+            uploaded_file: "my_file.csv".into(),
+        }
+    );
     let mut conn = "sqlite::memory:"
         .parse::<sqlx::any::AnyConnectOptions>()
         .unwrap()
@@ -305,7 +347,7 @@ async fn test_end_to_end() {
         .unwrap();
     let csv = "col2;col1\na;b\nc;d"; // order is different from the table
     let file = csv.as_bytes();
-    run_csv_import_on_path(&mut conn, &csv_import, file)
+    run_csv_import_insert(&mut conn, &csv_import, file)
         .await
         .unwrap();
     let rows: Vec<(String, String)> = sqlx::query_as("SELECT * FROM my_table")

src/webserver/database/csv_import.rs

@@ -92,7 +92,7 @@ pub fn sql_nonnull_to_json<'r>(mut get_ref: impl FnMut() -> sqlx::any::AnyValueR
 
 /// Takes the first column of a row and converts it to a string.
 pub fn row_to_string(row: &AnyRow) -> Option<String> {
-    let col = row.columns().get(0)?;
+    let col = row.columns().first()?;
     match sql_to_json(row, col) {
         serde_json::Value::String(s) => Some(s),
         serde_json::Value::Null => None,

src/webserver/database/sql_to_json.rs

@@ -7,6 +7,7 @@ use actix_web::dev::{fn_service, ServiceFactory, ServiceRequest};
 use actix_web::error::ErrorInternalServerError;
 use actix_web::http::header::{ContentType, Header, HttpDate, IfModifiedSince, LastModified};
 use actix_web::http::{header, StatusCode, Uri};
+use actix_web::web::PayloadConfig;
 use actix_web::{
     dev::ServiceResponse, middleware, middleware::Logger, web, web::Bytes, App, HttpResponse,
     HttpServer,
@@ -476,6 +477,7 @@ pub fn create_app(
         .wrap(middleware::NormalizePath::new(
             middleware::TrailingSlash::MergeOnly,
         ))
+        .app_data(PayloadConfig::default().limit(app_state.config.max_uploaded_file_size * 2))
         .app_data(app_state)
 }
 

src/webserver/http.rs

@@ -112,6 +112,34 @@ async fn test_file_upload() -> actix_web::Result<()> {
     Ok(())
 }
 
+#[actix_web::test]
+async fn test_csv_upload() -> actix_web::Result<()> {
+    let req = get_request_to("/tests/upload_csv_test.sql")
+        .await?
+        .insert_header(("content-type", "multipart/form-data; boundary=1234567890"))
+        .set_payload(
+            "--1234567890\r\n\
+            Content-Disposition: form-data; name=\"prices_file\"; filename=\"prices.csv\"\r\n\
+            Content-Type: text/csv\r\n\
+            \r\n\
+            price,quantity\r\n\
+            1,5\r\n\
+            2.5,4\r\n\
+            --1234567890--\r\n",
+        )
+        .to_srv_request();
+    let resp = main_handler(req).await?;
+
+    assert_eq!(resp.status(), StatusCode::OK);
+    let body = test::read_body(resp).await;
+    let body_str = String::from_utf8(body.to_vec()).unwrap();
+    assert!(
+        body_str.contains("total: 15"),
+        "{body_str}\nexpected to contain: total: 15"
+    );
+    Ok(())
+}
+
 async fn get_request_to(path: &str) -> actix_web::Result<TestRequest> {
     init_log();
     let config = test_config();

tests/index.rs

@@ -0,0 +1,5 @@
+create table bill(quantity text, price text);
+copy bill(quantity, price) from 'prices_file' with (format csv, header true);
+select 'text' as component,
+    'total: ' || sum(cast(quantity as float) * cast(price as float)) as contents
+from bill;