Escape quotes in bulk upload methods
All checks were successful
test / test (push) Successful in 10m36s

This commit is contained in:
2025-02-15 18:23:00 +10:30
parent c0e9e7ef13
commit 02772a01a8

View File

@@ -25,18 +25,24 @@ pub async fn upload_file_bulk(
let insert_from_file_query = match upload_node.db_type {
DBType::Postgres => Some(format!(
r#"COPY "{}" FROM '{}' DELIMITERS ',' CSV HEADER QUOTE '"';"#,
upload_node.table_name, upload_node.file_path
upload_node.table_name.replace("'", "''"),
upload_node.file_path.replace("'", "''")
)),
// This can't be parameterised as mysql doesn't allow LOAD DATA
// to be executed within a prepare. Therefore even dynamic sql
// can't work (like it can with mssql) since mysql won't allow it
DBType::Mysql => Some(format!(
"LOAD DATA INFILE '{}' INTO TABLE `{}`
FIELDS TERMINATED BY ',' ENCLOSED BY '\"'
LINES TERMINATED BY '\n'
IGNORE 1 LINES;",
upload_node.file_path, upload_node.table_name,
upload_node.file_path.replace("'", "''"),
upload_node.table_name.replace("'", "''"),
)),
DBType::Mssql => Some(format!(
"BULK INSERT [{}] FROM '{}' WITH ( FORMAT = 'CSV', FIRSTROW = 2 );",
upload_node.table_name, upload_node.file_path
upload_node.table_name.replace("'", "''"),
upload_node.file_path.replace("'", "''")
)),
_ => None,
};
@@ -47,7 +53,7 @@ pub async fn upload_file_bulk(
} else {
log!(
Level::Debug,
"Failed to bulk insert, trying sql insert instead"
"Failed to bulk insert, trying standard sql insert instead"
);
}
}
@@ -91,6 +97,10 @@ pub async fn upload_file_bulk(
// TODO: This is done because postgres won't do an implicit cast on text in the insert,
// if this is resolved we can go back to parameterised queries instead
// as mysql/sql server work fine with implicit cast on parameters
// Alternatively, we could inspect the table metadata, though a
// good DBA should prevent the db user from accessing this information
// or performing bad queries like deletes if a user somehow breaks
// quote escapes - we'll need to doc this limitation
.map(|value| format!("'{}'", value.replace("'", "''")))
.join(",")
)