vato007/ingey
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Escape quotes in bulk upload methods
All checks were successful
test / test (push) Successful in 10m36s
Details

Browse Source
This commit is contained in:
vato007
2025-02-15 18:23:00 +10:30
parent c0e9e7ef13
commit 02772a01a8
1 changed files with 14 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/graph/upload_to_db.rs
View File

@@ -25,18 +25,24 @@ pub async fn upload_file_bulk(
let insert_from_file_query = match upload_node.db_type { let insert_from_file_query = match upload_node.db_type {
DBType::Postgres => Some(format!( DBType::Postgres => Some(format!(
r#"COPY "{}" FROM '{}' DELIMITERS ',' CSV HEADER QUOTE '"';"#, r#"COPY "{}" FROM '{}' DELIMITERS ',' CSV HEADER QUOTE '"';"#,
upload_node.table_name, upload_node.file_path upload_node.table_name.replace("'", "''"),
upload_node.file_path.replace("'", "''")
)), )),
// This can't be parameterised as mysql doesn't allow LOAD DATA
// to be executed within a prepare. Therefore even dynamic sql
// can't work (like it can with mssql) since mysql won't allow it
DBType::Mysql => Some(format!( DBType::Mysql => Some(format!(
"LOAD DATA INFILE '{}' INTO TABLE `{}` "LOAD DATA INFILE '{}' INTO TABLE `{}`
FIELDS TERMINATED BY ',' ENCLOSED BY '\"' FIELDS TERMINATED BY ',' ENCLOSED BY '\"'
LINES TERMINATED BY '\n' LINES TERMINATED BY '\n'
IGNORE 1 LINES;", IGNORE 1 LINES;",
upload_node.file_path, upload_node.table_name, upload_node.file_path.replace("'", "''"),
upload_node.table_name.replace("'", "''"),
)), )),
DBType::Mssql => Some(format!( DBType::Mssql => Some(format!(
"BULK INSERT [{}] FROM '{}' WITH ( FORMAT = 'CSV', FIRSTROW = 2 );", "BULK INSERT [{}] FROM '{}' WITH ( FORMAT = 'CSV', FIRSTROW = 2 );",
upload_node.table_name, upload_node.file_path upload_node.table_name.replace("'", "''"),
upload_node.file_path.replace("'", "''")
)), )),
_ => None, _ => None,
}; };
@@ -47,7 +53,7 @@ pub async fn upload_file_bulk(
} else { } else {
log!( log!(
Level::Debug, Level::Debug,
"Failed to bulk insert, trying sql insert instead" "Failed to bulk insert, trying standard sql insert instead"
); );
} }
} }
@@ -91,6 +97,10 @@ pub async fn upload_file_bulk(
// TODO: This is done because postgres won't do an implicit cast on text in the insert, // TODO: This is done because postgres won't do an implicit cast on text in the insert,
// if this is resolved we can go back to parameterised queries instead // if this is resolved we can go back to parameterised queries instead
// as mysql/sql server work fine with implicit cast on parameters // as mysql/sql server work fine with implicit cast on parameters
// Alternatively, we could inspect the table metadata, though a
// good DBA should prevent the db user from accessing this information
// or performing bad queries like deletes if a user somehow breaks
// quote escapes - we'll need to doc this limitation
.map(|value| format!("'{}'", value.replace("'", "''"))) .map(|value| format!("'{}'", value.replace("'", "''")))
.join(",") .join(",")
) )