Start adding row-level splitting, refactor cli and graph into subcrates

src/graph/sql_rule.rs

@@ -0,0 +1,82 @@
+use std::fs::File;
+
+use polars::{
+    io::SerWriter,
+    prelude::{CsvWriter, LazyCsvReader, LazyFileListReader},
+};
+use polars_sql::SQLContext;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+use super::node::RunnableNode;
+
+#[derive(Serialize, Deserialize, Clone, JsonSchema)]
+pub struct CSVFile {
+    name: String,
+    path: String,
+}
+
+/**
+ * Run SQL over files using polars, export results to output file
+ */
+fn run_sql(files: &Vec<CSVFile>, output_path: &String, query: &String) -> anyhow::Result<()> {
+    let mut ctx = SQLContext::new();
+    for file in files {
+        let df = LazyCsvReader::new(&file.path).finish()?;
+        ctx.register(&file.name, df);
+    }
+    let result = ctx.execute(&query)?;
+    let mut file = File::create(output_path)?;
+    CsvWriter::new(&mut file).finish(&mut result.collect()?)?;
+    Ok(())
+}
+
+#[derive(Serialize, Deserialize, Clone, JsonSchema)]
+pub struct SQLNode {
+    pub files: Vec<CSVFile>,
+    pub output_file: String,
+    pub query: String,
+}
+
+pub struct SQLNodeRunner {
+    pub sql_node: SQLNode,
+}
+
+impl RunnableNode for SQLNodeRunner {
+    fn run(&self) -> anyhow::Result<()> {
+        run_sql(
+            &self.sql_node.files,
+            &self.sql_node.output_file,
+            &self.sql_node.query,
+        )
+    }
+}
+#[cfg(test)]
+mod tests {
+    use std::{fs::File, io::Read};
+
+    use super::{run_sql, CSVFile};
+
+    #[test]
+    fn basic_query_works() -> anyhow::Result<()> {
+        let output_path = "./testing/output/output.csv".to_owned();
+        run_sql(
+            &vec![CSVFile {
+                name: "Account".to_owned(),
+                path: "./testing/test.csv".to_owned(),
+            }],
+            &output_path,
+            &"SELECT * FROM Account WHERE Code = 'A195950'".to_owned(),
+        )?;
+        let mut output = String::new();
+        let mut output_file = File::open(output_path)?;
+        output_file.read_to_string(&mut output)?;
+        assert_eq!(
+            output,
+            "Code,Description,Type,CostOutput,PercentFixed
+A195950,A195950 Staff Related Other,E,GS,100.00
+"
+        );
+        Ok(())
+    }
+}