Move move_money into a wasm module, add dynamic node test that uses move_money

.gitea/workflows/test.yaml

@@ -11,5 +11,8 @@ jobs:
           lfs: true
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
+      # Build webassembly module
+      - run: cargo add cargo-component
+      - run: cargo component build --target wasm32-unknown-unknown --release -p move-money-dynamic
       # Limit to one test at a time as the CI infrastructure struggles if trying to start too many containers at once
       - run: cargo test --release

Cargo.lock

@@ -2371,6 +2371,17 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "move-money-dynamic"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "csv",
+ "itertools 0.14.0",
+ "serde",
+ "wit-bindgen-rt",
+]
+
 [[package]]
 name = "multiversion"
 version = "0.7.4"
@@ -5922,6 +5933,15 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "wit-bindgen-rt"
+version = "0.24.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b0780cf7046630ed70f689a098cd8d56c5c3b22f2a7379bbdb088879963ff96"
+dependencies = [
+ "bitflags 2.6.0",
+]
+
 [[package]]
 name = "wit-parser"
 version = "0.221.2"

Cargo.toml

@@ -3,6 +3,11 @@ name = "coster-rs"
 version = "0.1.0"
 edition = "2021"
 
+[workspace]
+members = ["move-money-dynamic"]
+
+resolver = "2"
+
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]

README.md

@@ -18,3 +18,11 @@ The schema will be written to a schema.json file.
 Setting the number of threads in overhead allocation: set RAYON_NUM_THREADS environment
 variable ([doc](https://github.com/rayon-rs/rayon/blob/master/FAQ.md)).
 Note multithreading is only currently used to calculate allocation percentages.
+
+## Building Webassembly modules
+
+Currently there's one webassembly module, move-money-dynamic. This can be built by running the following command:
+
+`cargo component build --target wasm32-unknown-unknown --release -p move-money-dynamic`
+
+Note that in order to run the 

move-money-dynamic/src/lib.rs

@@ -0,0 +1,47 @@
+use crate::bindings::{CsvReaders, CsvWriter, Guest, ReadMap};
+use crate::move_money::move_money;
+
+#[allow(warnings)]
+mod bindings;
+pub mod move_money;
+
+struct Component;
+
+impl Guest for Component {
+    fn evaluate(properties: ReadMap, readers: CsvReaders, writer: CsvWriter) -> () {
+        let accounts_reader = readers.read_into_string("Account");
+        let cost_centres_reader = readers.read_into_string("CostCentre");
+        let lines = readers.read_into_string("Line");
+        let rules = readers.read_into_string("Rule");
+
+        let use_numeric_accounts = properties
+            .get("use_numeric_accounts")
+            .map(|param| param == "true")
+            .unwrap_or(false);
+        let flush_pass = properties
+            .get("flush_pass")
+            .map(|param| param == "true")
+            .unwrap_or(false);
+
+        let mut output_writer = csv::Writer::from_writer(vec![]);
+        let result = move_money(
+            &mut csv::Reader::from_reader(rules.as_bytes()),
+            &mut csv::Reader::from_reader(lines.as_bytes()),
+            &mut csv::Reader::from_reader(accounts_reader.as_bytes()),
+            &mut csv::Reader::from_reader(cost_centres_reader.as_bytes()),
+            &mut output_writer,
+            use_numeric_accounts,
+            flush_pass,
+        );
+        match result {
+            Ok(_) => {
+                let inner = output_writer.into_inner().unwrap();
+                let wrapped = String::from_utf8(inner).unwrap();
+                writer.write_string(wrapped.as_str());
+            }
+            Err(e) => {}
+        }
+    }
+}
+
+bindings::export!(Component with_types_in bindings);

move-money-dynamic/src/move_money.rs

@@ -1,9 +1,19 @@
-use std::collections::{HashMap, HashSet};
-
 use itertools::Itertools;
 use serde::{Deserialize, Serialize, Serializer};
-
+use std::collections::{HashMap, HashSet};
-use crate::CsvAccount;
+#[derive(Deserialize)]
+pub struct CsvAccount {
+    #[serde(rename = "Code")]
+    pub code: String,
+    #[serde(rename = "Description")]
+    pub description: Option<String>,
+    #[serde(rename = "Type")]
+    pub account_type: String,
+    #[serde(rename = "CostOutput")]
+    pub cost_output: Option<String>,
+    #[serde(rename = "PercentFixed")]
+    pub percent_fixed: f64,
+}
 
 #[derive(Debug, Deserialize)]
 struct CsvMovementRule {
@@ -425,11 +435,11 @@ mod tests {
     #[test]
     fn move_money() {
         super::move_money(
-            &mut csv::Reader::from_path("testing/input/move_money/reclassrule.csv").unwrap(),
+            &mut csv::Reader::from_path("../testing/input/move_money/reclassrule.csv").unwrap(),
-            &mut csv::Reader::from_path("testing/input/move_money/line.csv").unwrap(),
+            &mut csv::Reader::from_path("../testing/input/move_money/line.csv").unwrap(),
-            &mut csv::Reader::from_path("testing/input/account.csv").unwrap(),
+            &mut csv::Reader::from_path("../testing/input/account.csv").unwrap(),
-            &mut csv::Reader::from_path("testing/input/costcentre.csv").unwrap(),
+            &mut csv::Reader::from_path("../testing/input/costcentre.csv").unwrap(),
-            &mut csv::Writer::from_path("testing/output/output.csv").unwrap(),
+            &mut csv::Writer::from_path("../testing/output/output.csv").unwrap(),
             false,
             true,
         )

move-money-dynamic/wit/dynamic_node.wit

@@ -0,0 +1,43 @@
+package vato007:ingey;
+
+interface types {
+    resource csv-row {
+        columns: func() -> list<string>;
+        values: func() -> list<string>;
+        entries: func() -> list<tuple<string, string>>;
+        value: func(name: string) -> option<string>;
+    }
+
+    resource csv-reader {
+        columns: func() -> list<string>;
+
+        next: func() -> result<csv-row, string>;
+        next-into-map: func() -> read-map;
+        has-next: func() -> bool;
+
+        // Get a row by values in one or more columns
+        query: func(values: list<tuple<string, string>>) -> csv-row;
+
+        read-into-string: func() -> string;
+    }
+
+    resource csv-readers {
+        get-reader: func(name: string) -> option<csv-reader>;
+        read-into-string: func(name: string) -> string;
+    }
+
+    resource csv-writer {
+        write-row: func(row: list<tuple<string, string>>);
+        write-string: func(row: string);
+    }
+
+    resource read-map {
+        get: func(key: string) -> option<string>;
+    }
+}
+
+// This will apply to csv files only for simplicity. A separate node should be created for arbitrary readers/writers
+world dynamic {
+    use types.{csv-readers, read-map, csv-writer};
+    export evaluate: func(properties: read-map, readers: csv-readers, writer: csv-writer);
+}

src/cli/commands.rs

@@ -4,29 +4,30 @@ use clap::Subcommand;
 
 #[derive(Subcommand)]
 pub enum Commands {
-    /// Moves money between accounts and departments, using the given rules and lines
+    // TODO: Use wasm to do this instead
-    MoveMoney {
+    // /// Moves money between accounts and departments, using the given rules and lines
-        #[arg(short = 'r', long, value_name = "FILE")]
+    // MoveMoney {
-        rules: PathBuf,
+    //     #[arg(short = 'r', long, value_name = "FILE")]
-
+    //     rules: PathBuf,
-        #[arg(short = 'l', long, value_name = "FILE")]
+    //
-        lines: PathBuf,
+    //     #[arg(short = 'l', long, value_name = "FILE")]
-
+    //     lines: PathBuf,
-        #[arg(short = 'a', long, value_name = "FILE")]
+    //
-        accounts: PathBuf,
+    //     #[arg(short = 'a', long, value_name = "FILE")]
-
+    //     accounts: PathBuf,
-        #[arg(short = 'c', long, value_name = "FILE")]
+    //
-        cost_centres: PathBuf,
+    //     #[arg(short = 'c', long, value_name = "FILE")]
-
+    //     cost_centres: PathBuf,
-        #[arg(short, long, value_name = "FILE")]
+    //
-        output: Option<PathBuf>,
+    //     #[arg(short, long, value_name = "FILE")]
-
+    //     output: Option<PathBuf>,
-        #[arg(short, long)]
+    //
-        use_numeric_accounts: bool,
+    //     #[arg(short, long)]
-
+    //     use_numeric_accounts: bool,
-        #[arg(short, long)]
+    //
-        flush_pass: bool,
+    //     #[arg(short, long)]
-    },
+    //     flush_pass: bool,
+    // },
     /// Allocates servicing department amounts to operating departments
     AllocateOverheads {
         #[arg(short, long, value_name = "FILE")]

src/cli/mod.rs

@@ -35,23 +35,6 @@ pub struct Cli {
 impl Cli {
     pub async fn run(self) -> anyhow::Result<()> {
         match self.command {
-            Commands::MoveMoney {
-                rules,
-                lines,
-                accounts,
-                cost_centres,
-                output,
-                use_numeric_accounts,
-                flush_pass,
-            } => crate::move_money(
-                &mut csv::Reader::from_path(rules)?,
-                &mut csv::Reader::from_path(lines)?,
-                &mut csv::Reader::from_path(accounts)?,
-                &mut csv::Reader::from_path(cost_centres)?,
-                &mut csv::Writer::from_path(output.unwrap_or(PathBuf::from("output.csv")))?,
-                use_numeric_accounts,
-                flush_pass,
-            ),
             Commands::AllocateOverheads {
                 lines,
                 accounts,

src/graph/dynamic/csv_readers.rs

@@ -3,6 +3,7 @@ use super::{
     dynamic_state::{vato007::ingey::types::HostCsvReaders, DynamicState},
 };
 use std::collections::HashMap;
+use std::fs;
 use wasmtime::component::Resource;
 
 pub struct CsvReadersData {
@@ -33,4 +34,18 @@ impl HostCsvReaders for DynamicState {
         self.resources.delete(rep)?;
         Ok(())
     }
+
+    fn read_into_string(&mut self, self_: Resource<CsvReadersData>, name: String) -> String {
+        let resource = self
+            .resources
+            .get(&self_)
+            .expect("Failed to find own resource");
+        let file_path = resource.readers.get(&name);
+        if let Some(path) = file_path.cloned() {
+            let file = fs::read_to_string(path);
+            file.unwrap_or(String::new())
+        } else {
+            String::new()
+        }
+    }
 }

src/graph/dynamic/csv_writer.rs

@@ -2,19 +2,22 @@ use super::dynamic_state::{vato007::ingey::types::HostCsvWriter, DynamicState};
 use crate::io::RecordSerializer;
 use csv::Writer;
 use std::collections::BTreeMap;
+use std::fs;
 use std::fs::File;
 
 pub struct CsvWriterData {
     writer: Writer<File>,
     wrote_header: bool,
+    path: String,
 }
 
 impl CsvWriterData {
     pub fn new(path: String) -> anyhow::Result<Self> {
-        let writer = Writer::from_path(path)?;
+        let writer = Writer::from_path(path.clone())?;
         Ok(CsvWriterData {
             writer,
             wrote_header: false,
+            path,
         })
     }
 }
@@ -47,4 +50,15 @@ impl HostCsvWriter for DynamicState {
         self.resources.delete(rep)?;
         Ok(())
     }
+    fn write_string(
+        &mut self,
+        self_: wasmtime::component::Resource<CsvWriterData>,
+        contents: String,
+    ) {
+        let resource = self
+            .resources
+            .get_mut(&self_)
+            .expect("Failed to find resource");
+        fs::write(&resource.path, contents);
+    }
 }

src/graph/dynamic/mod.rs

@@ -23,9 +23,10 @@ pub struct DynamicNode {
     pub wasm_file_path: String,
     pub input_file_paths: HashMap<String, String>,
     pub output_file: String,
+    pub properties: HashMap<String, String>,
 }
 
-// Node to run arbitrary webassembly code to transorm one or more csv files
+// Node to run arbitrary webassembly code to transform one or more csv files
 // TODO: Create a separate node for wit that allows arbitrary files
 pub struct DynamicNodeRunner {
     pub dynamic_node: DynamicNode,
@@ -41,7 +42,7 @@ impl RunnableNode for DynamicNodeRunner {
         let mut store = Store::new(&engine, DynamicState::new());
         let bindings = Dynamic::instantiate(&mut store, &component, &linker)?;
         let read_map = store.data_mut().resources.push(ReadMapData {
-            data: HashMap::new(),
+            data: self.dynamic_node.properties.clone(),
         })?;
         let readers = store.data_mut().resources.push(CsvReadersData {
             readers: self.dynamic_node.input_file_paths.clone(),
@@ -52,3 +53,42 @@ impl RunnableNode for DynamicNodeRunner {
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::graph::dynamic::{DynamicNode, DynamicNodeRunner};
+    use crate::graph::node::RunnableNode;
+    use std::collections::HashMap;
+
+    #[tokio::test]
+    async fn move_money_wasm() -> anyhow::Result<()> {
+        let mut input_file_paths = HashMap::new();
+        input_file_paths.insert("Account".to_owned(), "testing/input/account.csv".to_owned());
+        input_file_paths.insert(
+            "CostCentre".to_owned(),
+            "testing/input/costcentre.csv".to_owned(),
+        );
+        input_file_paths.insert(
+            "Line".to_owned(),
+            "testing/input/move_money/line.csv".to_owned(),
+        );
+        input_file_paths.insert(
+            "Rule".to_owned(),
+            "testing/input/move_money/reclassrule.csv".to_owned(),
+        );
+        let mut properties = HashMap::new();
+        properties.insert("flush_pass".to_owned(), "false".to_owned());
+        // properties.insert("flush_pass".to_owned(), "true".to_owned());
+        properties.insert("use_numeric_accounts".to_owned(), "false".to_owned());
+        let dynamic_node = DynamicNode {
+            input_file_paths,
+            output_file: "testing/output/output.csv".to_owned(),
+            wasm_file_path: "target/wasm32-unknown-unknown/release/move_money_dynamic.wasm"
+                .to_owned(),
+            properties,
+        };
+        let runner = DynamicNodeRunner { dynamic_node };
+        runner.run().await?;
+        Ok(())
+    }
+}

src/lib.rs

@@ -1,6 +1,3 @@
-// TODO: Module api can probably use a cleanup
-mod move_money;
-pub use self::move_money::*;
 use std::ffi::c_char;
 use std::ffi::CStr;
 use std::ffi::CString;
@@ -16,68 +13,6 @@ pub mod graph;
 mod io;
 pub mod link;
 
-#[no_mangle]
-pub extern "C" fn move_money_from_text(
-    rules: *const c_char,
-    lines: *const c_char,
-    accounts: *const c_char,
-    cost_centres: *const c_char,
-    use_numeric_accounts: bool,
-) -> *mut c_char {
-    let mut output_writer = csv::Writer::from_writer(vec![]);
-    let safe_rules = unwrap_c_char(rules);
-    let safe_lines = unwrap_c_char(lines);
-    let safe_accounts = unwrap_c_char(accounts);
-    let safe_cost_centres = unwrap_c_char(cost_centres);
-    move_money(
-        &mut csv::Reader::from_reader(safe_rules.to_bytes()),
-        &mut csv::Reader::from_reader(safe_lines.to_bytes()),
-        &mut csv::Reader::from_reader(safe_accounts.to_bytes()),
-        &mut csv::Reader::from_reader(safe_cost_centres.to_bytes()),
-        &mut output_writer,
-        use_numeric_accounts,
-        false,
-    )
-    .expect("Failed to move money");
-    // TODO: Replace all these unwraps with something more elegant
-    let inner = output_writer.into_inner().unwrap();
-    CString::new(String::from_utf8(inner).unwrap())
-        .unwrap()
-        .into_raw()
-
-    // Also some resources I looked at, in case things aren't going right:
-    // https://notes.huy.rocks/en/string-ffi-rust.html
-    // http://jakegoulding.com/rust-ffi-omnibus/string_return/
-    // https://rust-unofficial.github.io/patterns/idioms/ffi/passing-strings.html
-    // This looks like exactly what I'm doing too: https://mozilla.github.io/firefox-browser-architecture/experiments/2017-09-06-rust-on-ios.htmlcar
-}
-
-#[no_mangle]
-pub extern "C" fn move_money_from_file(
-    rules_file: *const c_char,
-    lines: *const c_char,
-    accounts: *const c_char,
-    cost_centres: *const c_char,
-    output_path: *const c_char,
-    use_numeric_accounts: bool,
-) {
-    let safe_rules = unwrap_c_char(rules_file);
-    let safe_lines = unwrap_c_char(lines);
-    let safe_accounts = unwrap_c_char(accounts);
-    let safe_cost_centres = unwrap_c_char(cost_centres);
-    let output_path = unwrap_c_char(output_path);
-    move_money(
-        &mut csv::Reader::from_reader(safe_rules.to_bytes()),
-        &mut csv::Reader::from_reader(safe_lines.to_bytes()),
-        &mut csv::Reader::from_reader(safe_accounts.to_bytes()),
-        &mut csv::Reader::from_reader(safe_cost_centres.to_bytes()),
-        &mut csv::Writer::from_path(output_path.to_str().unwrap()).unwrap(),
-        use_numeric_accounts,
-        false,
-    )
-    .expect("Failed to move money");
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn move_money_from_text_free(s: *mut c_char) {
     unsafe {

src/shared_models.rs

@@ -1,4 +1,4 @@
-use serde::Deserialize;
+use serde::{Deserialize, Serialize, Serializer};
 
 #[derive(Deserialize)]
 pub struct CsvAccount {
@@ -13,3 +13,20 @@ pub struct CsvAccount {
     #[serde(rename = "PercentFixed")]
     pub percent_fixed: f64,
 }
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct CsvCost {
+    #[serde(rename = "ACCOUNT")]
+    pub account: String,
+    #[serde(rename = "COSTCENTRE")]
+    pub department: String,
+    #[serde(serialize_with = "round_serialize")]
+    pub value: f64,
+}
+
+fn round_serialize<S>(x: &f64, s: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    s.serialize_f64((x * 100000.).round() / 100000.)
+}

wit/dynamic_node.wit

@@ -23,10 +23,12 @@ interface types {
 
     resource csv-readers {
         get-reader: func(name: string) -> option<csv-reader>;
+        read-into-string: func(name: string) -> string;
     }
 
     resource csv-writer {
         write-row: func(row: list<tuple<string, string>>);
+        write-string: func(row: string);
     }
 
     resource read-map {