diff --git a/.gitea/workflows/test.yaml b/.gitea/workflows/test.yaml index 6c57511..20cfbbd 100644 --- a/.gitea/workflows/test.yaml +++ b/.gitea/workflows/test.yaml @@ -11,5 +11,8 @@ jobs: lfs: true - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 + # Build webassembly module + - run: cargo add cargo-component + - run: cargo component build --target wasm32-unknown-unknown --release -p move-money-dynamic # Limit to one test at a time as the CI infrastructure struggles if trying to start too many containers at once - run: cargo test --release diff --git a/Cargo.lock b/Cargo.lock index 1f9144f..055565f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2371,6 +2371,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "move-money-dynamic" +version = "0.1.0" +dependencies = [ + "anyhow", + "csv", + "itertools 0.14.0", + "serde", + "wit-bindgen-rt", +] + [[package]] name = "multiversion" version = "0.7.4" @@ -5922,6 +5933,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0780cf7046630ed70f689a098cd8d56c5c3b22f2a7379bbdb088879963ff96" +dependencies = [ + "bitflags 2.6.0", +] + [[package]] name = "wit-parser" version = "0.221.2" diff --git a/Cargo.toml b/Cargo.toml index 0eec36d..160b8df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,11 @@ name = "coster-rs" version = "0.1.0" edition = "2021" +[workspace] +members = ["move-money-dynamic"] + +resolver = "2" + # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] diff --git a/README.md b/README.md index ba88a60..5b5a0e4 100644 --- a/README.md +++ b/README.md @@ -18,3 +18,11 @@ The schema will be written to a schema.json file. Setting the number of threads in overhead allocation: set RAYON_NUM_THREADS environment variable ([doc](https://github.com/rayon-rs/rayon/blob/master/FAQ.md)). Note multithreading is only currently used to calculate allocation percentages. + +## Building Webassembly modules + +Currently there's one webassembly module, move-money-dynamic. This can be built by running the following command: + +`cargo component build --target wasm32-unknown-unknown --release -p move-money-dynamic` + +Note that in order to run the \ No newline at end of file diff --git a/move-money-dynamic/src/lib.rs b/move-money-dynamic/src/lib.rs new file mode 100644 index 0000000..499e52d --- /dev/null +++ b/move-money-dynamic/src/lib.rs @@ -0,0 +1,47 @@ +use crate::bindings::{CsvReaders, CsvWriter, Guest, ReadMap}; +use crate::move_money::move_money; + +#[allow(warnings)] +mod bindings; +pub mod move_money; + +struct Component; + +impl Guest for Component { + fn evaluate(properties: ReadMap, readers: CsvReaders, writer: CsvWriter) -> () { + let accounts_reader = readers.read_into_string("Account"); + let cost_centres_reader = readers.read_into_string("CostCentre"); + let lines = readers.read_into_string("Line"); + let rules = readers.read_into_string("Rule"); + + let use_numeric_accounts = properties + .get("use_numeric_accounts") + .map(|param| param == "true") + .unwrap_or(false); + let flush_pass = properties + .get("flush_pass") + .map(|param| param == "true") + .unwrap_or(false); + + let mut output_writer = csv::Writer::from_writer(vec![]); + let result = move_money( + &mut csv::Reader::from_reader(rules.as_bytes()), + &mut csv::Reader::from_reader(lines.as_bytes()), + &mut csv::Reader::from_reader(accounts_reader.as_bytes()), + &mut csv::Reader::from_reader(cost_centres_reader.as_bytes()), + &mut output_writer, + use_numeric_accounts, + flush_pass, + ); + match result { + Ok(_) => { + let inner = output_writer.into_inner().unwrap(); + let wrapped = String::from_utf8(inner).unwrap(); + writer.write_string(wrapped.as_str()); + } + Err(e) => {} + } + } +} + +bindings::export!(Component with_types_in bindings); diff --git a/src/move_money.rs b/move-money-dynamic/src/move_money.rs similarity index 94% rename from src/move_money.rs rename to move-money-dynamic/src/move_money.rs index c6f7fa1..47d1aea 100644 --- a/src/move_money.rs +++ b/move-money-dynamic/src/move_money.rs @@ -1,9 +1,19 @@ -use std::collections::{HashMap, HashSet}; - use itertools::Itertools; use serde::{Deserialize, Serialize, Serializer}; - -use crate::CsvAccount; +use std::collections::{HashMap, HashSet}; +#[derive(Deserialize)] +pub struct CsvAccount { + #[serde(rename = "Code")] + pub code: String, + #[serde(rename = "Description")] + pub description: Option, + #[serde(rename = "Type")] + pub account_type: String, + #[serde(rename = "CostOutput")] + pub cost_output: Option, + #[serde(rename = "PercentFixed")] + pub percent_fixed: f64, +} #[derive(Debug, Deserialize)] struct CsvMovementRule { @@ -425,11 +435,11 @@ mod tests { #[test] fn move_money() { super::move_money( - &mut csv::Reader::from_path("testing/input/move_money/reclassrule.csv").unwrap(), - &mut csv::Reader::from_path("testing/input/move_money/line.csv").unwrap(), - &mut csv::Reader::from_path("testing/input/account.csv").unwrap(), - &mut csv::Reader::from_path("testing/input/costcentre.csv").unwrap(), - &mut csv::Writer::from_path("testing/output/output.csv").unwrap(), + &mut csv::Reader::from_path("../testing/input/move_money/reclassrule.csv").unwrap(), + &mut csv::Reader::from_path("../testing/input/move_money/line.csv").unwrap(), + &mut csv::Reader::from_path("../testing/input/account.csv").unwrap(), + &mut csv::Reader::from_path("../testing/input/costcentre.csv").unwrap(), + &mut csv::Writer::from_path("../testing/output/output.csv").unwrap(), false, true, ) diff --git a/move-money-dynamic/wit/dynamic_node.wit b/move-money-dynamic/wit/dynamic_node.wit new file mode 100644 index 0000000..e45cb91 --- /dev/null +++ b/move-money-dynamic/wit/dynamic_node.wit @@ -0,0 +1,43 @@ +package vato007:ingey; + +interface types { + resource csv-row { + columns: func() -> list; + values: func() -> list; + entries: func() -> list>; + value: func(name: string) -> option; + } + + resource csv-reader { + columns: func() -> list; + + next: func() -> result; + next-into-map: func() -> read-map; + has-next: func() -> bool; + + // Get a row by values in one or more columns + query: func(values: list>) -> csv-row; + + read-into-string: func() -> string; + } + + resource csv-readers { + get-reader: func(name: string) -> option; + read-into-string: func(name: string) -> string; + } + + resource csv-writer { + write-row: func(row: list>); + write-string: func(row: string); + } + + resource read-map { + get: func(key: string) -> option; + } +} + +// This will apply to csv files only for simplicity. A separate node should be created for arbitrary readers/writers +world dynamic { + use types.{csv-readers, read-map, csv-writer}; + export evaluate: func(properties: read-map, readers: csv-readers, writer: csv-writer); +} \ No newline at end of file diff --git a/src/cli/commands.rs b/src/cli/commands.rs index d5539f3..0e8dec5 100644 --- a/src/cli/commands.rs +++ b/src/cli/commands.rs @@ -4,29 +4,30 @@ use clap::Subcommand; #[derive(Subcommand)] pub enum Commands { - /// Moves money between accounts and departments, using the given rules and lines - MoveMoney { - #[arg(short = 'r', long, value_name = "FILE")] - rules: PathBuf, - - #[arg(short = 'l', long, value_name = "FILE")] - lines: PathBuf, - - #[arg(short = 'a', long, value_name = "FILE")] - accounts: PathBuf, - - #[arg(short = 'c', long, value_name = "FILE")] - cost_centres: PathBuf, - - #[arg(short, long, value_name = "FILE")] - output: Option, - - #[arg(short, long)] - use_numeric_accounts: bool, - - #[arg(short, long)] - flush_pass: bool, - }, + // TODO: Use wasm to do this instead + // /// Moves money between accounts and departments, using the given rules and lines + // MoveMoney { + // #[arg(short = 'r', long, value_name = "FILE")] + // rules: PathBuf, + // + // #[arg(short = 'l', long, value_name = "FILE")] + // lines: PathBuf, + // + // #[arg(short = 'a', long, value_name = "FILE")] + // accounts: PathBuf, + // + // #[arg(short = 'c', long, value_name = "FILE")] + // cost_centres: PathBuf, + // + // #[arg(short, long, value_name = "FILE")] + // output: Option, + // + // #[arg(short, long)] + // use_numeric_accounts: bool, + // + // #[arg(short, long)] + // flush_pass: bool, + // }, /// Allocates servicing department amounts to operating departments AllocateOverheads { #[arg(short, long, value_name = "FILE")] diff --git a/src/cli/mod.rs b/src/cli/mod.rs index c0ac61c..643c419 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -35,23 +35,6 @@ pub struct Cli { impl Cli { pub async fn run(self) -> anyhow::Result<()> { match self.command { - Commands::MoveMoney { - rules, - lines, - accounts, - cost_centres, - output, - use_numeric_accounts, - flush_pass, - } => crate::move_money( - &mut csv::Reader::from_path(rules)?, - &mut csv::Reader::from_path(lines)?, - &mut csv::Reader::from_path(accounts)?, - &mut csv::Reader::from_path(cost_centres)?, - &mut csv::Writer::from_path(output.unwrap_or(PathBuf::from("output.csv")))?, - use_numeric_accounts, - flush_pass, - ), Commands::AllocateOverheads { lines, accounts, diff --git a/src/graph/dynamic/csv_readers.rs b/src/graph/dynamic/csv_readers.rs index 14a5dfd..34adb84 100644 --- a/src/graph/dynamic/csv_readers.rs +++ b/src/graph/dynamic/csv_readers.rs @@ -3,6 +3,7 @@ use super::{ dynamic_state::{vato007::ingey::types::HostCsvReaders, DynamicState}, }; use std::collections::HashMap; +use std::fs; use wasmtime::component::Resource; pub struct CsvReadersData { @@ -33,4 +34,18 @@ impl HostCsvReaders for DynamicState { self.resources.delete(rep)?; Ok(()) } + + fn read_into_string(&mut self, self_: Resource, name: String) -> String { + let resource = self + .resources + .get(&self_) + .expect("Failed to find own resource"); + let file_path = resource.readers.get(&name); + if let Some(path) = file_path.cloned() { + let file = fs::read_to_string(path); + file.unwrap_or(String::new()) + } else { + String::new() + } + } } diff --git a/src/graph/dynamic/csv_writer.rs b/src/graph/dynamic/csv_writer.rs index 113f40f..30fbe85 100644 --- a/src/graph/dynamic/csv_writer.rs +++ b/src/graph/dynamic/csv_writer.rs @@ -2,19 +2,22 @@ use super::dynamic_state::{vato007::ingey::types::HostCsvWriter, DynamicState}; use crate::io::RecordSerializer; use csv::Writer; use std::collections::BTreeMap; +use std::fs; use std::fs::File; pub struct CsvWriterData { writer: Writer, wrote_header: bool, + path: String, } impl CsvWriterData { pub fn new(path: String) -> anyhow::Result { - let writer = Writer::from_path(path)?; + let writer = Writer::from_path(path.clone())?; Ok(CsvWriterData { writer, wrote_header: false, + path, }) } } @@ -47,4 +50,15 @@ impl HostCsvWriter for DynamicState { self.resources.delete(rep)?; Ok(()) } + fn write_string( + &mut self, + self_: wasmtime::component::Resource, + contents: String, + ) { + let resource = self + .resources + .get_mut(&self_) + .expect("Failed to find resource"); + fs::write(&resource.path, contents); + } } diff --git a/src/graph/dynamic/mod.rs b/src/graph/dynamic/mod.rs index ea00e26..fbef5c9 100644 --- a/src/graph/dynamic/mod.rs +++ b/src/graph/dynamic/mod.rs @@ -23,9 +23,10 @@ pub struct DynamicNode { pub wasm_file_path: String, pub input_file_paths: HashMap, pub output_file: String, + pub properties: HashMap, } -// Node to run arbitrary webassembly code to transorm one or more csv files +// Node to run arbitrary webassembly code to transform one or more csv files // TODO: Create a separate node for wit that allows arbitrary files pub struct DynamicNodeRunner { pub dynamic_node: DynamicNode, @@ -41,7 +42,7 @@ impl RunnableNode for DynamicNodeRunner { let mut store = Store::new(&engine, DynamicState::new()); let bindings = Dynamic::instantiate(&mut store, &component, &linker)?; let read_map = store.data_mut().resources.push(ReadMapData { - data: HashMap::new(), + data: self.dynamic_node.properties.clone(), })?; let readers = store.data_mut().resources.push(CsvReadersData { readers: self.dynamic_node.input_file_paths.clone(), @@ -52,3 +53,42 @@ impl RunnableNode for DynamicNodeRunner { Ok(()) } } + +#[cfg(test)] +mod tests { + use crate::graph::dynamic::{DynamicNode, DynamicNodeRunner}; + use crate::graph::node::RunnableNode; + use std::collections::HashMap; + + #[tokio::test] + async fn move_money_wasm() -> anyhow::Result<()> { + let mut input_file_paths = HashMap::new(); + input_file_paths.insert("Account".to_owned(), "testing/input/account.csv".to_owned()); + input_file_paths.insert( + "CostCentre".to_owned(), + "testing/input/costcentre.csv".to_owned(), + ); + input_file_paths.insert( + "Line".to_owned(), + "testing/input/move_money/line.csv".to_owned(), + ); + input_file_paths.insert( + "Rule".to_owned(), + "testing/input/move_money/reclassrule.csv".to_owned(), + ); + let mut properties = HashMap::new(); + properties.insert("flush_pass".to_owned(), "false".to_owned()); + // properties.insert("flush_pass".to_owned(), "true".to_owned()); + properties.insert("use_numeric_accounts".to_owned(), "false".to_owned()); + let dynamic_node = DynamicNode { + input_file_paths, + output_file: "testing/output/output.csv".to_owned(), + wasm_file_path: "target/wasm32-unknown-unknown/release/move_money_dynamic.wasm" + .to_owned(), + properties, + }; + let runner = DynamicNodeRunner { dynamic_node }; + runner.run().await?; + Ok(()) + } +} diff --git a/src/lib.rs b/src/lib.rs index 0e0d6fd..c0da76a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,3 @@ -// TODO: Module api can probably use a cleanup -mod move_money; -pub use self::move_money::*; use std::ffi::c_char; use std::ffi::CStr; use std::ffi::CString; @@ -16,68 +13,6 @@ pub mod graph; mod io; pub mod link; -#[no_mangle] -pub extern "C" fn move_money_from_text( - rules: *const c_char, - lines: *const c_char, - accounts: *const c_char, - cost_centres: *const c_char, - use_numeric_accounts: bool, -) -> *mut c_char { - let mut output_writer = csv::Writer::from_writer(vec![]); - let safe_rules = unwrap_c_char(rules); - let safe_lines = unwrap_c_char(lines); - let safe_accounts = unwrap_c_char(accounts); - let safe_cost_centres = unwrap_c_char(cost_centres); - move_money( - &mut csv::Reader::from_reader(safe_rules.to_bytes()), - &mut csv::Reader::from_reader(safe_lines.to_bytes()), - &mut csv::Reader::from_reader(safe_accounts.to_bytes()), - &mut csv::Reader::from_reader(safe_cost_centres.to_bytes()), - &mut output_writer, - use_numeric_accounts, - false, - ) - .expect("Failed to move money"); - // TODO: Replace all these unwraps with something more elegant - let inner = output_writer.into_inner().unwrap(); - CString::new(String::from_utf8(inner).unwrap()) - .unwrap() - .into_raw() - - // Also some resources I looked at, in case things aren't going right: - // https://notes.huy.rocks/en/string-ffi-rust.html - // http://jakegoulding.com/rust-ffi-omnibus/string_return/ - // https://rust-unofficial.github.io/patterns/idioms/ffi/passing-strings.html - // This looks like exactly what I'm doing too: https://mozilla.github.io/firefox-browser-architecture/experiments/2017-09-06-rust-on-ios.htmlcar -} - -#[no_mangle] -pub extern "C" fn move_money_from_file( - rules_file: *const c_char, - lines: *const c_char, - accounts: *const c_char, - cost_centres: *const c_char, - output_path: *const c_char, - use_numeric_accounts: bool, -) { - let safe_rules = unwrap_c_char(rules_file); - let safe_lines = unwrap_c_char(lines); - let safe_accounts = unwrap_c_char(accounts); - let safe_cost_centres = unwrap_c_char(cost_centres); - let output_path = unwrap_c_char(output_path); - move_money( - &mut csv::Reader::from_reader(safe_rules.to_bytes()), - &mut csv::Reader::from_reader(safe_lines.to_bytes()), - &mut csv::Reader::from_reader(safe_accounts.to_bytes()), - &mut csv::Reader::from_reader(safe_cost_centres.to_bytes()), - &mut csv::Writer::from_path(output_path.to_str().unwrap()).unwrap(), - use_numeric_accounts, - false, - ) - .expect("Failed to move money"); -} - #[no_mangle] pub unsafe extern "C" fn move_money_from_text_free(s: *mut c_char) { unsafe { diff --git a/src/shared_models.rs b/src/shared_models.rs index d41d204..8ca9447 100644 --- a/src/shared_models.rs +++ b/src/shared_models.rs @@ -1,4 +1,4 @@ -use serde::Deserialize; +use serde::{Deserialize, Serialize, Serializer}; #[derive(Deserialize)] pub struct CsvAccount { @@ -13,3 +13,20 @@ pub struct CsvAccount { #[serde(rename = "PercentFixed")] pub percent_fixed: f64, } + +#[derive(Debug, Serialize, Deserialize)] +pub struct CsvCost { + #[serde(rename = "ACCOUNT")] + pub account: String, + #[serde(rename = "COSTCENTRE")] + pub department: String, + #[serde(serialize_with = "round_serialize")] + pub value: f64, +} + +fn round_serialize(x: &f64, s: S) -> Result +where + S: Serializer, +{ + s.serialize_f64((x * 100000.).round() / 100000.) +} \ No newline at end of file diff --git a/wit/dynamic_node.wit b/wit/dynamic_node.wit index 95510d4..e45cb91 100644 --- a/wit/dynamic_node.wit +++ b/wit/dynamic_node.wit @@ -23,10 +23,12 @@ interface types { resource csv-readers { get-reader: func(name: string) -> option; + read-into-string: func(name: string) -> string; } resource csv-writer { write-row: func(row: list>); + write-string: func(row: string); } resource read-map {