vato007/ingey
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove unneeded dependencies, start work on csv parsing

Browse Source
This commit is contained in:
Piv
2023-01-28 20:33:04 +10:30
parent 10f24f65ac
commit a2091c13b4
4 changed files with 79 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
Cargo.lock generated
View File

@@ -23,7 +23,7 @@ version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [
"hermit-abi 0.1.19",
"hermit-abi",
"libc",
"winapi",
]
@@ -58,12 +58,6 @@ version = "1.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdead85bdec19c194affaeeb670c0e41fe23de31459efd1c174d049269cf02cc"
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "clap"
version = "3.2.23"
@@ -112,54 +106,9 @@ dependencies = [
"csv",
"itertools",
"nalgebra",
"num-rational",
"rayon",
"serde",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
dependencies = [
"cfg-if",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc"
dependencies = [
"cfg-if",
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f"
dependencies = [
"cfg-if",
]
[[package]]
name = "csv"
version = "1.1.6"
@@ -209,15 +158,6 @@ dependencies = [
"libc",
]
[[package]]
name = "hermit-abi"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
dependencies = [
"libc",
]
[[package]]
name = "indexmap"
version = "1.9.2"
@@ -270,15 +210,6 @@ version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
[[package]]
name = "memoffset"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
dependencies = [
"autocfg",
]
[[package]]
name = "nalgebra"
version = "0.31.0"
@@ -306,17 +237,6 @@ dependencies = [
"syn",
]
[[package]]
name = "num-bigint"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f"
dependencies = [
"autocfg",
"num-integer",
"num-traits",
]
[[package]]
name = "num-complex"
version = "0.4.1"
@@ -343,7 +263,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0"
dependencies = [
"autocfg",
"num-bigint",
"num-integer",
"num-traits",
]
@@ -357,16 +276,6 @@ dependencies = [
"autocfg",
]
[[package]]
name = "num_cpus"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
dependencies = [
"hermit-abi 0.2.6",
"libc",
]
[[package]]
name = "once_cell"
version = "1.12.0"
@@ -433,28 +342,6 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
[[package]]
name = "rayon"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-utils",
"num_cpus",
]
[[package]]
name = "regex-automata"
version = "0.1.10"
@@ -476,12 +363,6 @@ dependencies = [
"bytemuck",
]
[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "serde"
version = "1.0.137"

3
Cargo.toml
View File

@@ -12,12 +12,9 @@ nalgebra = "0.31.0"
# https://docs.rs/csv/1.1.6/csv/
csv = "1.1"
serde = { version = "1", features = ["derive"] }
# simba = { version = "0.7.1", features = ["partial_fixed_point_support"] }
# num = "0.4"
clap = { version = "3.1.18", features = ["derive"] }
anyhow = "1.0"
itertools = "0.10.3"
num-rational = "0.4.1"
rayon = "1.6.1"

7
src/lib.rs
View File

@@ -88,9 +88,8 @@ pub fn move_money_2(
initial_totals: HashMap<Unit, f64>,
rules: &Vec<MovementRule>,
) -> HashMap<Unit, f64> {
// TODO: Should probably validate that all the rules have departments that actually exist in initial_totals.
// Note: It's potentially a bit more intensive to use cloned totals, but it's much simpler code and, and since we're only working line-by-line
// it isn't really that much memory. in practice
// Note: It's potentially a bit more intensive to use cloned totals (rather than just update temp_total per rule),
// but it's much simpler code and, and since we're only working line-by-line, it isn't really that much memory in practice
let mut running_total = HashMap::from(initial_totals);
let mut temp_total = running_total.clone();
for rule in rules {
@@ -135,11 +134,9 @@ pub enum DepartmentType {
Overhead,
}
// TODO: Could also look at BigDecimal rather than f64 for higher precision (even i64 might be fine if we don't need to divide...)
// Note: remember these are overhead departments only when calculating the lu decomposition or pseudoinverse, and for each department,
// you either need -1 or rest negative for a row to subtract the initial amounts so we end up effectively 0 (simultaneous equations end
// up with negative there so yes this is expected)
// Also, we could potentially use this same struct for non-overhead departments when mapping from overhead to
pub struct OverheadAllocationRule {
from_overhead_department: String,
to_department: String,

89
src/main.rs
View File

@@ -1,6 +1,7 @@
use std::{error::Error, io::Write, path::PathBuf};
use std::{collections::HashMap, error::Error, io::Write, path::PathBuf};
use clap::{builder::PathBufValueParser, Parser, Subcommand};
use coster_rs::Unit;
use serde::Deserialize;
#[derive(Parser)]
@@ -47,7 +48,6 @@ enum Commands {
},
}
// TODO: Return error (implement the required trait to allow an error to be returned)
fn main() -> anyhow::Result<()> {
let cli = Cli::parse();
@@ -57,23 +57,67 @@ fn main() -> anyhow::Result<()> {
lines,
output,
} => move_money(rules, lines, output),
Commands::smush_rules { rules, output } => smush_rules(rules, output)?,
Commands::smush_rules { rules, output } => smush_rules(rules, output),
Commands::allocate_overheads {
rules,
lines,
output,
} => allocate_overheads(rules, lines, output)?,
};
Ok(())
} => allocate_overheads(rules, lines, output),
}
}
fn move_money(rules: PathBuf, lines: PathBuf, output: Option<PathBuf>) {
// read rules into required struct (basically map each line into an array)
fn move_money(rules: PathBuf, lines: PathBuf, output: Option<PathBuf>) -> anyhow::Result<()> {
let mut rdr = csv::Reader::from_path(lines)?;
let headers = rdr.headers()?;
let mut account_index = 0;
let mut department_index = 0;
for (index, field) in headers.iter().enumerate() {
if field == "Account" {
account_index = index;
} else if field == "Department" {
department_index = index;
}
}
let lines: HashMap<Unit, f64> = rdr
.records()
.map(|record| {
let record = record.unwrap();
let account = record.get(account_index).unwrap();
let department = record.get(department_index).unwrap();
let sum = record
.iter()
.enumerate()
.filter(|(i, _)| *i != account_index && *i != department_index)
.map(|(_, f)| f.parse::<f64>().unwrap())
.sum();
(
Unit {
account: account.into(),
department: department.into(),
},
sum,
)
})
.collect();
// read rules into required struct (basically map each line into an array). Need to figure out the input format...
let mut rdr = csv::Reader::from_path(rules)?;
for result in rdr.deserialize() {
let record: CsvMovementRule = result?;
// TODO: Problem is the from/to are ranges, that rely on the gl data, so need to get a list of
//
}
// Read gl lines data (all of it). For each line, sum the periods, and insert the summed
// line into a hashmap (need to read the whole gl in as we can move between every line)
// We use a record, everything that's not an account/department is assumed to be a period
// Then run move_moeny, and output the result into a new file at the given output location
// Then run move_moeny
// Ouput the list moved moneys
Ok(())
}
fn smush_rules(rules_path: PathBuf, output: Option<PathBuf>) -> anyhow::Result<()> {
@@ -86,14 +130,15 @@ fn allocate_overheads(
output: Option<PathBuf>,
) -> anyhow::Result<()> {
let mut rdr = csv::Reader::from_path(rules_path)?;
for result in rdr.deserialize() {
let record: CsvMovementRule = result?;
let record: CsvOverheadAllocationRule = result?;
}
let mut account_reader = csv::Reader::from_path(lines)?;
for result in account_reader.deserialize() {
let record: CsvAccountCost = result?;
let record: CsvCost = result?;
}
Ok(())
@@ -101,15 +146,31 @@ fn allocate_overheads(
#[derive(Debug, Deserialize)]
struct CsvMovementRule {
from_department: String,
to_department: String,
#[serde(rename = "FromCC")]
// Need strings to further split later
from_departments: String,
to_departments: String,
all_from_departments: bool,
all_to_departments: bool,
from_accounts: String,
to_accounts: String,
all_from_accounts: bool,
all_to_accounts: bool,
amount: f64,
is_percent: Option<bool>,
is_separator: Option<bool>,
}
#[derive(Debug, Deserialize)]
struct CsvAccountCost {
struct CsvOverheadAllocationRule {
from_overhead_department: String,
to_department: String,
percent: f64,
to_department_type: String,
}
#[derive(Debug, Deserialize)]
struct CsvCost {
account: String,
department: String,
value: f64,