vato007/ingey
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow showing from amounts in overhead allocation

Browse Source
This commit is contained in:
Piv
2023-03-11 20:19:28 +10:30
parent 7cd893cbf8
commit f5bc441fdb
9 changed files with 246 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.toml
View File

@@ -24,5 +24,6 @@ rayon = "1.6.0"
tokio = { version = "1.26.0", features = ["full"] }
sqlx = { version = "0.6", features = [ "runtime-tokio-rustls", "mssql" ] }
# More info on targets: https://doc.rust-lang.org/cargo/reference/cargo-targets.html#configuring-a-target
[lib]
crate-type = ["cdylib", "staticlib", "lib"]

4
FastCoster/FastCoster.xcodeproj/project.pbxproj
View File

@@ -12,6 +12,7 @@
5A1986FB2996502C00FA0471 /* FileButtonSelector.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5A1986FA2996502C00FA0471 /* FileButtonSelector.swift */; };
5A450751298CE6D500E3D402 /* CsvDocument.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5A450750298CE6D500E3D402 /* CsvDocument.swift */; };
5A45075B298D01EF00E3D402 /* libcoster_rs.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 5A45075A298D01EF00E3D402 /* libcoster_rs.a */; };
5A4995C829BC423900A1A107 /* TempFileDocument.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5A4995C729BC423900A1A107 /* TempFileDocument.swift */; };
5ADD9F2D298A713300F998F5 /* FastCosterApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5ADD9F2C298A713300F998F5 /* FastCosterApp.swift */; };
5ADD9F2F298A713300F998F5 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5ADD9F2E298A713300F998F5 /* ContentView.swift */; };
5ADD9F31298A713400F998F5 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 5ADD9F30298A713400F998F5 /* Assets.xcassets */; };
@@ -48,6 +49,7 @@
5A450755298CFFE400E3D402 /* create-lib.sh */ = {isa = PBXFileReference; lastKnownFileType = text.script.sh; path = "create-lib.sh"; sourceTree = "<group>"; };
5A450756298D00AE00E3D402 /* remove-lib.sh */ = {isa = PBXFileReference; lastKnownFileType = text.script.sh; path = "remove-lib.sh"; sourceTree = "<group>"; };
5A45075A298D01EF00E3D402 /* libcoster_rs.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libcoster_rs.a; path = "../costerrs/target/aarch64-apple-ios/release/libcoster_rs.a"; sourceTree = "<group>"; };
5A4995C729BC423900A1A107 /* TempFileDocument.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TempFileDocument.swift; sourceTree = "<group>"; };
5ADD9F29298A713300F998F5 /* FastCoster.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = FastCoster.app; sourceTree = BUILT_PRODUCTS_DIR; };
5ADD9F2C298A713300F998F5 /* FastCosterApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FastCosterApp.swift; sourceTree = "<group>"; };
5ADD9F2E298A713300F998F5 /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = "<group>"; };
@@ -140,6 +142,7 @@
5A1986F62996436500FA0471 /* OverheadAllocation.swift */,
5A1986F82996436D00FA0471 /* MoveMoney.swift */,
5A1986FA2996502C00FA0471 /* FileButtonSelector.swift */,
5A4995C729BC423900A1A107 /* TempFileDocument.swift */,
);
path = FastCoster;
sourceTree = "<group>";
@@ -348,6 +351,7 @@
5A1986F92996436D00FA0471 /* MoveMoney.swift in Sources */,
5ADD9F2D298A713300F998F5 /* FastCosterApp.swift in Sources */,
5A450751298CE6D500E3D402 /* CsvDocument.swift in Sources */,
5A4995C829BC423900A1A107 /* TempFileDocument.swift in Sources */,
5A1986F72996436500FA0471 /* OverheadAllocation.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;

25
FastCoster/FastCoster/OverheadAllocation.swift
View File

@@ -14,8 +14,11 @@ struct OverheadAllocation: View {
@State private var allocationStatistics: String?
@State private var costCentres: String?
@State private var showExportPicker = false
@State private var document: CsvDocument?
@State private var document: TempFileDocument?
@State private var accountType = "E"
@State private var show_from = false
private let tempPath = FileManager.default.temporaryDirectory.appendingPathComponent("OverheadAllocation.csv", conformingTo: .commaSeparatedText)
var body: some View {
VStack {
@@ -35,6 +38,9 @@ struct OverheadAllocation: View {
costCentres = result
}
TextField("Account Type", text: $accountType)
Toggle(isOn: $show_from) {
Text("Show from cc movements")
}
Button {
allocate_overheads()
} label: {
@@ -43,23 +49,28 @@ struct OverheadAllocation: View {
.fileExporter(isPresented: $showExportPicker, document: document, contentType: .commaSeparatedText) { result in
if case .success = result {
// TODO: Delete the temp file
do {
try FileManager.default.removeItem(at: tempPath)
} catch {
}
}else {
}
}
}
}.padding()
.textFieldStyle(.roundedBorder)
}
func allocate_overheads() {
DispatchQueue.global(qos: .userInitiated).async {
// Run move money
let result = allocate_overheads_from_text(lines, accounts, allocationStatistics, areas, costCentres, accountType, false);
let finalDocument = TempFileDocument(url: tempPath)
allocate_overheads_from_text_to_file(lines, accounts, allocationStatistics, areas, costCentres, accountType, tempPath.absoluteString, false, show_from)
DispatchQueue.main.async {
document = CsvDocument(data: String(cString: result!))
allocate_overheads_from_text_free(result)
document = finalDocument
showExportPicker = true;
}
}

32
FastCoster/FastCoster/TempFileDocument.swift Normal file
View File

@@ -0,0 +1,32 @@
//
// TempFileDocument.swift
// FastCoster
//
// Created by Michael Pivato on 11/3/2023.
//
import Foundation
import SwiftUI
import UniformTypeIdentifiers
struct TempFileDocument: FileDocument {
var tempFileUrl: URL?
var wrapper: FileWrapper?
static var readableContentTypes: [UTType] {[.commaSeparatedText]}
init(configuration: ReadConfiguration) throws {
wrapper = configuration.file
}
init(url: URL) {
tempFileUrl = url
}
func fileWrapper(configuration: WriteConfiguration) throws -> FileWrapper {
return try wrapper ?? FileWrapper(url: tempFileUrl!)
}
}

1
generate_c_header.sh Normal file
View File

@@ -0,0 +1 @@
cbindgen --config cbindgen.toml --crate coster-rs --output coster.h --lang c

72
src/lib.rs
View File

@@ -108,16 +108,21 @@ pub extern "C" fn allocate_overheads_from_text(
};
let mut output_writer = csv::Writer::from_writer(vec![]);
reciprocal_allocation(
csv::Reader::from_reader(lines.to_bytes()),
csv::Reader::from_reader(accounts.to_bytes()),
csv::Reader::from_reader(allocation_statistics.to_bytes()),
csv::Reader::from_reader(areas.to_bytes()),
csv::Reader::from_reader(cost_centres.to_bytes()),
&mut csv::Reader::from_reader(lines.to_bytes()),
&mut csv::Reader::from_reader(accounts.to_bytes()),
&mut csv::Reader::from_reader(allocation_statistics.to_bytes()),
&mut csv::Reader::from_reader(areas.to_bytes()),
&mut csv::Reader::from_reader(cost_centres.to_bytes()),
&mut output_writer,
use_numeric_accounts,
false,
true,
account_type.to_str().unwrap().to_owned(),
// This needs to be false when we return just a string, as the
// amount of data produced could easily go out of memory (9000ccs + 1000
// accounts can reach 2gb)
false,
0.1,
)
.expect("Failed to allocate overheads");
let inner = output_writer.into_inner().unwrap();
@@ -126,6 +131,63 @@ pub extern "C" fn allocate_overheads_from_text(
.into_raw()
}
#[no_mangle]
pub extern "C" fn allocate_overheads_from_text_to_file(
lines: *const c_char,
accounts: *const c_char,
allocation_statistics: *const c_char,
areas: *const c_char,
cost_centres: *const c_char,
account_type: *const c_char,
output_path: *const c_char,
use_numeric_accounts: bool,
show_from: bool,
) {
let lines = unsafe {
assert!(!lines.is_null());
CStr::from_ptr(lines)
};
let accounts = unsafe {
assert!(!accounts.is_null());
CStr::from_ptr(accounts)
};
let allocation_statistics = unsafe {
assert!(!allocation_statistics.is_null());
CStr::from_ptr(allocation_statistics)
};
let areas = unsafe {
assert!(!areas.is_null());
CStr::from_ptr(areas)
};
let cost_centres = unsafe {
assert!(!cost_centres.is_null());
CStr::from_ptr(cost_centres)
};
let account_type = unsafe {
assert!(!account_type.is_null());
CStr::from_ptr(account_type)
};
let output_path = unsafe {
assert!(!output_path.is_null());
CStr::from_ptr(output_path)
};
reciprocal_allocation(
&mut csv::Reader::from_reader(lines.to_bytes()),
&mut csv::Reader::from_reader(accounts.to_bytes()),
&mut csv::Reader::from_reader(allocation_statistics.to_bytes()),
&mut csv::Reader::from_reader(areas.to_bytes()),
&mut csv::Reader::from_reader(cost_centres.to_bytes()),
&mut csv::Writer::from_path(output_path.to_str().unwrap()).unwrap(),
use_numeric_accounts,
false,
true,
account_type.to_str().unwrap().to_owned(),
show_from,
0.1,
)
.expect("Failed to allocate overheads");
}
#[no_mangle]
pub extern "C" fn allocate_overheads_from_text_free(s: *mut c_char) {
unsafe {

20
src/main.rs
View File

@@ -63,6 +63,12 @@ enum Commands {
#[arg(short, long)]
exclude_negative_allocation_statistics: bool,
#[arg(short = 'f', long)]
show_from: bool,
#[arg(short, long, default_value = "0.1")]
zero_threshold: f64,
#[arg(short, long, value_name = "FILE")]
output: Option<PathBuf>,
},
@@ -120,18 +126,22 @@ fn main() -> anyhow::Result<()> {
use_numeric_accounts,
account_type,
exclude_negative_allocation_statistics,
show_from,
zero_threshold,
output,
} => coster_rs::reciprocal_allocation(
csv::Reader::from_path(lines)?,
csv::Reader::from_path(accounts)?,
csv::Reader::from_path(allocation_statistics)?,
csv::Reader::from_path(areas)?,
csv::Reader::from_path(cost_centres)?,
&mut csv::Reader::from_path(lines)?,
&mut csv::Reader::from_path(accounts)?,
&mut csv::Reader::from_path(allocation_statistics)?,
&mut csv::Reader::from_path(areas)?,
&mut csv::Reader::from_path(cost_centres)?,
&mut csv::Writer::from_path(output.unwrap_or(PathBuf::from("alloc_output.csv")))?,
use_numeric_accounts,
exclude_negative_allocation_statistics,
true,
account_type,
show_from,
zero_threshold,
),
Commands::CreateProducts {
definitions,

149
src/overhead_allocation.rs
View File

@@ -1,11 +1,12 @@
use std::{
collections::{HashMap, HashSet},
io::Read,
io::{Read, Write},
};
use csv::Writer;
use itertools::Itertools;
use nalgebra::{DMatrix, Dynamic, LU};
use serde::Deserialize;
use serde::{Deserialize, Serialize};
use crate::{CsvAccount, CsvCost};
@@ -62,6 +63,14 @@ pub struct AccountCost {
summed_department_costs: Vec<TotalDepartmentCost>,
}
#[derive(Debug, Serialize)]
struct MovedAmount {
account: String,
cost_centre: String,
value: f64,
from_cost_centre: String,
}
// TODO: Also need a way to dictate the order of the departments?
pub trait ReciprocalAllocationSolver {
fn solve(&self, costs: &DMatrix<f64>) -> DMatrix<f64>;
@@ -80,16 +89,18 @@ impl ReciprocalAllocationSolver for DMatrix<f64> {
}
pub fn reciprocal_allocation<Lines, Account, AllocationStatistic, Area, CostCentre, Output>(
lines: csv::Reader<Lines>,
accounts: csv::Reader<Account>,
allocation_statistics: csv::Reader<AllocationStatistic>,
areas: csv::Reader<Area>,
cost_centres: csv::Reader<CostCentre>,
lines: &mut csv::Reader<Lines>,
accounts: &mut csv::Reader<Account>,
allocation_statistics: &mut csv::Reader<AllocationStatistic>,
areas: &mut csv::Reader<Area>,
cost_centres: &mut csv::Reader<CostCentre>,
output: &mut csv::Writer<Output>,
use_numeric_accounts: bool,
exclude_negative_allocation_statistics: bool,
any_limit_criteria: bool,
account_type: String,
show_from: bool,
zero_threshold: f64,
) -> anyhow::Result<()>
where
Lines: Read,
@@ -99,13 +110,10 @@ where
CostCentre: Read,
Output: std::io::Write,
{
let mut lines_reader = lines;
let lines = lines_reader
let lines = lines
.deserialize()
.collect::<Result<Vec<CsvCost>, csv::Error>>()?;
let mut accounts = accounts;
let all_accounts_sorted: Vec<String> = if use_numeric_accounts {
accounts
.deserialize::<CsvAccount>()
@@ -129,8 +137,7 @@ where
.collect()
};
let mut allocation_statistics_reader = allocation_statistics;
let allocation_statistics = allocation_statistics_reader
let allocation_statistics = allocation_statistics
.deserialize::<CsvAllocationStatistic>()
.filter(|allocation_statistic| {
allocation_statistic.as_ref().unwrap().account_type == account_type
@@ -186,7 +193,6 @@ where
// Group ccs by area
let mut area_ccs: HashMap<String, Vec<String>> = HashMap::new();
let mut cost_centres = cost_centres;
let headers = cost_centres.headers()?;
// Group ccs by rollup, and group rollups into their slot
let mut rollups: HashMap<String, HashMap<String, Vec<String>>> = headers
@@ -216,7 +222,6 @@ where
}
}
let mut areas = areas;
let headers = areas.headers()?;
let limit_tos: Vec<String> = headers
.iter()
@@ -317,6 +322,24 @@ where
}
}
// Export initial totals for operating departments
if show_from {
for line in lines.iter() {
if !overhead_ccs.contains(&line.department) {
output.serialize(MovedAmount {
account: line.account.clone(),
cost_centre: line.department.clone(),
value: line.value,
from_cost_centre: line.department.clone(),
})?;
}
}
}
//TODO: Redistribute floating point errors so everything still sums to 1. Also reduces the amount of data produced
// Basically needs to take each mapping, and somehow sum the percentages less than the threshold, and evenly(?) redistribute
// them to the others within the same department... Wonder if this could have been done in the previous step somehow?
// Finally, for each cc match total produced previously, sum the overhead cc where overhead cc appears in other cc, then
// divide the other cc by this summed amount (thus getting the relative cost)
@@ -371,17 +394,21 @@ where
summed_department_costs: total_cost,
})
.collect(),
if show_from { Some(output) } else { None },
zero_threshold,
)?;
for cost in results {
for department in cost.summed_department_costs {
// Any consumers should assume missing cc/account value was 0 (we already ignore overhead, as they all 0 out)
if department.value > 0.00001 || department.value < -0.00001 {
output.serialize(CsvCost {
account: cost.account.clone(),
department: department.department,
value: department.value,
})?;
if !show_from {
for cost in results {
for department in cost.summed_department_costs {
// Any consumers should assume missing cc/account value was 0 (we already ignore overhead, as they all 0 out)
if department.value > 0.00001 || department.value < -0.00001 {
output.serialize(CsvCost {
account: cost.account.clone(),
department: department.department,
value: department.value,
})?;
}
}
}
}
@@ -435,10 +462,11 @@ fn remove_quote_and_padding(s: &str) -> String {
// Perform the reciprocal allocation (matrix) method to allocate servicing departments (indirect) costs
// to functional departments. Basically just a matrix solve, uses regression (moore-penrose pseudoinverse) when
// matrix is singular
fn reciprocal_allocation_impl(
fn reciprocal_allocation_impl<W: Write>(
allocations: Vec<OverheadAllocationRule>,
account_costs: Vec<AccountCost>,
// TODO: Throw an appropriate error
movement_writer: Option<&mut csv::Writer<W>>,
zero_threshold: f64,
) -> anyhow::Result<Vec<AccountCost>> {
let overhead_department_mappings = get_rules_indexes(&allocations, DepartmentType::Overhead);
@@ -473,6 +501,8 @@ fn reciprocal_allocation_impl(
account_costs,
overhead_department_mappings,
allocations,
movement_writer,
zero_threshold,
)
} else {
do_solve_reciprocal(
@@ -480,6 +510,8 @@ fn reciprocal_allocation_impl(
account_costs,
overhead_department_mappings,
allocations,
movement_writer,
zero_threshold,
)
}
}
@@ -512,6 +544,8 @@ fn do_solve_reciprocal<T: ReciprocalAllocationSolver>(
account_costs: Vec<AccountCost>,
overhead_department_mappings: HashMap<String, usize>,
allocations: Vec<OverheadAllocationRule>,
temp_writer: Option<&mut Writer<impl Write>>,
zero_threshold: f64,
) -> anyhow::Result<Vec<AccountCost>> {
let operating_department_mappings = get_rules_indexes(&allocations, DepartmentType::Operating);
let mut operating_overhead_mappings =
@@ -534,6 +568,7 @@ fn do_solve_reciprocal<T: ReciprocalAllocationSolver>(
operating_overhead_mappings,
);
let mut final_account_costs: Vec<AccountCost> = Vec::with_capacity(account_costs.len());
let mut temp_writer = temp_writer;
for total_costs in account_costs {
// TODO: There has to be a cleaner way to do this, perhaps by presorting things?
let mut overhead_slice_costs = vec![0.; overhead_department_mappings.len()];
@@ -565,6 +600,32 @@ fn do_solve_reciprocal<T: ReciprocalAllocationSolver>(
let operating_overhead_mappings = &operating_overhead_mappings_mat;
let calculated_overheads = &calculated_overheads;
// To get the from/to ccs like ppm does, we ignore the initial totals. Then for each overhead cc,
// we zero out all the calculated overheads except for this cc and do
// operating_overhead_mappings * calculated_overheads (basically the first part of the normal calculation)
if let Some(temp_writer) = temp_writer.as_mut() {
// TODO: A performance improvement will be to create another hashmap for index -> department, then just
// iterate over the actual indexes instead (will have preloading)
for (overhead_department, index) in overhead_department_mappings.iter() {
// Calculate each movement individually
let calculated =
operating_overhead_mappings.column(*index) * calculated_overheads[*index];
for (department, index) in &operating_department_mappings {
let value = *calculated.get(*index).unwrap();
if value > zero_threshold || value < -1. * zero_threshold {
temp_writer.serialize(MovedAmount {
account: total_costs.account.clone(),
cost_centre: department.clone(),
value,
from_cost_centre: overhead_department.clone(),
})?;
}
}
}
// Don't bother performing the second calculation, it's redundant
continue;
}
// Calculation: operating_overhead_usage . calculated_overheads + initial_totals
// Where operating_overhead_usage is the direct mapping from overhead -> operating department, calculated overheads is the
// solved overheads usages after taking into account usage between departments, and initial_totals is the initial values
@@ -579,9 +640,7 @@ fn do_solve_reciprocal<T: ReciprocalAllocationSolver>(
})
.collect();
// Redistribute floating point errors (only for ccs we actually allocated from/to)
// TODO: Still not sure if this is 100% correct, however it appears with this we match up
// with the line totals. I think this is because ppm just evenly redistributes floating point
// errors, whereas we keep the amounts proportional to the intial amounts
// TODO: Consider removing this once we're doing this above.
let initial_cost: f64 = total_costs
.summed_department_costs
.iter()
@@ -609,6 +668,8 @@ fn do_solve_reciprocal<T: ReciprocalAllocationSolver>(
#[cfg(test)]
mod tests {
use std::fs::File;
use crate::reciprocal_allocation;
use crate::AccountCost;
use crate::DepartmentType;
@@ -692,23 +753,27 @@ mod tests {
],
}];
let result = reciprocal_allocation_impl(allocation_rules, initial_totals).unwrap();
let result =
reciprocal_allocation_impl::<File>(allocation_rules, initial_totals, None, 0.001)
.unwrap();
assert_eq!(expected_final_allocations, result);
}
#[test]
fn test_basic_real() {
let result = reciprocal_allocation(
csv::Reader::from_path("test_line.csv").unwrap(),
csv::Reader::from_path("test_account.csv").unwrap(),
csv::Reader::from_path("test_alloc_stat.csv").unwrap(),
csv::Reader::from_path("test_area.csv").unwrap(),
csv::Reader::from_path("test_costcentre.csv").unwrap(),
&mut csv::Reader::from_path("test_line.csv").unwrap(),
&mut csv::Reader::from_path("test_account.csv").unwrap(),
&mut csv::Reader::from_path("test_alloc_stat.csv").unwrap(),
&mut csv::Reader::from_path("test_area.csv").unwrap(),
&mut csv::Reader::from_path("test_costcentre.csv").unwrap(),
&mut csv::Writer::from_path("test_output_alloc_stat.csv").unwrap(),
true,
false,
true,
"E".to_owned(),
false,
0.1,
);
assert!(result.is_ok());
}
@@ -716,16 +781,18 @@ mod tests {
#[test]
fn test_real() {
let result = reciprocal_allocation(
csv::Reader::from_path("output.csv").unwrap(),
csv::Reader::from_path("account.csv").unwrap(),
csv::Reader::from_path("allocstat.csv").unwrap(),
csv::Reader::from_path("area.csv").unwrap(),
csv::Reader::from_path("costcentre.csv").unwrap(),
&mut csv::Reader::from_path("output.csv").unwrap(),
&mut csv::Reader::from_path("account.csv").unwrap(),
&mut csv::Reader::from_path("allocstat.csv").unwrap(),
&mut csv::Reader::from_path("area.csv").unwrap(),
&mut csv::Reader::from_path("costcentre.csv").unwrap(),
&mut csv::Writer::from_path("output_alloc_stat.csv").unwrap(),
true,
false,
false,
true,
"E".to_owned(),
true,
0.1,
);
assert!(result.is_ok())
}

4
src/products/csv.rs
View File

@@ -1,9 +1,5 @@
use std::{collections::HashMap, io::Read};
use chrono::NaiveDateTime;
use csv::Position;
use serde::Serialize;
#[derive(Hash, PartialEq, PartialOrd, Ord, Eq)]
pub struct Filter {
// Equal/not equal