From d88fbe707a435eaaf1eb8f8d28531cffe13cfa8c Mon Sep 17 00:00:00 2001
From: Piv <18462828+Piv200@users.noreply.github.com>
Date: Sat, 11 Mar 2023 22:21:07 +1030
Subject: [PATCH] Use rayon to speed up percentage calculations, add readme

---
 README.md                  | 11 +++++++++++
 src/overhead_allocation.rs | 24 +++++++++++++++++-------
 2 files changed, 28 insertions(+), 7 deletions(-)
 create mode 100644 README.md

diff --git a/README.md b/README.md
new file mode 100644
index 0000000..ae7db17
--- /dev/null
+++ b/README.md
@@ -0,0 +1,11 @@
+# Fast Coster
+
+A business costing system that is super fast:
+
+- Money movements in less than a second
+- Overhead allocation in ~2s for large datasets (~10000 departments) and not flushing every from record - up to 10s when flushing from/to allocations
+
+## General Notes
+
+Setting the number of threads in overhead allocation: set RAYON_NUM_THREADS environment variable ([doc](https://github.com/rayon-rs/rayon/blob/master/FAQ.md)).
+Note multithreading is only currently used to calculate allocation percentages, it is not used in the main allocation processing.
diff --git a/src/overhead_allocation.rs b/src/overhead_allocation.rs
index cc5cea1..b870b7e 100644
--- a/src/overhead_allocation.rs
+++ b/src/overhead_allocation.rs
@@ -6,6 +6,10 @@ use std::{
 use csv::Writer;
 use itertools::Itertools;
 use nalgebra::{DMatrix, Dynamic, LU};
+use rayon::prelude::{
+    IntoParallelIterator, IntoParallelRefIterator, ParallelDrainFull, ParallelDrainRange,
+    ParallelIterator,
+};
 use serde::{Deserialize, Serialize};
 
 use crate::{CsvAccount, CsvCost};
@@ -232,8 +236,6 @@ where
 
     // Save overhead ccs, so we later know whether a to cc is overhead or operating
     let mut overhead_ccs: HashSet<String> = HashSet::new();
-    // overhead department -> total (summed limit to costs)
-    let mut overhead_cc_totals: HashMap<String, f64> = HashMap::new();
     // For each overhead area, get the cost centres in the area (overhead cost centres), and get all cost centres
     // that fit the limit to criteria for the area (skip any cases of overhead cc = other cc).
     // Then get the totals for the other ccs, by looking in the flat_department_costs, where the
@@ -286,7 +288,7 @@ where
                 limited_ccs.append(&mut other_ccs);
             }
             let mut totals: Vec<(String, String, f64)> = overhead_ccs
-                .iter()
+                .par_iter()
                 .flat_map(|overhead_cc| {
                     let limited = limited_ccs
                         .iter()
@@ -310,11 +312,12 @@ where
                         .filter(|(_, _, value)| *value != 0.)
                         .filter(|(from_cc, to_cc, _)| from_cc != to_cc)
                         .collect_vec();
+                    // TODO: Put me back if rayon proves problematic
                     // Insert is safe, since an overhead cc can only be a part of one area
-                    overhead_cc_totals.insert(
-                        overhead_cc.clone(),
-                        limited.iter().map(|(_, _, value)| value).sum(),
-                    );
+                    // overhead_cc_totals.insert(
+                    //     overhead_cc.clone(),
+                    //     limited.iter().map(|(_, _, value)| value).sum(),
+                    // );
                     limited
                 })
                 .collect();
@@ -322,6 +325,13 @@ where
         }
     }
 
+    // overhead department -> total (summed limit to costs)
+    let mut overhead_cc_totals: HashMap<String, f64> = HashMap::new();
+    // Using rayon and doing another pass later proves to be
+    for (overhead_cc, _, value) in overhead_other_total.iter() {
+        *overhead_cc_totals.entry(overhead_cc.clone()).or_insert(0.) += value;
+    }
+
     // Export initial totals for operating departments
     if show_from {
         for line in lines.iter() {