diff --git a/Cargo.lock b/Cargo.lock index c889bfe..3325b46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,6 +39,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "allocator-api2" version = "0.2.18" @@ -151,6 +166,28 @@ dependencies = [ "url", ] +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + [[package]] name = "async-trait" version = "0.1.81" @@ -253,6 +290,27 @@ dependencies = [ "generic-array", ] +[[package]] +name = "brotli" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -287,9 +345,18 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" + +[[package]] +name = "castaway" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0abae9be0aaf9ea96a3b1b8b1b55c602ca751eba1b1500220cea4ecbafe7c0d5" +dependencies = [ + "rustversion", +] [[package]] name = "cc" @@ -402,6 +469,21 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "compact_str" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6050c3a16ddab2e412160b31f2c871015704239bca62f72f6e5f0be631d3f644" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -493,6 +575,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-channel" version = "0.5.13" @@ -759,6 +850,16 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +[[package]] +name = "flate2" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "flume" version = "0.11.0" @@ -785,12 +886,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" -[[package]] -name = "foreign_vec" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee1b05cbd864bcaecbd3455d6d967862d446e4ebfc3c2e5e5b9841e53cba6673" - [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1029,7 +1124,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core", + "windows-core 0.52.0", ] [[package]] @@ -1059,6 +1154,7 @@ checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown", + "serde", ] [[package]] @@ -1069,9 +1165,9 @@ checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itertools" -version = "0.10.5" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" dependencies = [ "either", ] @@ -1522,6 +1618,10 @@ name = "parquet-format-safe" version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1131c54b167dd4e4799ce762e1ab01549ebb94d5bdd13e6ec1b467491c378e1f" +dependencies = [ + "async-trait", + "futures", +] [[package]] name = "parse-zoneinfo" @@ -1641,9 +1741,9 @@ dependencies = [ [[package]] name = "polars" -version = "0.41.3" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3351ea4570e54cd556e6755b78fe7a2c85368d820c0307cca73c96e796a7ba" +checksum = "0e248cf2f0069277f8fe80d413cfb9240c7dd1cfa382b5674c1b4afa57222747" dependencies = [ "getrandom 0.2.15", "polars-arrow", @@ -1661,9 +1761,9 @@ dependencies = [ [[package]] name = "polars-arrow" -version = "0.41.3" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba65fc4bcabbd64fca01fd30e759f8b2043f0963c57619e331d4b534576c0b47" +checksum = "2981d5b2f34c84069a39fceca0d36dffeb97db8cadba101e7ea6605c8d42294d" dependencies = [ "ahash", "atoi", @@ -1675,7 +1775,7 @@ dependencies = [ "either", "ethnum", "fast-float", - "foreign_vec", + "futures", "getrandom 0.2.15", "hashbrown", "itoa", @@ -1683,8 +1783,10 @@ dependencies = [ "lz4", "multiversion", "num-traits", + "parking_lot", "polars-arrow-format", "polars-error", + "polars-schema", "polars-utils", "ryu", "simdutf8", @@ -1706,9 +1808,9 @@ dependencies = [ [[package]] name = "polars-compute" -version = "0.41.3" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f099516af30ac9ae4b4480f4ad02aa017d624f2f37b7a16ad4e9ba52f7e5269" +checksum = "5a97b2a5c9b880ab7e52553c40a336fdb6e3244bf896b4d4917700defe8085d5" dependencies = [ "bytemuck", "either", @@ -1722,9 +1824,9 @@ dependencies = [ [[package]] name = "polars-core" -version = "0.41.3" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2439484be228b8c302328e2f953e64cfd93930636e5c7ceed90339ece7fef6c" +checksum = "d5bc2cadcca904a9dc4d2c2b437c346712806e9a678bf17c7e94ebf622faae76" dependencies = [ "ahash", "bitflags 2.6.0", @@ -1741,12 +1843,12 @@ dependencies = [ "polars-compute", "polars-error", "polars-row", + "polars-schema", "polars-utils", "rand 0.8.5", "rand_distr", "rayon", "regex", - "smartstring", "thiserror", "version_check", "xxhash-rust", @@ -1754,9 +1856,9 @@ dependencies = [ [[package]] name = "polars-error" -version = "0.41.3" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c9b06dfbe79cabe50a7f0a90396864b5ee2c0e0f8d6a9353b2343c29c56e937" +checksum = "56b0a8eb9b1e56a4640de6887d613cb4de73c4e09d491f3b779855d4c3bcb9ba" dependencies = [ "polars-arrow-format", "regex", @@ -1766,14 +1868,15 @@ dependencies = [ [[package]] name = "polars-expr" -version = "0.41.3" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c630385a56a867c410a20f30772d088f90ec3d004864562b84250b35268f97" +checksum = "34e9c0e8c7ba93aac64051b92dc68eac5a0e9543cf44ca784467db2c035821fe" dependencies = [ "ahash", "bitflags 2.6.0", "once_cell", "polars-arrow", + "polars-compute", "polars-core", "polars-io", "polars-ops", @@ -1781,20 +1884,23 @@ dependencies = [ "polars-time", "polars-utils", "rayon", - "smartstring", ] [[package]] name = "polars-io" -version = "0.41.3" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d7363cd14e4696a28b334a56bd11013ff49cc96064818ab3f91a126e453462d" +checksum = "454ebbebe1cb8cb4768adca44b8fc9431abc3c91d5927f6824e73f916bced911" dependencies = [ "ahash", + "async-trait", "atoi_simd", "bytes", "chrono", "fast-float", + "futures", + "glob", + "hashbrown", "home", "itoa", "memchr", @@ -1805,24 +1911,26 @@ dependencies = [ "polars-arrow", "polars-core", "polars-error", + "polars-parquet", + "polars-schema", "polars-time", "polars-utils", "rayon", "regex", "ryu", "simdutf8", - "smartstring", + "tokio", + "tokio-util", ] [[package]] name = "polars-lazy" -version = "0.41.3" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03877e74e42b5340ae52ded705f6d5d14563d90554c9177b01b91ed2412a56ed" +checksum = "7e61c062e833d2376de0a4cf745504449215cbf499cea293cb592e674ffb39ca" dependencies = [ "ahash", "bitflags 2.6.0", - "glob", "memchr", "once_cell", "polars-arrow", @@ -1836,16 +1944,16 @@ dependencies = [ "polars-time", "polars-utils", "rayon", - "smartstring", "version_check", ] [[package]] name = "polars-mem-engine" -version = "0.41.3" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea9e17771af750c94bf959885e4b3f5b14149576c62ef3ec1c9ef5827b2a30f" +checksum = "c0643812829cc990e1533a5bf48c21a1b3eaa46aabf2967b0f53f99097cbc74c" dependencies = [ + "memmap2", "polars-arrow", "polars-core", "polars-error", @@ -1860,9 +1968,9 @@ dependencies = [ [[package]] name = "polars-ops" -version = "0.41.3" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6066552eb577d43b307027fb38096910b643ffb2c89a21628c7e41caf57848d0" +checksum = "5ac14a136d87bea798f3db51d5987556deb2293da34bfc8b105ebffa05f6e810" dependencies = [ "ahash", "argminmax", @@ -1880,23 +1988,30 @@ dependencies = [ "polars-compute", "polars-core", "polars-error", + "polars-schema", "polars-utils", "rayon", "regex", - "smartstring", "unicode-reverse", "version_check", ] [[package]] name = "polars-parquet" -version = "0.41.3" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b35b2592a2e7ef7ce9942dc2120dc4576142626c0e661668e4c6b805042e461" +checksum = "491f5af321169259d5b1294c9fe8ed89faaeac34b4dec4abcedc0d1b3d11013a" dependencies = [ "ahash", + "async-stream", "base64 0.22.1", + "brotli", + "bytemuck", "ethnum", + "flate2", + "futures", + "hashbrown", + "lz4", "num-traits", "parquet-format-safe", "polars-arrow", @@ -1904,18 +2019,21 @@ dependencies = [ "polars-error", "polars-utils", "simdutf8", + "snap", "streaming-decompression", + "zstd", ] [[package]] name = "polars-pipe" -version = "0.41.3" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "021bce7768c330687d735340395a77453aa18dd70d57c184cbb302311e87c1b9" +checksum = "29215c31f599295cc0f803c42fc812cc518db6d5ed4d6c7cc03daf3976a0add5" dependencies = [ "crossbeam-channel", "crossbeam-queue", "enum_dispatch", + "futures", "hashbrown", "num-traits", "polars-arrow", @@ -1928,43 +2046,46 @@ dependencies = [ "polars-row", "polars-utils", "rayon", - "smartstring", "uuid", "version_check", ] [[package]] name = "polars-plan" -version = "0.41.3" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "220d0d7c02d1c4375802b2813dbedcd1a184df39c43b74689e729ede8d5c2921" +checksum = "e3f728df4bc643492a2057a0a125c7e550cbcfe35b391444653ad294be9ab190" dependencies = [ "ahash", + "bitflags 2.6.0", "bytemuck", + "bytes", + "chrono", "chrono-tz", "either", "hashbrown", + "memmap2", "once_cell", "percent-encoding", "polars-arrow", "polars-core", "polars-io", "polars-ops", + "polars-parquet", "polars-time", "polars-utils", "rayon", "recursive", "regex", - "smartstring", "strum_macros", "version_check", ] [[package]] name = "polars-row" -version = "0.41.3" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1d70d87a2882a64a43b431aea1329cb9a2c4100547c95c417cc426bb82408b3" +checksum = "4eb931f0929ca7498b3ed5056357d2d364cad42cce95383a7e3983dbceb4bed1" dependencies = [ "bytemuck", "polars-arrow", @@ -1973,10 +2094,22 @@ dependencies = [ ] [[package]] -name = "polars-sql" -version = "0.41.3" +name = "polars-schema" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6fc1c9b778862f09f4a347f768dfdd3d0ba9957499d306d83c7103e0fa8dc5b" +checksum = "9c7e1234b942d3244024ecbac9c7f5a48a52a815f8ca4b9d075fbba16afb1a39" +dependencies = [ + "indexmap", + "polars-error", + "polars-utils", + "version_check", +] + +[[package]] +name = "polars-sql" +version = "0.43.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce52bfd2ef1e2e18ac26d7d7ea3f9132b199cff06d975156703fa5badcfae187" dependencies = [ "hex", "once_cell", @@ -1987,6 +2120,7 @@ dependencies = [ "polars-ops", "polars-plan", "polars-time", + "polars-utils", "rand 0.8.5", "serde", "serde_json", @@ -1995,9 +2129,9 @@ dependencies = [ [[package]] name = "polars-time" -version = "0.41.3" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "179f98313a15c0bfdbc8cc0f1d3076d08d567485b9952d46439f94fbc3085df5" +checksum = "9925ab75e1d859ae2283ca09d7683198b0b9ff5afecd03f2c9180f3e36e35056" dependencies = [ "atoi", "bytemuck", @@ -2011,25 +2145,27 @@ dependencies = [ "polars-ops", "polars-utils", "regex", - "smartstring", ] [[package]] name = "polars-utils" -version = "0.41.3" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e6dd89fcccb1ec1a62f752c9a9f2d482a85e9255153f46efecc617b4996d50" +checksum = "b44846e1fc6ae1dfdc7f65a37af7d270d0a6a17a58fff76716561f5b887a8ad7" dependencies = [ "ahash", "bytemuck", + "bytes", + "compact_str", "hashbrown", "indexmap", + "libc", + "memmap2", "num-traits", "once_cell", "polars-error", "raw-cpuid", "rayon", - "smartstring", "stacker", "sysinfo", "version_check", @@ -2601,15 +2737,10 @@ dependencies = [ ] [[package]] -name = "smartstring" -version = "1.0.1" +name = "snap" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" -dependencies = [ - "autocfg", - "static_assertions", - "version_check", -] +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" @@ -2652,9 +2783,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.47.0" +version = "0.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "295e9930cd7a97e58ca2a070541a3ca502b17f5d1fa7157376d0fabd85324f25" +checksum = "a4a404d0e14905361b918cb8afdb73605e25c1d5029312bd9785142dcb3aa49e" dependencies = [ "log", ] @@ -2962,15 +3093,14 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.30.13" +version = "0.31.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a5b4ddaee55fb2bea2bf0e5000747e5f5c0de765e5a5ff87f4cd106439f4bb3" +checksum = "355dbe4f8799b304b05e1b0f05fc59b2a18d36645cf169607da45bde2f69a1be" dependencies = [ - "cfg-if", "core-foundation-sys", "libc", + "memchr", "ntapi", - "once_cell", "windows", ] @@ -3379,11 +3509,11 @@ dependencies = [ [[package]] name = "windows" -version = "0.52.0" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" dependencies = [ - "windows-core", + "windows-core 0.57.0", "windows-targets 0.52.6", ] @@ -3396,6 +3526,49 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-implement" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "windows-interface" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index 2935cb3..482df4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,25 +17,25 @@ serde = { version = "1", features = ["derive"] } clap = { version = "4", features = ["derive"] } anyhow = "1" -itertools = "0.10" -chrono = {version = "0.4", features = ["default", "serde"]} +itertools = "0.13.0" +chrono = { version = "0.4", features = ["default", "serde"] } rayon = "1.6.0" tokio = { version = "1.39", features = ["full"] } -sqlx = { version = "0.8", features = [ "runtime-tokio-rustls", "any" ] } +sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "any"] } rmp-serde = "1.1" tempfile = "3.7" -polars = {version = "0.41", features = ["lazy", "performant", "streaming", "cse", "dtype-datetime"]} -polars-sql = "0.41" +polars = { version = "0.43.1", features = ["lazy", "performant", "parquet", "streaming", "cse", "dtype-datetime"] } +polars-sql = "0.43.1" serde_json = "1.0.122" num_cpus = "1.16.0" -schemars = {version = "0.8.21", features = ["chrono"]} +schemars = { version = "0.8.21", features = ["chrono"] } log = "0.4.22" env_logger = "0.11.5" -tiberius = {version = "0.12.3", features = ["chrono", "tokio"]} +tiberius = { version = "0.12.3", features = ["chrono", "tokio"] } futures-io = "0.3.30" futures = "0.3.30" -tokio-util = {version = "0.7.11", features = ["compat"]} +tokio-util = { version = "0.7.11", features = ["compat"] } async-trait = "0.1.81" # More info on targets: https://doc.rust-lang.org/cargo/reference/cargo-targets.html#configuring-a-target diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 5140a07..352dc92 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -174,7 +174,7 @@ impl Cli { let reader = BufReader::new(file); let graph = serde_json::from_reader(reader)?; let graph = RunnableGraph::from_graph(graph); - // TODO: Possible to await here? + // TODO: Possible to await here? Actually needs awaiting to work graph.run_default_tasks(threads, |id, status| { info!("Node with id {} finished with status {:?}", id, status) }); diff --git a/src/graph/split.rs b/src/graph/split.rs index bbf7c96..2a63604 100644 --- a/src/graph/split.rs +++ b/src/graph/split.rs @@ -1,13 +1,15 @@ -use std::collections::BTreeMap; - use async_trait::async_trait; use chrono::DateTime; +// use polars::io::SerReader; +// use polars::prelude::ParquetReader; use polars::{ io::SerWriter, prelude::{CsvWriter, LazyCsvReader, LazyFileListReader}, }; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +// use std::fs::File; use tempfile::tempfile; use crate::io::RecordSerializer; @@ -131,7 +133,11 @@ fn split( let mut temp_path = tempfile()?; // This needs to be done for each split rule with a change column specified + // TODO: Add parquet support (both read and write) + // let file = File::open(input)?; + // let df = ParquetReader::new(file).finish()?; let df = LazyCsvReader::new(input).finish()?; + // TODO: Needs sorting let df = df.sort(["", ""], Default::default()); CsvWriter::new(&mut temp_path).finish(&mut df.collect()?)?; diff --git a/src/products/create_products.rs b/src/products/create_products.rs index 36ea4db..63b7b87 100644 --- a/src/products/create_products.rs +++ b/src/products/create_products.rs @@ -114,7 +114,7 @@ pub fn build_polars( .joins .get(&definition.source) .ok_or(anyhow!("Failed to get right join column"))?; - reader = reader.inner_join(join_reader, col(&left_column), col(&right_column)); + reader = reader.inner_join(join_reader, col(left_column), col(right_column)); } } // TODO: Also work out how to expand rows, so that transfers can have stuff like daily or change in x expanded into multiple rows @@ -135,7 +135,7 @@ pub fn build_polars( // to an actual filename. I don't want to be limited by a concept of 'sourcetype' at all, instead the definition should treat everything // the same, and just translate the imported csv format to the necessary files and columns in files that are expected to be input. Component::Field(source_type, column) => { - built_expression = built_expression + col(&column) + built_expression = built_expression + col(column) } } }