diff --git a/Cargo.lock b/Cargo.lock index 3325b46..f1594b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -190,9 +190,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.81" +version = "0.1.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", @@ -290,6 +290,56 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bollard" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d41711ad46fda47cd701f6908e59d1bd6b9a2b7464c0d0aeab95c6d37096ff8a" +dependencies = [ + "base64 0.22.1", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "home", + "http", + "http-body-util", + "hyper", + "hyper-named-pipe", + "hyper-rustls", + "hyper-util", + "hyperlocal", + "log", + "pin-project-lite", + "rustls 0.23.20", + "rustls-native-certs", + "rustls-pemfile 2.2.0", + "rustls-pki-types", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror", + "tokio", + "tokio-util", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.45.0-rc.26.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7c5415e3a6bc6d3e99eff6268e488fd4ee25e7b28c10f08fa6760bd9de16e4" +dependencies = [ + "serde", + "serde_repr", + "serde_with", +] + [[package]] name = "brotli" version = "6.0.0" @@ -376,9 +426,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -546,6 +596,7 @@ dependencies = [ "serde_json", "sqlx", "tempfile", + "testcontainers", "tiberius", "tokio", "tokio-util", @@ -680,6 +731,41 @@ dependencies = [ "memchr", ] +[[package]] +name = "darling" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.72", +] + +[[package]] +name = "darling_macro" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.72", +] + [[package]] name = "der" version = "0.7.9" @@ -691,6 +777,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", + "serde", +] + [[package]] name = "digest" version = "0.10.7" @@ -703,6 +799,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "docker_credential" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31951f49556e34d90ed28342e1df7e1cb7a229c4cab0aecc627b5d91edd41d07" +dependencies = [ + "base64 0.21.7", + "serde", + "serde_json", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -777,9 +884,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d" +checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0" dependencies = [ "anstream", "anstyle", @@ -850,6 +957,18 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +[[package]] +name = "filetime" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.59.0", +] + [[package]] name = "flate2" version = "1.0.30" @@ -871,6 +990,12 @@ dependencies = [ "spin", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foreign-types" version = "0.3.2" @@ -897,9 +1022,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -912,9 +1037,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -922,15 +1047,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -950,15 +1075,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -967,21 +1092,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -1041,6 +1166,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -1059,7 +1190,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" dependencies = [ - "hashbrown", + "hashbrown 0.14.5", ] [[package]] @@ -1107,12 +1238,144 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "http" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "humantime" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-named-pipe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" +dependencies = [ + "hex", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", + "winapi", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +dependencies = [ + "futures-util", + "http", + "hyper", + "hyper-util", + "rustls 0.23.20", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -1136,6 +1399,12 @@ dependencies = [ "cc", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.5.0" @@ -1146,6 +1415,17 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.2.6" @@ -1153,7 +1433,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.14.5", "serde", ] @@ -1223,6 +1503,17 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.6.0", + "libc", + "redox_syscall 0.5.3", +] + [[package]] name = "libsqlite3-sys" version = "0.28.0" @@ -1474,6 +1765,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.46" @@ -1623,6 +1920,31 @@ dependencies = [ "futures", ] +[[package]] +name = "parse-display" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax", +] + +[[package]] +name = "parse-display-derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax", + "structmeta", + "syn 2.0.72", +] + [[package]] name = "parse-zoneinfo" version = "0.3.1" @@ -1777,7 +2099,7 @@ dependencies = [ "fast-float", "futures", "getrandom 0.2.15", - "hashbrown", + "hashbrown 0.14.5", "itoa", "itoap", "lz4", @@ -1835,8 +2157,8 @@ dependencies = [ "chrono-tz", "comfy-table", "either", - "hashbrown", - "indexmap", + "hashbrown 0.14.5", + "indexmap 2.2.6", "num-traits", "once_cell", "polars-arrow", @@ -1900,7 +2222,7 @@ dependencies = [ "fast-float", "futures", "glob", - "hashbrown", + "hashbrown 0.14.5", "home", "itoa", "memchr", @@ -1979,9 +2301,9 @@ dependencies = [ "chrono", "chrono-tz", "either", - "hashbrown", + "hashbrown 0.14.5", "hex", - "indexmap", + "indexmap 2.2.6", "memchr", "num-traits", "polars-arrow", @@ -2010,7 +2332,7 @@ dependencies = [ "ethnum", "flate2", "futures", - "hashbrown", + "hashbrown 0.14.5", "lz4", "num-traits", "parquet-format-safe", @@ -2034,7 +2356,7 @@ dependencies = [ "crossbeam-queue", "enum_dispatch", "futures", - "hashbrown", + "hashbrown 0.14.5", "num-traits", "polars-arrow", "polars-compute", @@ -2063,7 +2385,7 @@ dependencies = [ "chrono", "chrono-tz", "either", - "hashbrown", + "hashbrown 0.14.5", "memmap2", "once_cell", "percent-encoding", @@ -2099,7 +2421,7 @@ version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c7e1234b942d3244024ecbac9c7f5a48a52a815f8ca4b9d075fbba16afb1a39" dependencies = [ - "indexmap", + "indexmap 2.2.6", "polars-error", "polars-utils", "version_check", @@ -2157,8 +2479,8 @@ dependencies = [ "bytemuck", "bytes", "compact_str", - "hashbrown", - "indexmap", + "hashbrown 0.14.5", + "indexmap 2.2.6", "libc", "memmap2", "num-traits", @@ -2171,6 +2493,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.19" @@ -2350,6 +2678,15 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -2480,10 +2817,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "ring", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.23.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki 0.102.8", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +dependencies = [ + "openssl-probe", + "rustls-pemfile 2.2.0", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -2493,6 +2857,21 @@ dependencies = [ "base64 0.21.7", ] +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -2503,6 +2882,17 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustls-webpki" +version = "0.102.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.17" @@ -2640,6 +3030,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2652,6 +3053,36 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.2.6", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.72", +] + [[package]] name = "sha1" version = "0.10.6" @@ -2821,17 +3252,17 @@ dependencies = [ "futures-intrusive", "futures-io", "futures-util", - "hashbrown", + "hashbrown 0.14.5", "hashlink", "hex", - "indexmap", + "indexmap 2.2.6", "log", "memchr", "once_cell", "paste", "percent-encoding", - "rustls", - "rustls-pemfile", + "rustls 0.21.12", + "rustls-pemfile 1.0.4", "serde", "serde_json", "sha2", @@ -3044,6 +3475,29 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "structmeta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn 2.0.72", +] + +[[package]] +name = "structmeta-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + [[package]] name = "strum" version = "0.26.3" @@ -3122,6 +3576,35 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "testcontainers" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f40cc2bd72e17f328faf8ca7687fe337e61bccd8acf9674fa78dd3792b045e1" +dependencies = [ + "async-trait", + "bollard", + "bollard-stubs", + "bytes", + "docker_credential", + "either", + "etcetera", + "futures", + "log", + "memchr", + "parse-display", + "pin-project-lite", + "serde", + "serde_json", + "serde_with", + "thiserror", + "tokio", + "tokio-stream", + "tokio-tar", + "tokio-util", + "url", +] + [[package]] name = "thiserror" version = "1.0.63" @@ -3169,6 +3652,37 @@ dependencies = [ "winauth", ] +[[package]] +name = "time" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinyvec" version = "1.8.0" @@ -3186,9 +3700,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.39.2" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" +checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" dependencies = [ "backtrace", "bytes", @@ -3213,6 +3727,16 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "tokio-rustls" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" +dependencies = [ + "rustls 0.23.20", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.15" @@ -3225,10 +3749,25 @@ dependencies = [ ] [[package]] -name = "tokio-util" -version = "0.7.11" +name = "tokio-tar" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +checksum = "9d5714c010ca3e5c27114c1cdeb9d14641ace49874aa5626d7149e47aedace75" +dependencies = [ + "filetime", + "futures-core", + "libc", + "redox_syscall 0.3.5", + "tokio", + "tokio-stream", + "xattr", +] + +[[package]] +name = "tokio-util" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" dependencies = [ "bytes", "futures-core", @@ -3238,6 +3777,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.40" @@ -3270,6 +3815,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typenum" version = "1.17.0" @@ -3345,6 +3896,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -3374,6 +3926,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -3587,6 +4148,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -3708,6 +4278,17 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "xattr" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +dependencies = [ + "libc", + "linux-raw-sys", + "rustix", +] + [[package]] name = "xxhash-rust" version = "0.8.12" diff --git a/Cargo.toml b/Cargo.toml index 482df4e..ec93c4d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,10 +18,10 @@ clap = { version = "4", features = ["derive"] } anyhow = "1" itertools = "0.13.0" -chrono = { version = "0.4", features = ["default", "serde"] } +chrono = { version = "0.4.39", features = ["default", "serde"] } -rayon = "1.6.0" -tokio = { version = "1.39", features = ["full"] } +rayon = "1.10.0" +tokio = { version = "1.42.0", features = ["full"] } sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "any"] } rmp-serde = "1.1" tempfile = "3.7" @@ -31,12 +31,13 @@ serde_json = "1.0.122" num_cpus = "1.16.0" schemars = { version = "0.8.21", features = ["chrono"] } log = "0.4.22" -env_logger = "0.11.5" +env_logger = "0.11.6" tiberius = { version = "0.12.3", features = ["chrono", "tokio"] } -futures-io = "0.3.30" -futures = "0.3.30" -tokio-util = { version = "0.7.11", features = ["compat"] } -async-trait = "0.1.81" +futures-io = "0.3.31" +futures = "0.3.31" +tokio-util = { version = "0.7.13", features = ["compat"] } +async-trait = "0.1.83" +testcontainers = "0.23.1" # More info on targets: https://doc.rust-lang.org/cargo/reference/cargo-targets.html#configuring-a-target [lib] diff --git a/src/graph/derive.rs b/src/graph/derive.rs index 0fde62d..5b14829 100644 --- a/src/graph/derive.rs +++ b/src/graph/derive.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::io::{RecordDeserializer, RecordSerializer}; +use crate::io::{DataSource, RecordDeserializer, RecordSerializer}; use super::node::RunnableNode; @@ -165,8 +165,8 @@ pub struct DeriveRule { #[derive(Serialize, Deserialize, Clone, JsonSchema)] pub struct DeriveNode { pub rules: Vec, - pub input_file_path: String, - pub output_file_path: String, + pub input_data_source: DataSource, + pub output_data_source: DataSource, pub copy_all_columns: bool, } @@ -302,8 +302,8 @@ pub struct DeriveNodeRunner { #[async_trait] impl RunnableNode for DeriveNodeRunner { async fn run(&self) -> anyhow::Result<()> { - let mut reader = csv::Reader::from_path(&self.derive_node.input_file_path)?; - let mut writer = csv::Writer::from_path(&self.derive_node.output_file_path)?; + let mut reader = csv::Reader::from_path(&self.derive_node.input_data_source.path)?; + let mut writer = csv::Writer::from_path(&self.derive_node.output_data_source.path)?; let rules: anyhow::Result> = self .derive_node .rules diff --git a/src/graph/filter.rs b/src/graph/filter.rs index dacafdd..7633999 100644 --- a/src/graph/filter.rs +++ b/src/graph/filter.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::io::{RecordDeserializer, RecordSerializer}; +use crate::io::{DataSource, RecordDeserializer, RecordSerializer}; use super::derive::{DataValidators, DeriveFilter}; @@ -42,8 +42,8 @@ pub fn filter_file( #[derive(Serialize, Deserialize, Clone, JsonSchema)] pub struct FilterNode { pub filters: Vec, - pub input_file_path: String, - pub output_file_path: String, + pub input_data_source: DataSource, + pub output_data_source: DataSource, } pub struct FilterNodeRunner { @@ -53,8 +53,8 @@ pub struct FilterNodeRunner { #[async_trait] impl RunnableNode for FilterNodeRunner { async fn run(&self) -> anyhow::Result<()> { - let mut reader = csv::Reader::from_path(&self.filter_node.input_file_path)?; - let mut writer = csv::Writer::from_path(&self.filter_node.output_file_path)?; + let mut reader = csv::Reader::from_path(&self.filter_node.input_data_source.path)?; + let mut writer = csv::Writer::from_path(&self.filter_node.output_data_source.path)?; let rules = derive::to_filter_rules(&self.filter_node.filters)?; filter_file(&rules, &mut reader, &mut writer) } @@ -62,7 +62,6 @@ impl RunnableNode for FilterNodeRunner { #[cfg(test)] mod tests { - use super::derive::{Comparator, FilterRule}; use super::filter_file; diff --git a/src/graph/mod.rs b/src/graph/mod.rs index a802c05..e99fb0b 100644 --- a/src/graph/mod.rs +++ b/src/graph/mod.rs @@ -308,9 +308,10 @@ impl RunnableGraph { #[cfg(test)] mod tests { - use chrono::Local; - use super::{NodeConfiguration, RunnableGraph}; + use crate::io::{DataSource, SourceType}; + use chrono::Local; + use std::path::PathBuf; #[tokio::test] async fn test_basic() -> anyhow::Result<()> { @@ -324,8 +325,8 @@ mod tests { name: "Hello".to_owned(), configuration: NodeConfiguration::FilterNode(super::FilterNode { filters: vec![], - input_file_path: "".to_owned(), - output_file_path: "".to_owned(), + input_data_source: DataSource { path: PathBuf::from(""), source_type: SourceType::CSV }, + output_data_source: DataSource { path: PathBuf::from(""), source_type: SourceType::CSV }, }), output_files: vec![], dynamic_configuration: None, diff --git a/src/graph/pull_from_db.rs b/src/graph/pull_from_db.rs index c0ee49c..3e553b9 100644 --- a/src/graph/pull_from_db.rs +++ b/src/graph/pull_from_db.rs @@ -1,21 +1,36 @@ use super::sql::QueryExecutor; use crate::graph::node::RunnableNode; use crate::graph::upload_to_db::{upload_file_bulk, DBType}; +use crate::io::{DataSource, RecordSerializer}; use async_trait::async_trait; +use polars::prelude::CsvWriter; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use sqlx::AnyPool; -use tiberius::Config; +use std::collections::BTreeMap; +use tiberius::{AuthMethod, Config, EncryptionLevel}; use tokio_util::compat::TokioAsyncWriteCompatExt; /** * Pull data from a db using a db query into a csv file that can be used by another node */ -async fn pull_from_db(executor: &mut impl QueryExecutor, node: &PullFromDBNode) {} +async fn pull_from_db(executor: &mut impl QueryExecutor, node: &PullFromDBNode) -> anyhow::Result<()> { + let mut output_file = csv::Writer::from_path(node.output_data_source.path.clone())?; + let mut first_row = true; + executor.get_rows(&node.query, &node.parameters, &mut move |row| { + if first_row { + output_file.write_header(&row)?; + first_row = false; + } + output_file.write_record(row.values())?; + Ok(()) + }).await?; + Ok(()) +} #[derive(Serialize, Deserialize, Clone, JsonSchema)] pub struct PullFromDBNode { - file_path: String, + output_data_source: DataSource, query: String, parameters: Vec, db_type: DBType, @@ -33,17 +48,75 @@ impl RunnableNode for PullFromDBNodeRunner { // TODO: Clean up grabbing of connection/executor so don't need to repeat this between upload/download match node.db_type { DBType::Mssql => { - let config = Config::from_jdbc_string(&node.connection_string)?; + let mut config = Config::from_jdbc_string(&node.connection_string)?; + // TODO: Restore encryption for remote hosts, doesn't work on localhost without encryption. + config.encryption(EncryptionLevel::NotSupported); let tcp = tokio::net::TcpStream::connect(config.get_addr()).await?; tcp.set_nodelay(true)?; let mut client = tiberius::Client::connect(config, tcp.compat_write()).await?; - pull_from_db(&mut client, &node).await; + pull_from_db(&mut client, &node).await?; } _ => { let mut pool = AnyPool::connect(&node.connection_string).await?; - pull_from_db(&mut pool, &node).await; + pull_from_db(&mut pool, &node).await?; } } Ok(()) } } + +#[cfg(test)] +mod tests { + use crate::graph::node::RunnableNode; + use crate::graph::pull_from_db::{PullFromDBNode, PullFromDBNodeRunner}; + use crate::graph::upload_to_db::DBType::Mssql; + use crate::io::DataSource; + use crate::io::SourceType::CSV; + use std::fs::File; + use std::io::Read; + use std::path::PathBuf; + use testcontainers::core::{IntoContainerPort, WaitFor}; + use testcontainers::runners::AsyncRunner; + use testcontainers::{GenericImage, ImageExt}; + + #[tokio::test] + async fn test_sql_server() -> anyhow::Result<()> { + let container = GenericImage::new("mcr.microsoft.com/mssql/server", "2022-latest") + .with_exposed_port(1433.tcp()) + .with_wait_for(WaitFor::message_on_stdout("Recovery is complete.".to_owned())) + .with_env_var("ACCEPT_EULA", "Y") + .with_env_var("MSSQL_SA_PASSWORD", "TestOnlyContainer123") + .start() + .await?; + let host = container.get_host().await?; + let port = container.get_host_port_ipv4(1433).await?; + let port = 1433; + let connection_string = format!("jdbc:sqlserver://{}:{};username=sa;password=TestOnlyContainer123", host, port).to_owned(); + let connection_string = "jdbc:sqlserver://localhost:1433;username=sa;password=TestOnlyContainer123;Encrypt=False".to_owned(); + + let runner = PullFromDBNodeRunner { + pull_from_db_node: PullFromDBNode { + db_type: Mssql, + query: "SELECT '1' Test".to_owned(), + parameters: vec![], + connection_string, + output_data_source: DataSource { + path: PathBuf::from("test_pull.csv"), + source_type: CSV, + }, + } + }; + runner.run().await?; + let mut result_contents = String::new(); + let result_length = File::open("test_pull.csv")?.read_to_string(&mut result_contents)?; + assert_eq!( + "Test +1 +", + result_contents + , + "Should pull the correct data from sql" + ); + Ok(()) + } +} diff --git a/src/graph/sql.rs b/src/graph/sql.rs index 357d4dd..c885937 100644 --- a/src/graph/sql.rs +++ b/src/graph/sql.rs @@ -1,21 +1,21 @@ -use std::borrow::Borrow; - use futures::TryStreamExt; use futures_io::{AsyncRead, AsyncWrite}; use itertools::Itertools; -use sqlx::{Any, AnyPool, Column, Pool, Row}; +use sqlx::{Any, Column, Pool, Row}; +use std::borrow::Borrow; +use std::collections::BTreeMap; use tiberius::{Client, Query}; -// TODO: This doesn't seem to work. Suggestion by compiler is to instead create an enum and implement -// the trait on the enum (basically use a match in the implementation depending on which enum we have) + pub trait QueryExecutor { - // TODO: Params binding for filtering the same query? // Retrieve data from a database async fn get_rows( &mut self, query: &str, params: &Vec, - ) -> anyhow::Result>>; + // TODO: This is looking pretty ugly, simpler way to handle it? Maybe with an iterator? + row_consumer: &mut impl FnMut(BTreeMap) -> anyhow::Result<()>, + ) -> anyhow::Result<()>; // Run a query that returns no results (e.g. bulk insert, insert) async fn execute_query(&mut self, query: &str, params: &Vec) -> anyhow::Result; @@ -26,32 +26,25 @@ impl QueryExecutor for Client { &mut self, query: &str, params: &Vec, - ) -> anyhow::Result>> { + row_consumer: &mut impl FnMut(BTreeMap) -> anyhow::Result<()>, + ) -> anyhow::Result<()> { let mut query = Query::new(query); for param in params { query.bind(param); } let query_result = query.query(self).await?; - let results = query_result.into_first_result().await?; - let results = results - .into_iter() - .map(|row| { - row.columns() - .into_iter() - .map(|column| { - ( - column.name().to_owned(), - match row.get(column.name()) { - Some(value) => value, - None => "", - } - .to_owned(), - ) - }) - .collect_vec() - }) - .collect(); - Ok(results) + let mut query_stream = query_result.into_row_stream(); + + while let Some(row) = query_stream.try_next().await? { + let mut returned_row = BTreeMap::new(); + // TODO: Check how empty columns are handled by tiberius + for column in row.columns().into_iter() { + returned_row.insert(column.name().to_owned(), row.get(column.name()).unwrap_or_else(|| "") + .to_owned()); + } + row_consumer(returned_row)?; + } + Ok(()) } async fn execute_query(&mut self, query: &str, params: &Vec) -> anyhow::Result { @@ -72,22 +65,22 @@ impl QueryExecutor for Pool { &mut self, query: &str, params: &Vec, - ) -> anyhow::Result>> { + row_consumer: &mut impl FnMut(BTreeMap) -> anyhow::Result<()>, + ) -> anyhow::Result<()> { let mut query = sqlx::query(query); for param in params { query = query.bind(param); } let mut rows = query.fetch(self.borrow()); - let mut results = vec![]; + while let Some(row) = rows.try_next().await? { - results.push( - row.columns() - .into_iter() - .map(|column| (column.name().to_owned(), row.get(column.name()))) - .collect(), - ); + let mut returned_row = BTreeMap::new(); + for column in row.columns().into_iter() { + returned_row.insert(column.name().to_owned(), row.get(column.name())); + } + row_consumer(returned_row)?; } - Ok(results) + Ok(()) } async fn execute_query(&mut self, query: &str, params: &Vec) -> anyhow::Result { diff --git a/src/graph/sql_rule.rs b/src/graph/sql_rule.rs index d3e9c9a..8645fd3 100644 --- a/src/graph/sql_rule.rs +++ b/src/graph/sql_rule.rs @@ -1,6 +1,10 @@ use std::fs::File; +use super::node::RunnableNode; +use crate::io::{DataSource, SourceType}; use async_trait::async_trait; +use polars::io::SerReader; +use polars::prelude::{IntoLazy, LazyFrame, ParquetReader, ScanArgsParquet}; use polars::{ io::SerWriter, prelude::{CsvWriter, LazyCsvReader, LazyFileListReader}, @@ -9,21 +13,24 @@ use polars_sql::SQLContext; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use super::node::RunnableNode; - #[derive(Serialize, Deserialize, Clone, JsonSchema)] -pub struct CSVFile { - name: String, - path: String, +pub struct SqlFile { + pub name: String, + pub data_source: DataSource, } /** * Run SQL over files using polars, export results to output file */ -fn run_sql(files: &Vec, output_path: &String, query: &String) -> anyhow::Result<()> { +fn run_sql(files: &Vec, output_path: &String, query: &String) -> anyhow::Result<()> { let mut ctx = SQLContext::new(); for file in files { - let df = LazyCsvReader::new(&file.path).finish()?; + let df = match file.data_source.source_type { + SourceType::CSV => LazyCsvReader::new(&file.data_source.path).finish()?, + SourceType::PARQUET => { + LazyFrame::scan_parquet(&file.data_source.path, ScanArgsParquet::default())? + } + }; ctx.register(&file.name, df); } let result = ctx.execute(&query)?; @@ -34,7 +41,7 @@ fn run_sql(files: &Vec, output_path: &String, query: &String) -> anyhow #[derive(Serialize, Deserialize, Clone, JsonSchema)] pub struct SQLNode { - pub files: Vec, + pub files: Vec, pub output_file: String, pub query: String, } @@ -55,17 +62,21 @@ impl RunnableNode for SQLNodeRunner { } #[cfg(test)] mod tests { + use super::{run_sql, SqlFile}; + use crate::io::{DataSource, SourceType}; + use std::path::PathBuf; use std::{fs::File, io::Read}; - use super::{run_sql, CSVFile}; - #[test] fn basic_query_works() -> anyhow::Result<()> { let output_path = "./testing/output/output.csv".to_owned(); run_sql( - &vec![CSVFile { + &vec![SqlFile { name: "Account".to_owned(), - path: "./testing/test.csv".to_owned(), + data_source: DataSource { + source_type: SourceType::CSV, + path: PathBuf::from("./testing/test.csv"), + } }], &output_path, &"SELECT * FROM Account WHERE Code = 'A195950'".to_owned(), @@ -76,7 +87,7 @@ mod tests { assert_eq!( output, "Code,Description,Type,CostOutput,PercentFixed -A195950,A195950 Staff Related Other,E,GS,100.00 +A195950,A195950 Staff Related Other,E,GS,100.0 " ); Ok(()) diff --git a/src/io.rs b/src/io.rs index 0480bf5..27c5672 100644 --- a/src/io.rs +++ b/src/io.rs @@ -1,11 +1,24 @@ +use anyhow::bail; +use rmp_serde::{decode::ReadReader, Deserializer, Serializer}; +use schemars::JsonSchema; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use std::path::PathBuf; use std::{ collections::BTreeMap, io::{Read, Seek, Write}, }; -use anyhow::bail; -use rmp_serde::{decode::ReadReader, Deserializer, Serializer}; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; +#[derive(Serialize, Deserialize, Clone, JsonSchema)] +pub enum SourceType { + CSV, + PARQUET, +} + +#[derive(Serialize, Deserialize, Clone, JsonSchema)] +pub struct DataSource { + pub path: PathBuf, + pub source_type: SourceType, +} pub trait RecordSerializer { fn serialize(&mut self, record: impl Serialize) -> anyhow::Result<()>;