Update dependencies, add some todos/notes for future changes

This commit is contained in:
2024-09-24 11:35:07 +09:30
parent bc5497ea16
commit 375e1f9638
5 changed files with 266 additions and 87 deletions

View File

@@ -1,13 +1,15 @@
use std::collections::BTreeMap;
use async_trait::async_trait;
use chrono::DateTime;
// use polars::io::SerReader;
// use polars::prelude::ParquetReader;
use polars::{
io::SerWriter,
prelude::{CsvWriter, LazyCsvReader, LazyFileListReader},
};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
// use std::fs::File;
use tempfile::tempfile;
use crate::io::RecordSerializer;
@@ -131,7 +133,11 @@ fn split(
let mut temp_path = tempfile()?;
// This needs to be done for each split rule with a change column specified
// TODO: Add parquet support (both read and write)
// let file = File::open(input)?;
// let df = ParquetReader::new(file).finish()?;
let df = LazyCsvReader::new(input).finish()?;
// TODO: Needs sorting
let df = df.sort(["", ""], Default::default());
CsvWriter::new(&mut temp_path).finish(&mut df.collect()?)?;