use anyhow::bail; use rmp_serde::{decode::ReadReader, Deserializer, Serializer}; use schemars::JsonSchema; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use std::path::PathBuf; use std::{ collections::BTreeMap, io::{Read, Seek, Write}, }; #[derive(Serialize, Deserialize, Clone, JsonSchema)] pub enum SourceType { CSV, PARQUET, } #[derive(Serialize, Deserialize, Clone, JsonSchema)] pub struct DataSource { pub path: PathBuf, pub source_type: SourceType, } pub trait RecordSerializer { fn serialize(&mut self, record: impl Serialize) -> anyhow::Result<()>; // For when serde serialization can't be used. Forcing BTreeMap to ensure keys/values are // sorted consistently fn write_header(&mut self, record: &BTreeMap) -> anyhow::Result<()>; fn write_record(&mut self, record: &BTreeMap) -> anyhow::Result<()>; fn flush(&mut self) -> anyhow::Result<()>; } impl RecordSerializer for csv::Writer { fn serialize(&mut self, record: impl Serialize) -> anyhow::Result<()> { self.serialize(record)?; Ok(()) } fn flush(&mut self) -> anyhow::Result<()> { self.flush()?; Ok(()) } fn write_header(&mut self, record: &BTreeMap) -> anyhow::Result<()> { self.write_record(record.keys())?; Ok(()) } fn write_record(&mut self, record: &BTreeMap) -> anyhow::Result<()> { self.write_record(record.values())?; Ok(()) } } impl RecordSerializer for Serializer { fn serialize(&mut self, record: impl Serialize) -> anyhow::Result<()> { record.serialize(self)?; Ok(()) } fn flush(&mut self) -> anyhow::Result<()> { Ok(()) } fn write_header(&mut self, _: &BTreeMap) -> anyhow::Result<()> { Ok(()) } fn write_record(&mut self, record: &BTreeMap) -> anyhow::Result<()> { self.serialize(record)?; Ok(()) } } pub trait RecordDeserializer { fn deserialize(&mut self) -> Result, anyhow::Error>; } impl RecordDeserializer for csv::Reader { fn deserialize(&mut self) -> Result, anyhow::Error> { match self.deserialize().next() { None => Ok(Option::None), Some(result) => match result { Ok(ok) => Ok(Option::Some(ok)), Err(err) => bail!(err), }, } } } impl RecordDeserializer for Deserializer> { fn deserialize(&mut self) -> Result, anyhow::Error> { match Deserialize::deserialize(self) { Ok(value) => Ok(value), Err(value) => Err(anyhow::Error::from(value)), } } }