vato007/ingey
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add some notes on customising reader/deserialiser to use one other than csv that interops with serde.

Browse Source
This commit is contained in:
Piv
2023-07-29 15:02:45 +09:30
parent 12add6e5c1
commit e7a6e31015
2 changed files with 60 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/filter.rs
View File

@@ -1,8 +1,4 @@
use std::{ use std::{collections::HashMap, io::Read, str::FromStr};
collections::HashMap,
io::{Read, Write},
str::FromStr,
};
use crate::io::RecordSerializer; use crate::io::RecordSerializer;

83
src/io.rs
View File

@@ -1,12 +1,7 @@
use std::{ use std::io::{Read, Write};
collections::HashMap,
io::{Read, Write},
};
use anyhow::bail;
use csv::DeserializeRecordsIter;
use rmp_serde::{Deserializer, Serializer}; use rmp_serde::{Deserializer, Serializer};
use serde::{de::DeserializeOwned, Serialize}; use serde::{Deserialize, Serialize};
pub trait RecordSerializer { pub trait RecordSerializer {
fn serialize(&mut self, record: impl Serialize) -> anyhow::Result<()>; fn serialize(&mut self, record: impl Serialize) -> anyhow::Result<()>;
@@ -26,28 +21,68 @@ impl<W: Write> RecordSerializer for Serializer<W> {
} }
} }
// TODO: Want to be able to deserialise with serde over a reader like we currently do with the writer // // pub struct RecordDeserializerIterWrapper<
// pub trait RecordDeserializer<D: DeserializeOwned, I: Iterator<Item = Result<D, anyhow::Error>>> { // // D: DeserializeOwned,
// fn deserialize<I>(&mut self) -> I; // // I: Iterator<Item = Result<D, anyhow::Error>>,
// // > {
// // pub inner: I,
// // }
// // impl<D: DeserializeOwned, I: Iterator<Item = Result<D, anyhow::Error>>> Iterator
// // for RecordDeserializerIterWrapper<D, I>
// // {
// // type Item = Result<D, anyhow::Error>;
// // fn next(&mut self) -> Option<Self::Item> {
// // self.inner.next()
// // }
// // }
// // This person has the same issue as me (it's exactly the same, just with different serde implementations)
// // https://stackoverflow.com/questions/69691366/using-serde-for-two-deserialization-formats
// // impl <I: Iterator<Item = Iterator for RecordDeserializerIterWrapper<>
// // TODO: Want to be able to deserialise with serde over a reader like we currently do with the writer (rather than forcing to deserialize to a hashmap)
// // However, this doesn't really work when returning an arbitrary iterator, so the iterator should be specific, like what's done in
// // This from rust messagepack is basically what I'm looking for? Would prefer to be an iterator but might still work?
// // https://github.com/3Hren/msgpack-rust/blob/master/rmp-serde/src/decode.rs#L1017
// // I think what might be possible is to have a struct for messagepack that wraps the deserializer, implements the iter trait (for result deserializeowned),
// // then just always return that struct (avoids the issues of returning an arbitrary trait). Really something like I did with RecordSerializer would be the best
// // But the problem is I can't easily return arbitrary iterators without boxing (then specifying what I'm returning anyway), really in java the implementing
// // class could return whatever implementation it wants...
// // Maybe I just create a custom deserializer that implements Deserializer for both csv and msgpack? Then just return the struct? Seems like
// // the opposite of recordserializer though which is kind of annoying.
// // Really what I would like is to be able to just pass a deserializer, but the issues is csv doesn't export its deserializer (though it does have an implementation)
// // Actually it's even more complex than that, as I want to also be able to seek in the underlying reader somehow, I think this'll only be supported by
// // csv though? For now just leave it and stick with csv reading instead
// pub trait RecordDeserializer {
// // TODO: Don't do this, deserializer is for messagepack only, want it to easily support other deserializers
// fn deserializeT<D: Deserialize<'de>>(&mut self) -> Result<D, anyhow::Error>;
// } // }
// impl<R: Read, D: DeserializeOwned, I: Iterator<Item = Result<D, anyhow::Error>>> // // impl<R: Read> RecordDeserializer for csv::Reader<R> {
// RecordDeserializer<D, I> for csv::Reader<R> // // fn deserialize<D: DeserializeOwned>(
// { // // &mut self,
// fn deserialize(&mut self) -> I { // // ) -> Box<dyn Iterator<Item = Result<D, anyhow::Error>>> {
// self.deserialize().into_iter().map(|r| match r { // // Box::new(self.deserialize().into_iter().map(|r| match r {
// Ok(ok) => Ok(ok), // // Ok(ok) => Ok(ok),
// Err(err) => bail!(err), // // Err(err) => bail!(err),
// }) // // }))
// } // // }
// } // // }
// impl<R: Read> RecordDeserializer for Deserializer<R> { // impl<R: Read> RecordDeserializer for Deserializer<R> {
// fn deserialize<I, D>(&mut self) -> I // fn deserializeT<'de, D>(&mut self) -> Result<D, anyhow::Error>
// where // where
// D: DeserializeOwned, // D: Deserialize<'de>,
// I: Iterator<Item = D>,
// { // {
// todo!() // match Deserialize::deserialize(self) {
// Ok(ok) => Ok(ok),
// Err(e) => Err(anyhow::Error::from(e)),
// }
// } // }
// } // }