From 5d0b7fa903a127064816389b83a433e42803cb61 Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Mon, 13 Aug 2018 09:25:48 +0300 Subject: [PATCH] Add Envelope parsing caching Concerns #28 --- melib/Cargo.toml | 5 ++- melib/src/mailbox/backends/maildir.rs | 49 +++++++++++++++++++-- melib/src/mailbox/backends/mod.rs | 1 + melib/src/mailbox/email/attachment_types.rs | 10 ++--- melib/src/mailbox/email/attachments.rs | 4 +- melib/src/mailbox/email/mod.rs | 16 +++---- ui/src/components/mail/listing/mod.rs | 9 ++-- ui/src/state.rs | 17 ++++--- 8 files changed, 81 insertions(+), 30 deletions(-) diff --git a/melib/Cargo.toml b/melib/Cargo.toml index 19c9c0f7..ba732883 100644 --- a/melib/Cargo.toml +++ b/melib/Cargo.toml @@ -17,7 +17,8 @@ memmap = "0.5.2" nom = "3.2.0" notify = "4.0.1" notify-rust = "^3" -serde = "^1.0.8" -serde_derive = "^1.0.8" termion = "1.5.1" xdg = "2.1.0" +serde = "1.0.71" +serde_derive = "1.0.71" +bincode = "1.0.1" diff --git a/melib/src/mailbox/backends/maildir.rs b/melib/src/mailbox/backends/maildir.rs index 1cdd2589..e6ef374a 100644 --- a/melib/src/mailbox/backends/maildir.rs +++ b/melib/src/mailbox/backends/maildir.rs @@ -19,6 +19,10 @@ * along with meli. If not, see . */ +extern crate xdg; +extern crate serde_derive; +extern crate bincode; + use async::*; use conf::AccountSettings; use error::{MeliError, Result}; @@ -43,6 +47,8 @@ extern crate crossbeam; use memmap::{Mmap, Protection}; use std::collections::hash_map::DefaultHasher; use std::fs; +use std::io; +use std::io::Read; use std::sync::{Mutex, Arc}; use std::hash::Hasher; use std::path::{Path, PathBuf}; @@ -169,6 +175,7 @@ impl<'a> BackendOp for MaildirOp { /// Maildir backend https://cr.yp.to/proto/maildir.html #[derive(Debug)] pub struct MaildirType { + name: String, folders: Vec, hash_index: Arc>>, @@ -278,6 +285,7 @@ impl MaildirType { } folders[0].children = recurse_folders(&mut folders, &path); MaildirType { + name: f.name().to_string(), folders, hash_index: Arc::new(Mutex::new(FnvHashMap::with_capacity_and_hasher(0, Default::default()))), path: f.root_folder().to_string(), @@ -294,6 +302,7 @@ impl MaildirType { pub fn multicore(&mut self, cores: usize, folder: &Folder) -> Async>> { let mut w = AsyncBuilder::new(); + let cache_dir = xdg::BaseDirectories::with_profile("meli", &self.name).unwrap(); let handle = { let tx = w.tx(); // TODO: Avoid clone @@ -306,6 +315,7 @@ impl MaildirType { thread::Builder::new() .name(name) .spawn(move || { + let cache_dir = cache_dir.clone(); let mut path = PathBuf::from(path); path.push("cur"); let iter = path.read_dir()?; @@ -322,12 +332,14 @@ impl MaildirType { let mut threads = Vec::with_capacity(cores); if !files.is_empty() { crossbeam::scope(|scope| { + let cache_dir = cache_dir.clone(); let chunk_size = if count / cores > 0 { count / cores } else { count }; for chunk in files.chunks(chunk_size) { + let cache_dir = cache_dir.clone(); let mut tx = tx.clone(); let map = map.clone(); let s = scope.spawn(move || { @@ -338,6 +350,24 @@ impl MaildirType { let map = map.clone(); let len = c.len(); for file in c { + let ri = file.rfind("/").unwrap() + 1; + let file_name = &file[ri..]; + if let Some(cached) = cache_dir.find_cache_file(file_name) { + // TODO:: error checking + let reader = io::BufReader::new(fs::File::open(cached).unwrap()); + let env: Envelope = bincode::deserialize_from(reader).unwrap(); + { + let mut map = map.lock().unwrap(); + let hash = env.hash(); + if (*map).contains_key(&hash) { + continue; + } + (*map).insert(hash, (0, file.to_string())); + local_r.push(env); + continue; + } + + } let e_copy = file.to_string(); /* * get hash @@ -350,10 +380,11 @@ impl MaildirType { { let mut hasher = DefaultHasher::new(); let hash = { - let slice = Mmap::open_path(&e_copy, Protection::Read).unwrap(); + let mut buf = Vec::new(); + let mut f = fs::File::open(&e_copy).unwrap(); + f.read_to_end(&mut buf); /* Unwrap is safe since we use ? above. */ - hasher.write( - unsafe { slice.as_slice() }); + hasher.write(&buf); hasher.finish() }; { @@ -366,7 +397,19 @@ impl MaildirType { // TODO: Check cache let op = Box::new(MaildirOp::new(hash, map.clone())); if let Some(mut e) = Envelope::from_token(op, hash) { + if let Ok(cached) = cache_dir.place_cache_file(file_name) { + let f = match fs::File::create(cached) { + Ok(f) => f, + Err(e) => { + panic!("{}",e); + } + }; + let writer = io::BufWriter::new(f); + bincode::serialize_into(writer, &e).unwrap(); + } local_r.push(e); + + } else { continue; } diff --git a/melib/src/mailbox/backends/mod.rs b/melib/src/mailbox/backends/mod.rs index 189bc304..65ad69e1 100644 --- a/melib/src/mailbox/backends/mod.rs +++ b/melib/src/mailbox/backends/mod.rs @@ -64,6 +64,7 @@ impl Backends { } self.map[key]() } + pub fn register(&mut self, key: String, backend: Box BackendCreator>) -> () { if self.map.contains_key(&key) { panic!("{} is an already registered backend", key); diff --git a/melib/src/mailbox/email/attachment_types.rs b/melib/src/mailbox/email/attachment_types.rs index 8d9fb39f..1ce91fff 100644 --- a/melib/src/mailbox/email/attachment_types.rs +++ b/melib/src/mailbox/email/attachment_types.rs @@ -2,7 +2,7 @@ use mailbox::email::parser::BytesExt; use std::fmt::{Display, Formatter, Result as FmtResult}; use std::str; -#[derive(Clone, Copy, Debug, PartialEq, Serialize)] +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] pub enum Charset { Ascii, UTF8, @@ -52,7 +52,7 @@ impl<'a> From<&'a [u8]> for Charset { } } -#[derive(Clone, Debug, PartialEq, Serialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub enum MultipartType { Mixed, Alternative, @@ -73,7 +73,7 @@ impl Display for MultipartType { } } -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub enum ContentType { Text { charset: Charset }, Multipart { boundary: Vec }, @@ -108,7 +108,7 @@ impl ContentType { } } -#[derive(Clone, Debug, PartialEq, Serialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub enum ContentSubType { Plain, Html, @@ -134,7 +134,7 @@ impl Display for ContentSubType { } } } -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub enum ContentTransferEncoding { _8Bit, _7Bit, diff --git a/melib/src/mailbox/email/attachments.rs b/melib/src/mailbox/email/attachments.rs index 163a09e3..b065a16c 100644 --- a/melib/src/mailbox/email/attachments.rs +++ b/melib/src/mailbox/email/attachments.rs @@ -25,7 +25,7 @@ use std::str; pub use mailbox::email::attachment_types::*; -#[derive(Clone, Serialize)] +#[derive(Clone, Serialize, Deserialize)] pub enum AttachmentType { Data { tag: Vec, @@ -71,7 +71,7 @@ pub struct AttachmentBuilder { raw: Vec, } -#[derive(Clone, Serialize)] +#[derive(Clone, Serialize, Deserialize)] pub struct Attachment { content_type: (ContentType, ContentSubType), content_transfer_encoding: ContentTransferEncoding, diff --git a/melib/src/mailbox/email/mod.rs b/melib/src/mailbox/email/mod.rs index d148fa27..ab91ee07 100644 --- a/melib/src/mailbox/email/mod.rs +++ b/melib/src/mailbox/email/mod.rs @@ -42,21 +42,21 @@ use std::string::String; use chrono; use chrono::TimeZone; -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct GroupAddress { raw: Vec, display_name: StrBuilder, mailbox_list: Vec
, } -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct MailboxAddress { raw: Vec, display_name: StrBuilder, address_spec: StrBuilder, } -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub enum Address { Mailbox(MailboxAddress), Group(GroupAddress), @@ -108,7 +108,7 @@ impl fmt::Display for Address { } /// Helper struct to return slices from a struct field on demand. -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] struct StrBuilder { offset: usize, length: usize, @@ -133,7 +133,7 @@ impl StrBuilder { } /// `MessageID` is accessed through the `StrBuild` trait. -#[derive(Clone, Serialize)] +#[derive(Clone, Serialize, Deserialize)] pub struct MessageID(Vec, StrBuilder); impl StrBuild for MessageID { @@ -184,14 +184,14 @@ impl fmt::Debug for MessageID { } } -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] struct References { raw: Vec, refs: Vec, } bitflags! { - #[derive(Default, Serialize)] + #[derive(Default, Serialize, Deserialize)] pub struct Flag: u8 { const PASSED = 0b00000001; const REPLIED = 0b00000010; @@ -236,7 +236,7 @@ impl EnvelopeBuilder { /// Access to the underlying email object in the account's backend (for example the file or the /// entry in an IMAP server) is given through `operation_token`. For more information see /// `BackendOp`. -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Envelope { date: String, from: Vec
, diff --git a/ui/src/components/mail/listing/mod.rs b/ui/src/components/mail/listing/mod.rs index 21863ac6..611de7ab 100644 --- a/ui/src/components/mail/listing/mod.rs +++ b/ui/src/components/mail/listing/mod.rs @@ -21,7 +21,7 @@ use super::*; -use melib::mailbox::backends::BackendOp; +//use melib::mailbox::backends::BackendOp; mod compact; pub use self::compact::*; @@ -220,7 +220,7 @@ impl MailListing { container, &indentations, len, - context.accounts[self.cursor_pos.0].backend.operation(envelope.hash()) + // context.accounts[self.cursor_pos.0].backend.operation(envelope.hash()) ), &mut content, fg_color, @@ -421,7 +421,7 @@ impl MailListing { container: &Container, indentations: &[bool], idx_width: usize, - op: Box, + //op: Box, ) -> String { let has_sibling = container.has_sibling(); let has_parent = container.has_parent(); @@ -458,10 +458,13 @@ impl MailListing { if show_subject { s.push_str(&format!("{:.85}", envelope.subject())); } + /* + * Very slow since we have to build all attachments let attach_count = envelope.body(op).count_attachments(); if attach_count > 1 { s.push_str(&format!(" {}∞ ", attach_count - 1)); } + */ s } fn format_date(envelope: &Envelope) -> String { diff --git a/ui/src/state.rs b/ui/src/state.rs index bd4aebbc..c6f85c64 100644 --- a/ui/src/state.rs +++ b/ui/src/state.rs @@ -80,7 +80,7 @@ pub struct State { startup_thread: Option>, - threads: FnvHashMap>, + threads: FnvHashMap, thread::JoinHandle<()>)>, } impl Drop for State { @@ -131,12 +131,14 @@ impl State { default => {}, startup_rx.recv() -> _ => { sender.send(ThreadEvent::ThreadJoin(thread::current().id())); - return; + break; } } sender.send(ThreadEvent::UIEvent(UIEventType::StartupCheck)); thread::sleep(dur); } + startup_rx.recv(); + return; }) .unwrap() }; @@ -162,11 +164,11 @@ impl State { input_thread, }, - startup_thread: Some(startup_tx), + startup_thread: Some(startup_tx.clone()), threads: FnvHashMap::with_capacity_and_hasher(1, Default::default()), }; s.threads - .insert(startup_thread.thread().id(), startup_thread); + .insert(startup_thread.thread().id(), (startup_tx.clone(), startup_thread)); write!( s.stdout(), "{}{}{}", @@ -218,15 +220,16 @@ impl State { }) .unwrap() }; - self.startup_thread = Some(startup_tx); + self.startup_thread = Some(startup_tx.clone()); self.threads - .insert(startup_thread.thread().id(), startup_thread); + .insert(startup_thread.thread().id(), (startup_tx, startup_thread)); } /// If an owned thread returns a `ThreadEvent::ThreadJoin` event to `State` then it must remove /// the thread from its list and `join` it. pub fn join(&mut self, id: thread::ThreadId) { - let handle = self.threads.remove(&id).unwrap(); + let (tx, handle) = self.threads.remove(&id).unwrap(); + tx.send(true); handle.join().unwrap(); }