Add Envelope parsing caching

Concerns #28

melib/Cargo.toml

@@ -17,7 +17,8 @@ memmap = "0.5.2"
 nom = "3.2.0"
 notify = "4.0.1"
 notify-rust = "^3"
-serde = "^1.0.8"
-serde_derive = "^1.0.8"
 termion = "1.5.1"
 xdg = "2.1.0"
+serde = "1.0.71"
+serde_derive = "1.0.71"
+bincode = "1.0.1"

melib/src/mailbox/backends/maildir.rs

@@ -19,6 +19,10 @@
  * along with meli. If not, see <http://www.gnu.org/licenses/>.
  */
 
+extern crate xdg;
+extern crate serde_derive;
+extern crate bincode;
+
 use async::*;
 use conf::AccountSettings;
 use error::{MeliError, Result};
@@ -43,6 +47,8 @@ extern crate crossbeam;
 use memmap::{Mmap, Protection};
 use std::collections::hash_map::DefaultHasher;
 use std::fs;
+use std::io;
+use std::io::Read;
 use std::sync::{Mutex, Arc};
 use std::hash::Hasher;
 use std::path::{Path, PathBuf};
@@ -169,6 +175,7 @@ impl<'a> BackendOp for MaildirOp {
 /// Maildir backend https://cr.yp.to/proto/maildir.html
 #[derive(Debug)]
 pub struct MaildirType {
+    name: String,
     folders: Vec<MaildirFolder>,
     hash_index: Arc<Mutex<FnvHashMap<u64, (usize, String)>>>,
 
@@ -278,6 +285,7 @@ impl MaildirType {
         }
         folders[0].children = recurse_folders(&mut folders, &path);
         MaildirType {
+            name: f.name().to_string(),
             folders,
             hash_index: Arc::new(Mutex::new(FnvHashMap::with_capacity_and_hasher(0, Default::default()))),
             path: f.root_folder().to_string(),
@@ -294,6 +302,7 @@ impl MaildirType {
 
     pub fn multicore(&mut self, cores: usize, folder: &Folder) -> Async<Result<Vec<Envelope>>> {
         let mut w = AsyncBuilder::new();
+        let cache_dir = xdg::BaseDirectories::with_profile("meli", &self.name).unwrap();
         let handle = {
             let tx = w.tx();
             // TODO: Avoid clone
@@ -306,6 +315,7 @@ impl MaildirType {
             thread::Builder::new()
                 .name(name)
                 .spawn(move || {
+                    let cache_dir = cache_dir.clone();
                     let mut path = PathBuf::from(path);
                     path.push("cur");
                     let iter = path.read_dir()?;
@@ -322,12 +332,14 @@ impl MaildirType {
                     let mut threads = Vec::with_capacity(cores);
                     if !files.is_empty() {
                         crossbeam::scope(|scope| {
+                            let cache_dir = cache_dir.clone();
                             let chunk_size = if count / cores > 0 {
                                 count / cores
                             } else {
                                 count
                             };
                             for chunk in files.chunks(chunk_size) {
+                                let cache_dir = cache_dir.clone();
                                 let mut tx = tx.clone();
                                 let map = map.clone();
                                 let s = scope.spawn(move || {
@@ -338,6 +350,24 @@ impl MaildirType {
                                         let map = map.clone();
                                         let len = c.len();
                                         for file in c {
+                                            let ri = file.rfind("/").unwrap() + 1;
+                                            let file_name = &file[ri..];
+                                            if let Some(cached) = cache_dir.find_cache_file(file_name) {
+                                                // TODO:: error checking
+                                                let reader = io::BufReader::new(fs::File::open(cached).unwrap());
+                                                let env: Envelope = bincode::deserialize_from(reader).unwrap();
+                                                    {
+                                                        let mut map = map.lock().unwrap();
+                                                        let hash = env.hash();
+                                                        if (*map).contains_key(&hash) {
+                                                            continue;
+                                                        }
+                                                        (*map).insert(hash, (0, file.to_string()));
+                                                        local_r.push(env);
+                                                        continue;
+                                                    }
+
+                                            }
                                             let e_copy = file.to_string();
                                             /*
                                              * get hash
@@ -350,10 +380,11 @@ impl MaildirType {
                                             {
                                                 let mut hasher = DefaultHasher::new();
                                                     let hash = {
-                                                        let slice = Mmap::open_path(&e_copy, Protection::Read).unwrap();
+                                                        let mut buf = Vec::new();
+                                                        let mut f = fs::File::open(&e_copy).unwrap();
+                                                        f.read_to_end(&mut buf);
                                                         /* Unwrap is safe since we use ? above. */
-                                                        hasher.write(
+                                                        hasher.write(&buf);
-                                                            unsafe { slice.as_slice() });
                                                         hasher.finish()
                                                     };
                                                     {
@@ -366,7 +397,19 @@ impl MaildirType {
                                                     // TODO: Check cache
                                                     let op = Box::new(MaildirOp::new(hash, map.clone()));
                                                     if let Some(mut e) = Envelope::from_token(op, hash) {
+                                            if let Ok(cached) = cache_dir.place_cache_file(file_name) {
+                                                let f = match fs::File::create(cached) {
+                                                    Ok(f) => f,
+                                                    Err(e) => {
+                                                        panic!("{}",e);
+                                                    }
+                                                };
+                                                let writer = io::BufWriter::new(f);
+                                                bincode::serialize_into(writer, &e).unwrap();
+                                            }
                                                         local_r.push(e);
+
+
                                                     } else {
                                                         continue;
                                                     }

melib/src/mailbox/backends/mod.rs

@@ -64,6 +64,7 @@ impl Backends {
         }
         self.map[key]()
     }
+
     pub fn register(&mut self, key: String, backend: Box<Fn() -> BackendCreator>) -> () {
         if self.map.contains_key(&key) {
             panic!("{} is an already registered backend", key);

melib/src/mailbox/email/attachment_types.rs

@@ -2,7 +2,7 @@ use mailbox::email::parser::BytesExt;
 use std::fmt::{Display, Formatter, Result as FmtResult};
 use std::str;
 
-#[derive(Clone, Copy, Debug, PartialEq, Serialize)]
+#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
 pub enum Charset {
     Ascii,
     UTF8,
@@ -52,7 +52,7 @@ impl<'a> From<&'a [u8]> for Charset {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Serialize)]
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub enum MultipartType {
     Mixed,
     Alternative,
@@ -73,7 +73,7 @@ impl Display for MultipartType {
     }
 }
 
-#[derive(Clone, Debug, Serialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub enum ContentType {
     Text { charset: Charset },
     Multipart { boundary: Vec<u8> },
@@ -108,7 +108,7 @@ impl ContentType {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Serialize)]
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub enum ContentSubType {
     Plain,
     Html,
@@ -134,7 +134,7 @@ impl Display for ContentSubType {
         }
     }
 }
-#[derive(Clone, Debug, Serialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub enum ContentTransferEncoding {
     _8Bit,
     _7Bit,

melib/src/mailbox/email/attachments.rs

@@ -25,7 +25,7 @@ use std::str;
 
 pub use mailbox::email::attachment_types::*;
 
-#[derive(Clone, Serialize)]
+#[derive(Clone, Serialize, Deserialize)]
 pub enum AttachmentType {
     Data {
         tag: Vec<u8>,
@@ -71,7 +71,7 @@ pub struct AttachmentBuilder {
     raw: Vec<u8>,
 }
 
-#[derive(Clone, Serialize)]
+#[derive(Clone, Serialize, Deserialize)]
 pub struct Attachment {
     content_type: (ContentType, ContentSubType),
     content_transfer_encoding: ContentTransferEncoding,

melib/src/mailbox/email/mod.rs

@@ -42,21 +42,21 @@ use std::string::String;
 use chrono;
 use chrono::TimeZone;
 
-#[derive(Clone, Debug, Serialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct GroupAddress {
     raw: Vec<u8>,
     display_name: StrBuilder,
     mailbox_list: Vec<Address>,
 }
 
-#[derive(Clone, Debug, Serialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct MailboxAddress {
     raw: Vec<u8>,
     display_name: StrBuilder,
     address_spec: StrBuilder,
 }
 
-#[derive(Clone, Debug, Serialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub enum Address {
     Mailbox(MailboxAddress),
     Group(GroupAddress),
@@ -108,7 +108,7 @@ impl fmt::Display for Address {
 }
 
 /// Helper struct to return slices from a struct field on demand.
-#[derive(Clone, Debug, Serialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 struct StrBuilder {
     offset: usize,
     length: usize,
@@ -133,7 +133,7 @@ impl StrBuilder {
 }
 
 /// `MessageID` is accessed through the `StrBuild` trait.
-#[derive(Clone, Serialize)]
+#[derive(Clone, Serialize, Deserialize)]
 pub struct MessageID(Vec<u8>, StrBuilder);
 
 impl StrBuild for MessageID {
@@ -184,14 +184,14 @@ impl fmt::Debug for MessageID {
     }
 }
 
-#[derive(Clone, Debug, Serialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 struct References {
     raw: Vec<u8>,
     refs: Vec<MessageID>,
 }
 
 bitflags! {
-    #[derive(Default, Serialize)]
+    #[derive(Default, Serialize, Deserialize)]
     pub struct Flag: u8 {
         const PASSED  =  0b00000001;
         const REPLIED =  0b00000010;
@@ -236,7 +236,7 @@ impl EnvelopeBuilder {
 ///  Access to the underlying email object in the account's backend (for example the file or the
 ///  entry in an IMAP server) is given through `operation_token`. For more information see
 ///  `BackendOp`.
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Envelope {
     date: String,
     from: Vec<Address>,

ui/src/components/mail/listing/mod.rs

@@ -21,7 +21,7 @@
 
 use super::*;
 
-use melib::mailbox::backends::BackendOp;
+//use melib::mailbox::backends::BackendOp;
 mod compact;
 pub use self::compact::*;
 
@@ -220,7 +220,7 @@ impl MailListing {
                         container,
                         &indentations,
                         len,
-                        context.accounts[self.cursor_pos.0].backend.operation(envelope.hash())
+                    //    context.accounts[self.cursor_pos.0].backend.operation(envelope.hash())
                     ),
                     &mut content,
                     fg_color,
@@ -421,7 +421,7 @@ impl MailListing {
         container: &Container,
         indentations: &[bool],
         idx_width: usize,
-        op: Box<BackendOp>,
+        //op: Box<BackendOp>,
     ) -> String {
         let has_sibling = container.has_sibling();
         let has_parent = container.has_parent();
@@ -458,10 +458,13 @@ impl MailListing {
         if show_subject {
             s.push_str(&format!("{:.85}", envelope.subject()));
         }
+        /*
+         * Very slow since we have to build all attachments
         let attach_count = envelope.body(op).count_attachments();
         if attach_count > 1 {
             s.push_str(&format!(" {}∞ ", attach_count - 1));
         }
+         */
         s
     }
     fn format_date(envelope: &Envelope) -> String {

ui/src/state.rs

@@ -80,7 +80,7 @@ pub struct State<W: Write> {
 
     startup_thread: Option<chan::Sender<bool>>,
 
-    threads: FnvHashMap<thread::ThreadId, thread::JoinHandle<()>>,
+    threads: FnvHashMap<thread::ThreadId, (chan::Sender<bool>, thread::JoinHandle<()>)>,
 }
 
 impl<W: Write> Drop for State<W> {
@@ -131,12 +131,14 @@ impl State<std::io::Stdout> {
                             default => {},
                             startup_rx.recv() -> _ => {
                                 sender.send(ThreadEvent::ThreadJoin(thread::current().id()));
-                                return;
+                                break;
                             }
                         }
                         sender.send(ThreadEvent::UIEvent(UIEventType::StartupCheck));
                         thread::sleep(dur);
                     }
+                    startup_rx.recv();
+                    return;
                 })
                 .unwrap()
         };
@@ -162,11 +164,11 @@ impl State<std::io::Stdout> {
 
                 input_thread,
             },
-            startup_thread: Some(startup_tx),
+            startup_thread: Some(startup_tx.clone()),
             threads: FnvHashMap::with_capacity_and_hasher(1, Default::default()),
         };
         s.threads
-            .insert(startup_thread.thread().id(), startup_thread);
+            .insert(startup_thread.thread().id(), (startup_tx.clone(), startup_thread));
         write!(
             s.stdout(),
             "{}{}{}",
@@ -218,15 +220,16 @@ impl State<std::io::Stdout> {
                 })
                 .unwrap()
         };
-        self.startup_thread = Some(startup_tx);
+        self.startup_thread = Some(startup_tx.clone());
         self.threads
-            .insert(startup_thread.thread().id(), startup_thread);
+            .insert(startup_thread.thread().id(), (startup_tx, startup_thread));
     }
 
     /// If an owned thread returns a `ThreadEvent::ThreadJoin` event to `State` then it must remove
     /// the thread from its list and `join` it.
     pub fn join(&mut self, id: thread::ThreadId) {
-        let handle = self.threads.remove(&id).unwrap();
+        let (tx, handle) = self.threads.remove(&id).unwrap();
+        tx.send(true);
         handle.join().unwrap();
     }