llange
/
meli
forked from meli/meli
1
Fork 0
Code Issues Pull Requests 1 Projects Releases Wiki Activity

Add Envelope parsing caching 

Concerns #28
embed
Manos Pitsidianakis 2018-08-13 09:25:48 +03:00
parent 5889494e9e
commit 5d0b7fa903
Signed by: Manos Pitsidianakis
GPG Key ID: 73627C2F690DF710
8 changed files with 81 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
melib/Cargo.toml
View File

@ -17,7 +17,8 @@ memmap = "0.5.2"
nom = "3.2.0"
notify = "4.0.1"
notify-rust = "^3"
serde = "^1.0.8"
serde_derive = "^1.0.8"
termion = "1.5.1"
xdg = "2.1.0"
serde = "1.0.71"
serde_derive = "1.0.71"
bincode = "1.0.1"

49
melib/src/mailbox/backends/maildir.rs
View File

@ -19,6 +19,10 @@
* along with meli. If not, see <http://www.gnu.org/licenses/>.
*/
extern crate xdg;
extern crate serde_derive;
extern crate bincode;
use async::*;
use conf::AccountSettings;
use error::{MeliError, Result};
@ -43,6 +47,8 @@ extern crate crossbeam;
use memmap::{Mmap, Protection};
use std::collections::hash_map::DefaultHasher;
use std::fs;
use std::io;
use std::io::Read;
use std::sync::{Mutex, Arc};
use std::hash::Hasher;
use std::path::{Path, PathBuf};
@ -169,6 +175,7 @@ impl<'a> BackendOp for MaildirOp {
/// Maildir backend https://cr.yp.to/proto/maildir.html
#[derive(Debug)]
pub struct MaildirType {
name: String,
folders: Vec<MaildirFolder>,
hash_index: Arc<Mutex<FnvHashMap<u64, (usize, String)>>>,
@ -278,6 +285,7 @@ impl MaildirType {
}
folders[0].children = recurse_folders(&mut folders, &path);
MaildirType {
name: f.name().to_string(),
folders,
hash_index: Arc::new(Mutex::new(FnvHashMap::with_capacity_and_hasher(0, Default::default()))),
path: f.root_folder().to_string(),
@ -294,6 +302,7 @@ impl MaildirType {
pub fn multicore(&mut self, cores: usize, folder: &Folder) -> Async<Result<Vec<Envelope>>> {
let mut w = AsyncBuilder::new();
let cache_dir = xdg::BaseDirectories::with_profile("meli", &self.name).unwrap();
let handle = {
let tx = w.tx();
// TODO: Avoid clone
@ -306,6 +315,7 @@ impl MaildirType {
thread::Builder::new()
.name(name)
.spawn(move || {
let cache_dir = cache_dir.clone();
let mut path = PathBuf::from(path);
path.push("cur");
let iter = path.read_dir()?;
@ -322,12 +332,14 @@ impl MaildirType {
let mut threads = Vec::with_capacity(cores);
if !files.is_empty() {
crossbeam::scope(|scope| {
let cache_dir = cache_dir.clone();
let chunk_size = if count / cores > 0 {
count / cores
} else {
count
};
for chunk in files.chunks(chunk_size) {
let cache_dir = cache_dir.clone();
let mut tx = tx.clone();
let map = map.clone();
let s = scope.spawn(move || {
@ -338,6 +350,24 @@ impl MaildirType {
let map = map.clone();
let len = c.len();
for file in c {
let ri = file.rfind("/").unwrap() + 1;
let file_name = &file[ri..];
if let Some(cached) = cache_dir.find_cache_file(file_name) {
// TODO:: error checking
let reader = io::BufReader::new(fs::File::open(cached).unwrap());
let env: Envelope = bincode::deserialize_from(reader).unwrap();
{
let mut map = map.lock().unwrap();
let hash = env.hash();
if (*map).contains_key(&hash) {
continue;
}
(*map).insert(hash, (0, file.to_string()));
local_r.push(env);
continue;
}
}
let e_copy = file.to_string();
/*
* get hash
@ -350,10 +380,11 @@ impl MaildirType {
{
let mut hasher = DefaultHasher::new();
let hash = {
let slice = Mmap::open_path(&e_copy, Protection::Read).unwrap();
let mut buf = Vec::new();
let mut f = fs::File::open(&e_copy).unwrap();
f.read_to_end(&mut buf);
/* Unwrap is safe since we use ? above. */
hasher.write(
unsafe { slice.as_slice() });
hasher.write(&buf);
hasher.finish()
};
{
@ -366,7 +397,19 @@ impl MaildirType {
// TODO: Check cache
let op = Box::new(MaildirOp::new(hash, map.clone()));
if let Some(mut e) = Envelope::from_token(op, hash) {
if let Ok(cached) = cache_dir.place_cache_file(file_name) {
let f = match fs::File::create(cached) {
Ok(f) => f,
Err(e) => {
panic!("{}",e);
}
};
let writer = io::BufWriter::new(f);
bincode::serialize_into(writer, &e).unwrap();
}
local_r.push(e);
} else {
continue;
}

1
melib/src/mailbox/backends/mod.rs
View File

@ -64,6 +64,7 @@ impl Backends {
}
self.map[key]()
}
pub fn register(&mut self, key: String, backend: Box<Fn() -> BackendCreator>) -> () {
if self.map.contains_key(&key) {
panic!("{} is an already registered backend", key);

10
melib/src/mailbox/email/attachment_types.rs
View File

@ -2,7 +2,7 @@ use mailbox::email::parser::BytesExt;
use std::fmt::{Display, Formatter, Result as FmtResult};
use std::str;
#[derive(Clone, Copy, Debug, PartialEq, Serialize)]
#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
pub enum Charset {
Ascii,
UTF8,
@ -52,7 +52,7 @@ impl<'a> From<&'a [u8]> for Charset {
}
}
#[derive(Clone, Debug, PartialEq, Serialize)]
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum MultipartType {
Mixed,
Alternative,
@ -73,7 +73,7 @@ impl Display for MultipartType {
}
}
#[derive(Clone, Debug, Serialize)]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum ContentType {
Text { charset: Charset },
Multipart { boundary: Vec<u8> },
@ -108,7 +108,7 @@ impl ContentType {
}
}
#[derive(Clone, Debug, PartialEq, Serialize)]
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum ContentSubType {
Plain,
Html,
@ -134,7 +134,7 @@ impl Display for ContentSubType {
}
}
}
#[derive(Clone, Debug, Serialize)]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum ContentTransferEncoding {
_8Bit,
_7Bit,

4
melib/src/mailbox/email/attachments.rs
View File

@ -25,7 +25,7 @@ use std::str;
pub use mailbox::email::attachment_types::*;
#[derive(Clone, Serialize)]
#[derive(Clone, Serialize, Deserialize)]
pub enum AttachmentType {
Data {
tag: Vec<u8>,
@ -71,7 +71,7 @@ pub struct AttachmentBuilder {
raw: Vec<u8>,
}
#[derive(Clone, Serialize)]
#[derive(Clone, Serialize, Deserialize)]
pub struct Attachment {
content_type: (ContentType, ContentSubType),
content_transfer_encoding: ContentTransferEncoding,

16
melib/src/mailbox/email/mod.rs
View File

@ -42,21 +42,21 @@ use std::string::String;
use chrono;
use chrono::TimeZone;
#[derive(Clone, Debug, Serialize)]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct GroupAddress {
raw: Vec<u8>,
display_name: StrBuilder,
mailbox_list: Vec<Address>,
}
#[derive(Clone, Debug, Serialize)]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct MailboxAddress {
raw: Vec<u8>,
display_name: StrBuilder,
address_spec: StrBuilder,
}
#[derive(Clone, Debug, Serialize)]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum Address {
Mailbox(MailboxAddress),
Group(GroupAddress),
@ -108,7 +108,7 @@ impl fmt::Display for Address {
}
/// Helper struct to return slices from a struct field on demand.
#[derive(Clone, Debug, Serialize)]
#[derive(Clone, Debug, Serialize, Deserialize)]
struct StrBuilder {
offset: usize,
length: usize,
@ -133,7 +133,7 @@ impl StrBuilder {
}
/// `MessageID` is accessed through the `StrBuild` trait.
#[derive(Clone, Serialize)]
#[derive(Clone, Serialize, Deserialize)]
pub struct MessageID(Vec<u8>, StrBuilder);
impl StrBuild for MessageID {
@ -184,14 +184,14 @@ impl fmt::Debug for MessageID {
}
}
#[derive(Clone, Debug, Serialize)]
#[derive(Clone, Debug, Serialize, Deserialize)]
struct References {
raw: Vec<u8>,
refs: Vec<MessageID>,
}
bitflags! {
#[derive(Default, Serialize)]
#[derive(Default, Serialize, Deserialize)]
pub struct Flag: u8 {
const PASSED = 0b00000001;
const REPLIED = 0b00000010;
@ -236,7 +236,7 @@ impl EnvelopeBuilder {
/// Access to the underlying email object in the account's backend (for example the file or the
/// entry in an IMAP server) is given through `operation_token`. For more information see
/// `BackendOp`.
#[derive(Debug, Clone, Serialize)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Envelope {
date: String,
from: Vec<Address>,

9
ui/src/components/mail/listing/mod.rs
View File

@ -21,7 +21,7 @@
use super::*;
use melib::mailbox::backends::BackendOp;
//use melib::mailbox::backends::BackendOp;
mod compact;
pub use self::compact::*;
@ -220,7 +220,7 @@ impl MailListing {
container,
&indentations,
len,
context.accounts[self.cursor_pos.0].backend.operation(envelope.hash())
// context.accounts[self.cursor_pos.0].backend.operation(envelope.hash())
),
&mut content,
fg_color,
@ -421,7 +421,7 @@ impl MailListing {
container: &Container,
indentations: &[bool],
idx_width: usize,
op: Box<BackendOp>,
//op: Box<BackendOp>,
) -> String {
let has_sibling = container.has_sibling();
let has_parent = container.has_parent();
@ -458,10 +458,13 @@ impl MailListing {
if show_subject {
s.push_str(&format!("{:.85}", envelope.subject()));
}
/*
* Very slow since we have to build all attachments
let attach_count = envelope.body(op).count_attachments();
if attach_count > 1 {
s.push_str(&format!(" {}∞ ", attach_count - 1));
}
*/
s
}
fn format_date(envelope: &Envelope) -> String {

17
ui/src/state.rs
View File

@ -80,7 +80,7 @@ pub struct State<W: Write> {
startup_thread: Option<chan::Sender<bool>>,
threads: FnvHashMap<thread::ThreadId, thread::JoinHandle<()>>,
threads: FnvHashMap<thread::ThreadId, (chan::Sender<bool>, thread::JoinHandle<()>)>,
}
impl<W: Write> Drop for State<W> {
@ -131,12 +131,14 @@ impl State<std::io::Stdout> {
default => {},
startup_rx.recv() -> _ => {
sender.send(ThreadEvent::ThreadJoin(thread::current().id()));
return;
break;
}
}
sender.send(ThreadEvent::UIEvent(UIEventType::StartupCheck));
thread::sleep(dur);
}
startup_rx.recv();
return;
})
.unwrap()
};
@ -162,11 +164,11 @@ impl State<std::io::Stdout> {
input_thread,
},
startup_thread: Some(startup_tx),
startup_thread: Some(startup_tx.clone()),
threads: FnvHashMap::with_capacity_and_hasher(1, Default::default()),
};
s.threads
.insert(startup_thread.thread().id(), startup_thread);
.insert(startup_thread.thread().id(), (startup_tx.clone(), startup_thread));
write!(
s.stdout(),
"{}{}{}",
@ -218,15 +220,16 @@ impl State<std::io::Stdout> {
})
.unwrap()
};
self.startup_thread = Some(startup_tx);
self.startup_thread = Some(startup_tx.clone());
self.threads
.insert(startup_thread.thread().id(), startup_thread);
.insert(startup_thread.thread().id(), (startup_tx, startup_thread));
}
/// If an owned thread returns a `ThreadEvent::ThreadJoin` event to `State` then it must remove
/// the thread from its list and `join` it.
pub fn join(&mut self, id: thread::ThreadId) {
let handle = self.threads.remove(&id).unwrap();
let (tx, handle) = self.threads.remove(&id).unwrap();
tx.send(true);
handle.join().unwrap();
}