imap: get() ENVELOPE instead of RFC822.HEADER; it's faster

embed
Manos Pitsidianakis 2019-09-08 10:27:10 +03:00
parent 335a1011de
commit d1d11356db
Signed by: Manos Pitsidianakis
GPG Key ID: 73627C2F690DF710
3 changed files with 267 additions and 48 deletions

View File

@ -45,6 +45,8 @@ use crate::email::*;
use crate::error::{MeliError, Result};
use fnv::{FnvHashMap, FnvHashSet};
use native_tls::TlsConnector;
use std::collections::hash_map::DefaultHasher;
use std::hash::Hasher;
use std::iter::FromIterator;
use std::net::SocketAddr;
use std::str::FromStr;
@ -116,7 +118,7 @@ impl MailBackend for ImapType {
while exists > 1 {
let mut envelopes = vec![];
exit_on_error!(&tx,
conn.send_command(format!("UID FETCH {}:{} (FLAGS RFC822.HEADER)", std::cmp::max(exists.saturating_sub(20000), 1), exists).as_bytes())
conn.send_command(format!("UID FETCH {}:{} (FLAGS ENVELOPE)", std::cmp::max(exists.saturating_sub(20000), 1), exists).as_bytes())
conn.read_response(&mut response)
);
debug!(
@ -124,21 +126,23 @@ impl MailBackend for ImapType {
response.len(),
response.lines().collect::<Vec<&str>>().len()
);
match protocol_parser::uid_fetch_response(response.as_bytes())
match protocol_parser::uid_fetch_envelopes_response(response.as_bytes())
.to_full_result()
.map_err(MeliError::from)
{
Ok(v) => {
debug!("responses len is {}", v.len());
for (uid, flags, b) in v {
if let Ok(e) = Envelope::from_bytes(&b, flags) {
hash_index
.lock()
.unwrap()
.insert(e.hash(), (uid, folder_hash));
uid_index.lock().unwrap().insert(uid, e.hash());
envelopes.push(e);
}
for (uid, flags, mut env) in v {
let mut h = DefaultHasher::new();
h.write_usize(uid);
h.write(folder_path.as_bytes());
env.set_hash(h.finish());
hash_index
.lock()
.unwrap()
.insert(env.hash(), (uid, folder_hash));
uid_index.lock().unwrap().insert(uid, env.hash());
envelopes.push(env);
}
}
Err(e) => {
@ -496,17 +500,12 @@ impl ImapType {
exit_on_error!(s,
conn.send_command( format!( "LOGIN \"{}\" \"{}\"", get_conf_val!(s["server_username"]), get_conf_val!(s["server_password"])).as_bytes())
conn.read_lines(&mut res, String::new())
std::io::stderr().write(res.as_bytes())
);
m.capabilities = match protocol_parser::capabilities(res.as_bytes())
.to_full_result()
.map_err(MeliError::from)
{
Ok(c) => {
eprintln!("cap len {}", c.len());
FnvHashSet::from_iter(c.into_iter().map(|s| s.to_vec()))
}
Ok(c) => FnvHashSet::from_iter(c.into_iter().map(|s| s.to_vec())),
Err(e) => {
eprintln!(
"Could not login in account `{}`: {}",

View File

@ -4,6 +4,10 @@ use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use std::str::FromStr;
macro_rules! to_str (
($v:expr) => (unsafe{ std::str::from_utf8_unchecked($v) })
);
macro_rules! dbg_dmp (
($i: expr, $submac:ident!( $($args:tt)* )) => (
{
@ -371,9 +375,18 @@ pub fn flags(input: &str) -> IResult<&str, Flag> {
let mut ret = Flag::default();
let mut input = input;
while input.starts_with("\\") {
input = &input[1..];
let match_end = input.find(|c: char| !c.is_ascii_alphabetic()).or_else(|| input.find(" ")).unwrap_or(input.len());
while !input.starts_with(")") && !input.is_empty() {
if input.starts_with("\\") {
input = &input[1..];
}
let mut match_end = 0;
while match_end < input.len() {
if input[match_end..].starts_with(" ") || input[match_end..].starts_with(")") {
break;
}
match_end += 1;
}
match &input[..match_end] {
"Answered" => {
ret.set(Flag::REPLIED, true);
@ -392,13 +405,10 @@ pub fn flags(input: &str) -> IResult<&str, Flag> {
}
f => {
debug!("unknown Flag token value: {}", f);
break;
}
}
input = &input[match_end..];
if input.starts_with(" \\") {
input = &input[1..];
}
input = input.trim_start();
}
IResult::Done(input, ret)
}
@ -412,3 +422,205 @@ pub fn byte_flags(input: &[u8]) -> IResult<&[u8], Flag> {
}
}
/*
* The fields of the envelope structure are in the following
* order: date, subject, from, sender, reply-to, to, cc, bcc,
* in-reply-to, and message-id. The date, subject, in-reply-to,
* and message-id fields are strings. The from, sender, reply-to,
* to, cc, and bcc fields are parenthesized lists of address
* structures.
* An address structure is a parenthesized list that describes an
* electronic mail address. The fields of an address structure
* are in the following order: personal name, [SMTP]
* at-domain-list (source route), mailbox name, and host name.
*/
/*
* * 12 FETCH (FLAGS (\Seen) INTERNALDATE "17-Jul-1996 02:44:25 -0700"
* RFC822.SIZE 4286 ENVELOPE ("Wed, 17 Jul 1996 02:23:25 -0700 (PDT)"
* "IMAP4rev1 WG mtg summary and minutes"
* (("Terry Gray" NIL "gray" "cac.washington.edu"))
* (("Terry Gray" NIL "gray" "cac.washington.edu"))
* (("Terry Gray" NIL "gray" "cac.washington.edu"))
* ((NIL NIL "imap" "cac.washington.edu"))
* ((NIL NIL "minutes" "CNRI.Reston.VA.US")
* ("John Klensin" NIL "KLENSIN" "MIT.EDU")) NIL NIL
* "<B27397-0100000@cac.washington.edu>")
*/
named!(
pub envelope<Envelope>,
do_parse!(
tag!("(")
>> opt!(is_a!("\r\n\t "))
>> date: quoted_or_nil
>> opt!(is_a!("\r\n\t "))
>> subject: quoted_or_nil
>> opt!(is_a!("\r\n\t "))
>> from: envelope_addresses
>> opt!(is_a!("\r\n\t "))
>> sender: envelope_addresses
>> opt!(is_a!("\r\n\t "))
>> reply_to: envelope_addresses
>> opt!(is_a!("\r\n\t "))
>> to: envelope_addresses
>> opt!(is_a!("\r\n\t "))
>> cc: envelope_addresses
>> opt!(is_a!("\r\n\t "))
>> bcc: envelope_addresses
>> opt!(is_a!("\r\n\t "))
>> in_reply_to: quoted_or_nil
>> opt!(is_a!("\r\n\t "))
>> message_id: quoted_or_nil
>> opt!(is_a!("\r\n\t "))
>> tag!(")")
>> ({
let mut env = Envelope::new(0);
if let Some(date) = date {
env.set_date(&date);
if let Some(d) = crate::email::parser::date(env.date_as_str().as_bytes()) {
env.set_datetime(d);
}
}
if let Some(subject) = subject {
env.set_subject(subject.to_vec());
}
if let Some(from) = from {
env.set_from(from);
}
if let Some(to) = to {
env.set_to(to);
}
if let Some(cc) = cc {
env.set_cc(cc);
}
if let Some(bcc) = bcc {
env.set_bcc(bcc);
}
if let Some(in_reply_to) = in_reply_to {
env.set_in_reply_to(&in_reply_to);
env.push_references(&in_reply_to);
}
if let Some(message_id) = message_id {
env.set_message_id(&message_id);
}
env
})
));
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_envelope() {
// FIXME add a proper test
/*
use std::io::Read;
let mut buffer: Vec<u8> = Vec::new();
let _ = std::fs::File::open("/tmp/a").unwrap().read_to_end(&mut buffer);
debug!(envelope(&buffer));
*/
}
}
/* Helper to build StrBuilder for Address structs */
macro_rules! str_builder {
($offset:expr, $length:expr) => {
StrBuilder {
offset: $offset,
length: $length,
}
}
}
// Parse a list of addresses in the format of the ENVELOPE structure
named!(pub envelope_addresses<Option<Vec<Address>>>,
alt_complete!(map!(tag!("NIL"), |_| None) |
do_parse!(
tag!("(")
>> envelopes: many1!(delimited!(ws!(tag!("(")), envelope_address, tag!(")")))
>> tag!(")")
>> ({
Some(envelopes)
})
)
));
// Parse an address in the format of the ENVELOPE structure eg
// ("Terry Gray" NIL "gray" "cac.washington.edu")
named!(
pub envelope_address<Address>,
do_parse!(
name: alt_complete!(quoted | map!(tag!("NIL"), |_| Vec::new()))
>> is_a!("\r\n\t ")
>> alt_complete!(quoted| map!(tag!("NIL"), |_| Vec::new()))
>> is_a!("\r\n\t ")
>> mailbox_name: dbg_dmp!(alt_complete!(quoted | map!(tag!("NIL"), |_| Vec::new())))
>> is_a!("\r\n\t ")
>> host_name: alt_complete!(quoted | map!(tag!("NIL"), |_| Vec::new()))
>> ({
Address::Mailbox(MailboxAddress {
raw: format!("{}{}<{}@{}>", to_str!(&name), if name.is_empty() { "" } else { " " }, to_str!(&mailbox_name), to_str!(&host_name)).into_bytes(),
display_name: str_builder!(0, name.len()),
address_spec: str_builder!(if name.is_empty() { 1 } else { name.len() + 2 }, mailbox_name.len() + host_name.len() + 1),
})
})
));
// Read a literal ie a byte sequence prefixed with a tag containing its length delimited in {}s
named!(pub literal<&[u8]>,length_bytes!(delimited!(tag!("{"), map_res!(digit, |s| { usize::from_str(unsafe { std::str::from_utf8_unchecked(s) }) }), tag!("}\r\n"))));
// Return a byte sequence surrounded by "s and decoded if necessary
pub fn quoted(input: &[u8]) -> IResult<&[u8], Vec<u8>> {
if let IResult::Done(r, o) = literal(input) {
return match crate::email::parser::phrase(o) {
IResult::Done(_, out) => IResult::Done(r, out),
e => e,
};
}
if input.is_empty() || input[0] != b'"' {
return IResult::Error(nom::ErrorKind::Custom(0));
}
let mut i = 1;
while i < input.len() {
if input[i] == b'\"' { //&& input[i - 1] != b'\\' {
return match crate::email::parser::phrase(&input[1..i]) {
IResult::Done(_, out) => IResult::Done(&input[i+1..], out),
e=> e,
};
}
i += 1;
}
return IResult::Error(nom::ErrorKind::Custom(0));
}
named!(
pub quoted_or_nil<Option<Vec<u8>>>,
alt_complete!(map!(ws!(tag!("NIL")), |_| None) | map!(quoted, |v| Some(v))));
named!(
pub uid_fetch_envelopes_response<Vec<(usize, Option<Flag>, Envelope)>>,
many0!(
do_parse!(
tag!("* ")
>> take_while!(call!(is_digit))
>> tag!(" FETCH (")
>> uid_flags: permutation!(preceded!(ws!(tag!("UID ")), map_res!(digit, |s| { usize::from_str(unsafe { std::str::from_utf8_unchecked(s) }) })), opt!(preceded!(ws!(tag!("FLAGS ")), delimited!(tag!("("), byte_flags, tag!(")")))))
>> tag!(" ENVELOPE ")
>> env: ws!(envelope)
>> tag!(")\r\n")
>> ((uid_flags.0, uid_flags.1, env))
)
)
);

View File

@ -52,16 +52,16 @@ use chrono::TimeZone;
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct GroupAddress {
raw: Vec<u8>,
display_name: StrBuilder,
mailbox_list: Vec<Address>,
pub raw: Vec<u8>,
pub display_name: StrBuilder,
pub mailbox_list: Vec<Address>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct MailboxAddress {
raw: Vec<u8>,
display_name: StrBuilder,
address_spec: StrBuilder,
pub raw: Vec<u8>,
pub display_name: StrBuilder,
pub address_spec: StrBuilder,
}
#[derive(Clone, Serialize, Deserialize)]
@ -149,9 +149,9 @@ impl fmt::Debug for Address {
/// Helper struct to return slices from a struct field on demand.
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
struct StrBuilder {
offset: usize,
length: usize,
pub struct StrBuilder {
pub offset: usize,
pub length: usize,
}
/// Structs implementing this trait must contain a `StrBuilder` field.
@ -177,7 +177,7 @@ impl StrBuilder {
/// `MessageID` is accessed through the `StrBuild` trait.
#[derive(Clone, Serialize, Deserialize, Default)]
pub struct MessageID(Vec<u8>, StrBuilder);
pub struct MessageID(pub Vec<u8>, pub StrBuilder);
impl StrBuild for MessageID {
fn new(string: &[u8], slice: &[u8]) -> Self {
@ -334,12 +334,13 @@ pub struct Envelope {
impl fmt::Debug for Envelope {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "Envelope {{\ndate: {}\n,from:{:#?}\nto {:#?}\nmessage_id: {},\n references: {:#?}\nhash: {}\n
}}",
write!(f, "Envelope {{\n\tsubject: {}\n\tdate: {},\n\tfrom:{:#?},\n\tto {:#?},\n\tmessage_id: {},\n\tin_reply_to: {:?}\n\treferences: {:#?},\n\thash: {}\n}}",
self.subject(),
self.date,
self.from,
self.to,
self.message_id_display(),
self.in_reply_to_display(),
self.references,
self.hash)
}
@ -667,22 +668,22 @@ impl Envelope {
pub fn message_id_raw(&self) -> Cow<str> {
String::from_utf8_lossy(self.message_id.raw())
}
fn set_date(&mut self, new_val: &[u8]) {
pub fn set_date(&mut self, new_val: &[u8]) {
self.date = String::from_utf8_lossy(new_val).into_owned();
}
fn set_bcc(&mut self, new_val: Vec<Address>) {
pub fn set_bcc(&mut self, new_val: Vec<Address>) {
self.bcc = new_val;
}
fn set_cc(&mut self, new_val: Vec<Address>) {
pub fn set_cc(&mut self, new_val: Vec<Address>) {
self.cc = new_val;
}
fn set_from(&mut self, new_val: Vec<Address>) {
pub fn set_from(&mut self, new_val: Vec<Address>) {
self.from = new_val;
}
fn set_to(&mut self, new_val: Vec<Address>) {
pub fn set_to(&mut self, new_val: Vec<Address>) {
self.to = new_val;
}
fn set_in_reply_to(&mut self, new_val: &[u8]) {
pub fn set_in_reply_to(&mut self, new_val: &[u8]) {
let slice = match parser::message_id(new_val).to_full_result() {
Ok(v) => v,
Err(_) => {
@ -692,22 +693,24 @@ impl Envelope {
};
self.in_reply_to = Some(MessageID::new(new_val, slice));
}
fn set_subject(&mut self, new_val: Vec<u8>) {
pub fn set_subject(&mut self, new_val: Vec<u8>) {
self.subject = Some(new_val);
}
fn set_message_id(&mut self, new_val: &[u8]) {
pub fn set_message_id(&mut self, new_val: &[u8]) {
let slice = match parser::message_id(new_val).to_full_result() {
Ok(v) => v,
Err(_) => {
Err(e) => {
debug!(e);
return;
}
};
self.message_id = MessageID::new(new_val, slice);
}
fn push_references(&mut self, new_val: &[u8]) {
pub fn push_references(&mut self, new_val: &[u8]) {
let slice = match parser::message_id(new_val).to_full_result() {
Ok(v) => v,
Err(_) => {
Err(e) => {
debug!(e);
return;
}
};
@ -736,7 +739,7 @@ impl Envelope {
}
}
}
fn set_references(&mut self, new_val: &[u8]) {
pub fn set_references(&mut self, new_val: &[u8]) {
match self.references {
Some(ref mut s) => {
s.raw = new_val.into();
@ -764,6 +767,11 @@ impl Envelope {
pub fn other_headers(&self) -> &FnvHashMap<String, String> {
&self.other_headers
}
pub fn other_headers_mut(&mut self) -> &mut FnvHashMap<String, String> {
&mut self.other_headers
}
pub fn thread(&self) -> ThreadHash {
self.thread
}