melib/imap: fetch References header along with ENVELOPE

Threading was broken if information was needed from References header.
For example, mailman might alter some Message-IDs when using its NNTP
bridge and the complete references are necessary to rebuild the thread,
which is only available in References whereas ENVELOPE has only
In-Reply-To.
jmap-eventsource
Manos Pitsidianakis 2020-10-18 17:41:50 +03:00
parent b9f4d718c7
commit 7a9c150f33
Signed by: Manos Pitsidianakis
GPG Key ID: 73627C2F690DF710
5 changed files with 75 additions and 41 deletions

View File

@ -1668,10 +1668,10 @@ async fn fetch_hlpr(state: &mut FetchState) -> Result<Vec<Envelope>> {
let mut envelopes = vec![];
debug!("{} max_uid_left= {}", mailbox_hash, max_uid_left);
let command = if max_uid_left == 1 {
"UID FETCH 1 (UID FLAGS ENVELOPE BODYSTRUCTURE)".to_string()
"UID FETCH 1 (UID FLAGS ENVELOPE BODY.PEEK[HEADER.FIELDS (REFERENCES)] BODYSTRUCTURE)".to_string()
} else {
format!(
"UID FETCH {}:{} (UID FLAGS ENVELOPE BODYSTRUCTURE)",
"UID FETCH {}:{} (UID FLAGS ENVELOPE BODY.PEEK[HEADER.FIELDS (REFERENCES)] BODYSTRUCTURE)",
std::cmp::max(
std::cmp::max(max_uid_left.saturating_sub(chunk_size), 1),
1
@ -1701,6 +1701,7 @@ async fn fetch_hlpr(state: &mut FetchState) -> Result<Vec<Envelope>> {
ref mut envelope,
ref mut flags,
ref raw_fetch_value,
ref references,
..
} in v.iter_mut()
{
@ -1716,6 +1717,21 @@ async fn fetch_hlpr(state: &mut FetchState) -> Result<Vec<Envelope>> {
let uid = uid.unwrap();
let env = envelope.as_mut().unwrap();
env.set_hash(generate_envelope_hash(&mailbox_path, &uid));
if let Some(value) = references {
let parse_result = crate::email::parser::address::msg_id_list(value);
if let Ok((_, value)) = parse_result {
let prev_val = env.references.take();
for v in value {
env.push_references(v);
}
if let Some(prev) = prev_val {
for v in prev.refs {
env.push_references(v);
}
}
}
env.set_references(value);
}
let mut tag_lck = uid_store.tag_index.write().unwrap();
if let Some((flags, keywords)) = flags {
if !flags.intersects(Flag::SEEN) {

View File

@ -140,7 +140,7 @@ mod sqlite3_m {
CREATE INDEX IF NOT EXISTS envelope_idx ON envelopes(hash);
CREATE INDEX IF NOT EXISTS mailbox_idx ON mailbox(mailbox_hash);",
),
version: 1,
version: 2,
};
impl ToSql for ModSequence {
@ -445,6 +445,7 @@ mod sqlite3_m {
modseq,
flags: _,
body: _,
references: _,
envelope: Some(envelope),
raw_fetch_value: _,
} = item

View File

@ -77,40 +77,6 @@ impl ImapConnection {
}
}
pub async fn build_cache(
&mut self,
cache_handle: &mut Box<dyn ImapCache>,
mailbox_hash: MailboxHash,
) -> Result<()> {
debug!("build_cache {}", mailbox_hash);
let mut response = Vec::with_capacity(8 * 1024);
// 1 get uidvalidity, highestmodseq
let select_response = self
.select_mailbox(mailbox_hash, &mut response, true)
.await?
.unwrap();
self.uid_store
.uidvalidity
.lock()
.unwrap()
.insert(mailbox_hash, select_response.uidvalidity);
if let Some(v) = select_response.highestmodseq {
self.uid_store
.highestmodseqs
.lock()
.unwrap()
.insert(mailbox_hash, v);
}
cache_handle.clear(mailbox_hash, &select_response)?;
self.send_command(b"UID FETCH 1:* (UID FLAGS ENVELOPE BODYSTRUCTURE)")
.await?;
self.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
.await?;
let fetches = protocol_parser::fetch_responses(&response)?.1;
cache_handle.insert_envelopes(mailbox_hash, &fetches)?;
Ok(())
}
//rfc4549_Synchronization_Operations_for_Disconnected_IMAP4_Clients
pub async fn resync_basic(
&mut self,
@ -170,7 +136,7 @@ impl ImapConnection {
// 2. tag1 UID FETCH <lastseenuid+1>:* <descriptors>
self.send_command(
format!(
"UID FETCH {}:* (UID FLAGS ENVELOPE BODYSTRUCTURE)",
"UID FETCH {}:* (UID FLAGS ENVELOPE BODY.PEEK[HEADER.FIELDS (REFERENCES)] BODYSTRUCTURE)",
max_uid + 1
)
.as_bytes(),
@ -189,12 +155,28 @@ impl ImapConnection {
ref uid,
ref mut envelope,
ref mut flags,
ref references,
..
} in v.iter_mut()
{
let uid = uid.unwrap();
let env = envelope.as_mut().unwrap();
env.set_hash(generate_envelope_hash(&mailbox_path, &uid));
if let Some(value) = references {
let parse_result = crate::email::parser::address::msg_id_list(value);
if let Ok((_, value)) = parse_result {
let prev_val = env.references.take();
for v in value {
env.push_references(v);
}
if let Some(prev) = prev_val {
for v in prev.refs {
env.push_references(v);
}
}
}
env.set_references(value);
}
let mut tag_lck = self.uid_store.tag_index.write().unwrap();
if let Some((flags, keywords)) = flags {
env.set_flags(*flags);
@ -457,7 +439,7 @@ impl ImapConnection {
// 2. tag1 UID FETCH <lastseenuid+1>:* <descriptors>
self.send_command(
format!(
"UID FETCH {}:* (UID FLAGS ENVELOPE BODYSTRUCTURE) (CHANGEDSINCE {})",
"UID FETCH {}:* (UID FLAGS ENVELOPE BODY.PEEK[HEADER.FIELDS (REFERENCES)] BODYSTRUCTURE) (CHANGEDSINCE {})",
cached_max_uid + 1,
cached_highestmodseq,
)
@ -477,12 +459,28 @@ impl ImapConnection {
ref uid,
ref mut envelope,
ref mut flags,
ref references,
..
} in v.iter_mut()
{
let uid = uid.unwrap();
let env = envelope.as_mut().unwrap();
env.set_hash(generate_envelope_hash(&mailbox_path, &uid));
if let Some(value) = references {
let parse_result = crate::email::parser::address::msg_id_list(value);
if let Ok((_, value)) = parse_result {
let prev_val = env.references.take();
for v in value {
env.push_references(v);
}
if let Some(prev) = prev_val {
for v in prev.refs {
env.push_references(v);
}
}
}
env.set_references(value);
}
let mut tag_lck = self.uid_store.tag_index.write().unwrap();
if let Some((flags, keywords)) = flags {
env.set_flags(*flags);

View File

@ -464,6 +464,7 @@ pub struct FetchResponse<'a> {
pub modseq: Option<ModSequence>,
pub flags: Option<(Flag, Vec<String>)>,
pub body: Option<&'a [u8]>,
pub references: Option<&'a [u8]>,
pub envelope: Option<Envelope>,
pub raw_fetch_value: &'a [u8],
}
@ -520,6 +521,7 @@ pub fn fetch_response(input: &[u8]) -> ImapParseResult<FetchResponse<'_>> {
modseq: None,
flags: None,
body: None,
references: None,
envelope: None,
raw_fetch_value: &[],
};
@ -617,6 +619,22 @@ pub fn fetch_response(input: &[u8]) -> ImapParseResult<FetchResponse<'_>> {
let (rest, _has_attachments) = bodystructure_has_attachments(&input[i..])?;
has_attachments = _has_attachments;
i += input[i..].len() - rest.len();
} else if input[i..].starts_with(b"BODY[HEADER.FIELDS (REFERENCES)] ") {
i += b"BODY[HEADER.FIELDS (REFERENCES)] ".len();
if let Ok((rest, mut references)) = astring_token(&input[i..]) {
if !references.trim().is_empty() {
if let Ok((_, (_, v))) = crate::email::parser::headers::header(&references) {
references = v;
}
ret.references = Some(references);
}
i += input.len() - i - rest.len();
} else {
return debug!(Err(MeliError::new(format!(
"Unexpected input while parsing UID FETCH response. Got: `{:.40}`",
String::from_utf8_lossy(&input[i..])
))));
}
} else if input[i..].starts_with(b")\r\n") {
i += b")\r\n".len();
break;
@ -860,7 +878,6 @@ pub fn untagged_responses(input: &[u8]) -> ImapParseResult<Option<UntaggedRespon
#[test]
fn test_untagged_responses() {
use std::convert::TryInto;
use UntaggedResponse::*;
assert_eq!(
untagged_responses(b"* 2 EXISTS\r\n")
@ -880,6 +897,7 @@ fn test_untagged_responses() {
modseq: Some(ModSequence(std::num::NonZeroU64::new(1365_u64).unwrap())),
flags: Some((Flag::SEEN, vec![])),
body: None,
references: None,
envelope: None,
raw_fetch_value: &b"* 1079 FETCH (UID 1103 MODSEQ (1365) FLAGS (\\Seen))\r\n"[..],
})
@ -895,6 +913,7 @@ fn test_untagged_responses() {
modseq: None,
flags: Some((Flag::SEEN, vec![])),
body: None,
references: None,
envelope: None,
raw_fetch_value: &b"* 1 FETCH (FLAGS (\\Seen))\r\n"[..],
})
@ -1052,7 +1071,6 @@ pub fn select_response(input: &[u8]) -> Result<SelectResponse> {
#[test]
fn test_select_response() {
use std::convert::TryInto;
let r = b"* FLAGS (\\Answered \\Flagged \\Deleted \\Seen \\Draft)\r\n* OK [PERMANENTFLAGS (\\Answered \\Flagged \\Deleted \\Seen \\Draft \\*)] Flags permitted.\r\n* 45 EXISTS\r\n* 0 RECENT\r\n* OK [UNSEEN 16] First unseen.\r\n* OK [UIDVALIDITY 1554422056] UIDs valid\r\n* OK [UIDNEXT 50] Predicted next UID\r\n";
assert_eq!(

View File

@ -416,6 +416,7 @@ impl ImapConnection {
modseq,
flags,
body: _,
references: _,
envelope: _,
raw_fetch_value: _,
}) => {