meli
/
wasm-demo
4
Fork 0
Code Issues Pull Requests Releases Wiki Activity

melib/imap: fix cache not being updated in some events

master
Manos Pitsidianakis 2020-08-27 17:25:05 +03:00
parent 6302d9d618
commit 209bd98814
Signed by: Manos Pitsidianakis
GPG Key ID: 73627C2F690DF710
3 changed files with 241 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
melib/src/backends/imap/cache.rs
View File

@ -50,6 +50,7 @@ impl core::fmt::Display for ModSequence {
#[derive(Debug)] #[derive(Debug)]
pub struct CachedEnvelope { pub struct CachedEnvelope {
pub inner: Envelope, pub inner: Envelope,
pub uid: UID,
pub mailbox_hash: MailboxHash, pub mailbox_hash: MailboxHash,
pub modsequence: Option<ModSequence>, pub modsequence: Option<ModSequence>,
} }
@ -206,6 +207,7 @@ mod sqlite3_m {
env.hash(), env.hash(),
CachedEnvelope { CachedEnvelope {
inner: env, inner: env,
uid,
mailbox_hash, mailbox_hash,
modsequence: modseq, modsequence: modseq,
}, },
@ -279,6 +281,104 @@ mod sqlite3_m {
tx.commit()?; tx.commit()?;
Ok(()) Ok(())
} }
pub fn update(
&mut self,
mailbox_hash: MailboxHash,
refresh_events: &[(UID, RefreshEvent)],
) -> Result<()> {
debug!(
"update with refresh_events mailbox_hash {} len {}",
mailbox_hash,
refresh_events.len()
);
if self.mailbox_state(mailbox_hash)?.is_none() {
debug!(self.mailbox_state(mailbox_hash)?.is_none());
let uidvalidity = self
.uid_store
.uidvalidity
.lock()
.unwrap()
.get(&mailbox_hash)
.cloned();
let highestmodseq = self
.uid_store
.highestmodseqs
.lock()
.unwrap()
.get(&mailbox_hash)
.cloned();
debug!(&uidvalidity);
debug!(&highestmodseq);
if let Some(uidvalidity) = uidvalidity {
debug!(self.clear(
mailbox_hash,
uidvalidity,
highestmodseq.and_then(|v| v.ok()),
))?;
}
}
let Self {
ref mut connection,
ref uid_store,
} = self;
let tx = connection.transaction()?;
let mut hash_index_lck = uid_store.hash_index.lock().unwrap();
for (uid, event) in refresh_events {
match debug!(&event.kind) {
RefreshEventKind::Remove(env_hash) => {
hash_index_lck.remove(&env_hash);
tx.execute(
"DELETE FROM envelopes WHERE mailbox_hash = ?1 AND uid = ?2;",
sqlite3::params![mailbox_hash as i64, *uid as i64],
)
.chain_err_summary(|| {
format!(
"Could not remove envelope {} uid {} from mailbox {} account {}",
env_hash, *uid, mailbox_hash, uid_store.account_name
)
})?;
}
RefreshEventKind::NewFlags(env_hash, (flags, tags)) => {
let mut stmt = tx.prepare(
"SELECT envelope FROM envelopes WHERE mailbox_hash = ?1 AND uid = ?2;",
)?;
let mut ret: Vec<Envelope> = stmt
.query_map(sqlite3::params![mailbox_hash as i64, *uid as i64], |row| {
Ok(row.get(0)?)
})?
.collect::<std::result::Result<_, _>>()?;
if let Some(mut env) = ret.pop() {
env.set_flags(*flags);
env.labels_mut().clear();
env.labels_mut().extend(tags.iter().map(|t| tag_hash!(t)));
tx.execute(
"UPDATE envelopes SET envelope = ?1 WHERE mailbox_hash = ?2 AND uid = ?3;",
sqlite3::params![&env, mailbox_hash as i64, *uid as i64],
)
.chain_err_summary(|| {
format!(
"Could not update envelope {} uid {} from mailbox {} account {}",
env_hash, *uid, mailbox_hash, uid_store.account_name
)
})?;
uid_store
.envelopes
.lock()
.unwrap()
.entry(*env_hash)
.and_modify(|entry| {
entry.inner = env;
});
}
}
_ => {}
}
}
tx.commit()?;
Ok(())
}
} }
} }

102
melib/src/backends/imap/cache/sync.rs vendored
View File

@ -128,7 +128,7 @@ impl ImapConnection {
//rfc4549_Synchronization_Operations_for_Disconnected_IMAP4_Clients //rfc4549_Synchronization_Operations_for_Disconnected_IMAP4_Clients
pub async fn resync_basic( pub async fn resync_basic(
&mut self, &mut self,
cache_handle: CacheHandle, mut cache_handle: CacheHandle,
mailbox_hash: MailboxHash, mailbox_hash: MailboxHash,
) -> Result<Option<Vec<Envelope>>> { ) -> Result<Option<Vec<Envelope>>> {
let mut payload = vec![]; let mut payload = vec![];
@ -319,34 +319,41 @@ impl ImapConnection {
.labels_mut() .labels_mut()
.extend(tags.iter().map(|t| tag_hash!(t))); .extend(tags.iter().map(|t| tag_hash!(t)));
}); });
refresh_events.push(RefreshEvent { refresh_events.push((
mailbox_hash, uid,
account_hash: self.uid_store.account_hash, RefreshEvent {
kind: RefreshEventKind::NewFlags(env_hash, (flags, tags)), mailbox_hash,
}); account_hash: self.uid_store.account_hash,
kind: RefreshEventKind::NewFlags(env_hash, (flags, tags)),
},
));
} }
} }
for env_hash in valid_envs.difference( for env_hash in env_lck
&env_lck .iter()
.iter() .filter_map(|(h, cenv)| {
.filter_map(|(h, cenv)| { if cenv.mailbox_hash == mailbox_hash {
if cenv.mailbox_hash == mailbox_hash { Some(*h)
Some(*h) } else {
} else { None
None }
} })
}) .collect::<BTreeSet<EnvelopeHash>>()
.collect(), .difference(&valid_envs)
) { {
refresh_events.push((
env_lck[env_hash].uid,
RefreshEvent {
mailbox_hash,
account_hash: self.uid_store.account_hash,
kind: RefreshEventKind::Remove(*env_hash),
},
));
env_lck.remove(env_hash); env_lck.remove(env_hash);
refresh_events.push(RefreshEvent {
mailbox_hash,
account_hash: self.uid_store.account_hash,
kind: RefreshEventKind::Remove(*env_hash),
});
} }
drop(env_lck); drop(env_lck);
for ev in refresh_events { cache_handle.update(mailbox_hash, &refresh_events)?;
for (_uid, ev) in refresh_events {
self.add_refresh_event(ev); self.add_refresh_event(ev);
} }
Ok(Some(payload.into_iter().map(|(_, env)| env).collect())) Ok(Some(payload.into_iter().map(|(_, env)| env).collect()))
@ -356,7 +363,7 @@ impl ImapConnection {
//Section 6.1 //Section 6.1
pub async fn resync_condstore( pub async fn resync_condstore(
&mut self, &mut self,
cache_handle: CacheHandle, mut cache_handle: CacheHandle,
mailbox_hash: MailboxHash, mailbox_hash: MailboxHash,
) -> Result<Option<Vec<Envelope>>> { ) -> Result<Option<Vec<Envelope>>> {
let mut payload = vec![]; let mut payload = vec![];
@ -598,11 +605,14 @@ impl ImapConnection {
.labels_mut() .labels_mut()
.extend(tags.iter().map(|t| tag_hash!(t))); .extend(tags.iter().map(|t| tag_hash!(t)));
}); });
refresh_events.push(RefreshEvent { refresh_events.push((
mailbox_hash, uid,
account_hash: self.uid_store.account_hash, RefreshEvent {
kind: RefreshEventKind::NewFlags(env_hash, (flags, tags)), mailbox_hash,
}); account_hash: self.uid_store.account_hash,
kind: RefreshEventKind::NewFlags(env_hash, (flags, tags)),
},
));
} }
} }
self.uid_store self.uid_store
@ -622,9 +632,9 @@ impl ImapConnection {
for uid in v { for uid in v {
valid_envs.insert(generate_envelope_hash(&mailbox_path, &uid)); valid_envs.insert(generate_envelope_hash(&mailbox_path, &uid));
} }
let mut env_lck = self.uid_store.envelopes.lock().unwrap(); {
for env_hash in valid_envs.difference( let mut env_lck = self.uid_store.envelopes.lock().unwrap();
&env_lck for env_hash in env_lck
.iter() .iter()
.filter_map(|(h, cenv)| { .filter_map(|(h, cenv)| {
if cenv.mailbox_hash == mailbox_hash { if cenv.mailbox_hash == mailbox_hash {
@ -633,17 +643,23 @@ impl ImapConnection {
None None
} }
}) })
.collect(), .collect::<BTreeSet<EnvelopeHash>>()
) { .difference(&valid_envs)
env_lck.remove(env_hash); {
refresh_events.push(RefreshEvent { refresh_events.push((
mailbox_hash, env_lck[env_hash].uid,
account_hash: self.uid_store.account_hash, RefreshEvent {
kind: RefreshEventKind::Remove(*env_hash), mailbox_hash,
}); account_hash: self.uid_store.account_hash,
kind: RefreshEventKind::Remove(*env_hash),
},
));
env_lck.remove(env_hash);
}
drop(env_lck);
} }
drop(env_lck); cache_handle.update(mailbox_hash, &refresh_events)?;
for ev in refresh_events { for (_uid, ev) in refresh_events {
self.add_refresh_event(ev); self.add_refresh_event(ev);
} }
Ok(Some(payload.into_iter().map(|(_, env)| env).collect())) Ok(Some(payload.into_iter().map(|(_, env)| env).collect()))

109
melib/src/backends/imap/untagged.rs
View File

@ -19,7 +19,7 @@
* along with meli. If not, see <http://www.gnu.org/licenses/>. * along with meli. If not, see <http://www.gnu.org/licenses/>.
*/ */
use super::{ImapConnection, MailboxSelection}; use super::{ImapConnection, MailboxSelection, UID};
use crate::backends::imap::protocol_parser::{ use crate::backends::imap::protocol_parser::{
generate_envelope_hash, FetchResponse, ImapLineSplit, RequiredResponses, UntaggedResponse, generate_envelope_hash, FetchResponse, ImapLineSplit, RequiredResponses, UntaggedResponse,
}; };
@ -58,6 +58,7 @@ impl ImapConnection {
let mailbox = let mailbox =
std::clone::Clone::clone(&self.uid_store.mailboxes.lock().await[&mailbox_hash]); std::clone::Clone::clone(&self.uid_store.mailboxes.lock().await[&mailbox_hash]);
let mut cache_handle = super::cache::CacheHandle::get(self.uid_store.clone())?;
let mut response = String::with_capacity(8 * 1024); let mut response = String::with_capacity(8 * 1024);
let untagged_response = let untagged_response =
match super::protocol_parser::untagged_responses(line).map(|(_, v, _)| v) { match super::protocol_parser::untagged_responses(line).map(|(_, v, _)| v) {
@ -81,23 +82,38 @@ impl ImapConnection {
.or_default() .or_default()
.remove(n); .remove(n);
debug!("expunge {}, UID = {}", n, deleted_uid); debug!("expunge {}, UID = {}", n, deleted_uid);
let deleted_hash: crate::email::EnvelopeHash = self let deleted_hash: crate::email::EnvelopeHash = match self
.uid_store .uid_store
.uid_index .uid_index
.lock() .lock()
.unwrap() .unwrap()
.remove(&(mailbox_hash, deleted_uid)) .remove(&(mailbox_hash, deleted_uid))
.unwrap(); {
Some(v) => v,
None => return Ok(true),
};
self.uid_store self.uid_store
.hash_index .hash_index
.lock() .lock()
.unwrap() .unwrap()
.remove(&deleted_hash); .remove(&deleted_hash);
self.add_refresh_event(RefreshEvent { let mut event: [(UID, RefreshEvent); 1] = [(
account_hash: self.uid_store.account_hash, deleted_uid,
mailbox_hash, RefreshEvent {
kind: Remove(deleted_hash), account_hash: self.uid_store.account_hash,
}); mailbox_hash,
kind: Remove(deleted_hash),
},
)];
cache_handle.update(mailbox_hash, &event)?;
self.add_refresh_event(std::mem::replace(
&mut event[0].1,
RefreshEvent {
account_hash: self.uid_store.account_hash,
mailbox_hash,
kind: Rescan,
},
));
} }
UntaggedResponse::Exists(n) => { UntaggedResponse::Exists(n) => {
/* UID FETCH ALL UID, cross-ref, then FETCH difference headers /* UID FETCH ALL UID, cross-ref, then FETCH difference headers
@ -166,11 +182,23 @@ impl ImapConnection {
mailbox.unseen.lock().unwrap().insert_new(env.hash()); mailbox.unseen.lock().unwrap().insert_new(env.hash());
} }
mailbox.exists.lock().unwrap().insert_new(env.hash()); mailbox.exists.lock().unwrap().insert_new(env.hash());
self.add_refresh_event(RefreshEvent { let mut event: [(UID, RefreshEvent); 1] = [(
account_hash: self.uid_store.account_hash, uid,
mailbox_hash, RefreshEvent {
kind: Create(Box::new(env)), account_hash: self.uid_store.account_hash,
}); mailbox_hash,
kind: Create(Box::new(env)),
},
)];
cache_handle.update(mailbox_hash, &event)?;
self.add_refresh_event(std::mem::replace(
&mut event[0].1,
RefreshEvent {
account_hash: self.uid_store.account_hash,
mailbox_hash,
kind: Rescan,
},
));
} }
} }
} }
@ -256,11 +284,23 @@ impl ImapConnection {
} }
mailbox.exists.lock().unwrap().insert_new(env.hash()); mailbox.exists.lock().unwrap().insert_new(env.hash());
self.add_refresh_event(RefreshEvent { let mut event: [(UID, RefreshEvent); 1] = [(
account_hash: self.uid_store.account_hash, uid,
mailbox_hash, RefreshEvent {
kind: Create(Box::new(env)), account_hash: self.uid_store.account_hash,
}); mailbox_hash,
kind: Create(Box::new(env)),
},
)];
cache_handle.update(mailbox_hash, &event)?;
self.add_refresh_event(std::mem::replace(
&mut event[0].1,
RefreshEvent {
account_hash: self.uid_store.account_hash,
mailbox_hash,
kind: Rescan,
},
));
} }
} }
} }
@ -334,9 +374,13 @@ impl ImapConnection {
} }
}; };
debug!("fetch uid {} {:?}", uid, flags); debug!("fetch uid {} {:?}", uid, flags);
let lck = self.uid_store.uid_index.lock().unwrap(); let env_hash = self
let env_hash = lck.get(&(mailbox_hash, uid)).copied(); .uid_store
drop(lck); .uid_index
.lock()
.unwrap()
.get(&(mailbox_hash, uid))
.copied();
if let Some(env_hash) = env_hash { if let Some(env_hash) = env_hash {
if !flags.0.intersects(crate::email::Flag::SEEN) { if !flags.0.intersects(crate::email::Flag::SEEN) {
mailbox.unseen.lock().unwrap().insert_new(env_hash); mailbox.unseen.lock().unwrap().insert_new(env_hash);
@ -357,12 +401,23 @@ impl ImapConnection {
.unwrap() .unwrap()
.insert(env_hash, modseq); .insert(env_hash, modseq);
} }
let mut event: [(UID, RefreshEvent); 1] = [(
self.add_refresh_event(RefreshEvent { uid,
account_hash: self.uid_store.account_hash, RefreshEvent {
mailbox_hash, account_hash: self.uid_store.account_hash,
kind: NewFlags(env_hash, flags), mailbox_hash,
}); kind: NewFlags(env_hash, flags),
},
)];
cache_handle.update(mailbox_hash, &event)?;
self.add_refresh_event(std::mem::replace(
&mut event[0].1,
RefreshEvent {
account_hash: self.uid_store.account_hash,
mailbox_hash,
kind: Rescan,
},
));
}; };
} }
} }