imap: add UIDVALIDITY check

On UIDVALIDITY change, discard cache and force rescan.
sql
Manos Pitsidianakis 2019-11-10 23:02:23 +02:00
parent 0cbc44fd0e
commit c1902f96b5
Signed by: Manos Pitsidianakis
GPG Key ID: 73627C2F690DF710
4 changed files with 136 additions and 116 deletions

View File

@ -82,6 +82,15 @@ impl std::ops::Deref for IsSubscribedFn {
}
}
type Capabilities = FnvHashSet<Vec<u8>>;
#[derive(Debug)]
pub struct UIDStore {
uidvalidity: Arc<Mutex<FnvHashMap<FolderHash, UID>>>,
hash_index: Arc<Mutex<FnvHashMap<EnvelopeHash, (UID, FolderHash)>>>,
uid_index: Arc<Mutex<FnvHashMap<UID, EnvelopeHash>>>,
byte_cache: Arc<Mutex<FnvHashMap<UID, EnvelopeCache>>>,
}
#[derive(Debug)]
pub struct ImapType {
account_name: String,
@ -89,12 +98,9 @@ pub struct ImapType {
is_subscribed: Arc<IsSubscribedFn>,
connection: Arc<Mutex<ImapConnection>>,
server_conf: ImapServerConf,
uid_store: Arc<UIDStore>,
folders: Arc<Mutex<FnvHashMap<FolderHash, ImapFolder>>>,
hash_index: Arc<Mutex<FnvHashMap<EnvelopeHash, (UID, FolderHash)>>>,
uid_index: Arc<Mutex<FnvHashMap<usize, EnvelopeHash>>>,
byte_cache: Arc<Mutex<FnvHashMap<UID, EnvelopeCache>>>,
}
impl MailBackend for ImapType {
@ -114,8 +120,7 @@ impl MailBackend for ImapType {
let mut w = AsyncBuilder::new();
let handle = {
let tx = w.tx();
let hash_index = self.hash_index.clone();
let uid_index = self.uid_index.clone();
let uid_store = self.uid_store.clone();
let folder_path = folder.path().to_string();
let folder_hash = folder.hash();
let folder_exists = self.folders.lock().unwrap()[&folder_hash].exists.clone();
@ -133,14 +138,18 @@ impl MailBackend for ImapType {
conn.send_command(format!("EXAMINE {}", folder_path).as_bytes())
conn.read_response(&mut response)
);
let examine_response = protocol_parser::select_response(&response)
.to_full_result()
.map_err(MeliError::from);
let examine_response = protocol_parser::select_response(&response);
exit_on_error!(&tx, examine_response);
let mut exists: usize = match examine_response.unwrap() {
SelectResponse::Ok(ok) => ok.uidnext - 1,
SelectResponse::Bad(b) => b.exists,
};
let examine_response = examine_response.unwrap();
let mut exists: usize = examine_response.uidnext - 1;
{
let mut uidvalidities = uid_store.uidvalidity.lock().unwrap();
let v = uidvalidities
.entry(folder_hash)
.or_insert(examine_response.uidvalidity);
*v = examine_response.uidvalidity;
}
{
let mut folder_exists = folder_exists.lock().unwrap();
*folder_exists = exists;
@ -171,11 +180,12 @@ impl MailBackend for ImapType {
if let Some(flags) = flags {
env.set_flags(flags);
}
hash_index
uid_store
.hash_index
.lock()
.unwrap()
.insert(env.hash(), (uid, folder_hash));
uid_index.lock().unwrap().insert(uid, env.hash());
uid_store.uid_index.lock().unwrap().insert(uid, env.hash());
envelopes.push(env);
}
}
@ -211,8 +221,7 @@ impl MailBackend for ImapType {
let conn = ImapConnection::new_connection(&self.server_conf);
let main_conn = self.connection.clone();
let is_online = self.online.clone();
let hash_index = self.hash_index.clone();
let uid_index = self.uid_index.clone();
let uid_store = self.uid_store.clone();
let handle = std::thread::Builder::new()
.name(format!("{} imap connection", self.account_name.as_str(),))
.spawn(move || {
@ -225,8 +234,7 @@ impl MailBackend for ImapType {
conn,
is_online,
main_conn,
hash_index,
uid_index,
uid_store,
folders,
sender,
work_context,
@ -262,14 +270,14 @@ impl MailBackend for ImapType {
}
fn operation(&self, hash: EnvelopeHash) -> Box<dyn BackendOp> {
let (uid, folder_hash) = self.hash_index.lock().unwrap()[&hash];
let (uid, folder_hash) = self.uid_store.hash_index.lock().unwrap()[&hash];
Box::new(ImapOp::new(
uid,
self.folders.lock().unwrap()[&folder_hash]
.path()
.to_string(),
self.connection.clone(),
self.byte_cache.clone(),
self.uid_store.clone(),
))
}
@ -432,9 +440,12 @@ impl ImapType {
folders: Arc::new(Mutex::new(Default::default())),
connection: Arc::new(Mutex::new(connection)),
hash_index: Default::default(),
uid_index: Default::default(),
byte_cache: Default::default(),
uid_store: Arc::new(UIDStore {
uidvalidity: Default::default(),
hash_index: Default::default(),
uid_index: Default::default(),
byte_cache: Default::default(),
}),
}
}
@ -543,7 +554,7 @@ impl ImapType {
for l in lines.by_ref() {
if l.starts_with("* SEARCH") {
use std::iter::FromIterator;
let uid_index = self.uid_index.lock()?;
let uid_index = self.uid_store.uid_index.lock()?;
return Ok(crate::structs::StackVec::from_iter(
l["* SEARCH".len()..]
.trim()

View File

@ -37,7 +37,7 @@ pub struct ImapOp {
folder_path: String,
flags: Cell<Option<Flag>>,
connection: Arc<Mutex<ImapConnection>>,
byte_cache: Arc<Mutex<FnvHashMap<UID, EnvelopeCache>>>,
uid_store: Arc<UIDStore>,
}
impl ImapOp {
@ -45,7 +45,7 @@ impl ImapOp {
uid: usize,
folder_path: String,
connection: Arc<Mutex<ImapConnection>>,
byte_cache: Arc<Mutex<FnvHashMap<UID, EnvelopeCache>>>,
uid_store: Arc<UIDStore>,
) -> Self {
ImapOp {
uid,
@ -55,7 +55,7 @@ impl ImapOp {
body: None,
folder_path,
flags: Cell::new(None),
byte_cache,
uid_store,
}
}
}
@ -67,7 +67,7 @@ impl BackendOp for ImapOp {
fn as_bytes(&mut self) -> Result<&[u8]> {
if self.bytes.is_none() {
let mut bytes_cache = self.byte_cache.lock()?;
let mut bytes_cache = self.uid_store.byte_cache.lock()?;
let cache = bytes_cache.entry(self.uid).or_default();
if cache.bytes.is_some() {
self.bytes = cache.bytes.clone();
@ -122,7 +122,7 @@ impl BackendOp for ImapOp {
return Ok(result);
}
if self.headers.is_none() {
let mut bytes_cache = self.byte_cache.lock()?;
let mut bytes_cache = self.uid_store.byte_cache.lock()?;
let cache = bytes_cache.entry(self.uid).or_default();
if cache.headers.is_some() {
self.headers = cache.headers.clone();
@ -175,7 +175,7 @@ impl BackendOp for ImapOp {
return Ok(result);
}
if self.body.is_none() {
let mut bytes_cache = self.byte_cache.lock()?;
let mut bytes_cache = self.uid_store.byte_cache.lock()?;
let cache = bytes_cache.entry(self.uid).or_default();
if cache.body.is_some() {
self.body = cache.body.clone();
@ -223,7 +223,7 @@ impl BackendOp for ImapOp {
if self.flags.get().is_some() {
return self.flags.get().unwrap();
}
let mut bytes_cache = self.byte_cache.lock().unwrap();
let mut bytes_cache = self.uid_store.byte_cache.lock().unwrap();
let cache = bytes_cache.entry(self.uid).or_default();
if cache.flags.is_some() {
self.flags.set(cache.flags);
@ -295,7 +295,7 @@ impl BackendOp for ImapOp {
}
conn.send_command(format!("EXAMINE \"{}\"", &self.folder_path,).as_bytes())?;
conn.read_response(&mut response)?;
let mut bytes_cache = self.byte_cache.lock()?;
let mut bytes_cache = self.uid_store.byte_cache.lock()?;
let cache = bytes_cache.entry(self.uid).or_default();
cache.flags = Some(flag);
Ok(())

View File

@ -278,14 +278,8 @@ named!(
do_parse!(tag!("* SEARCH\r\n") >> ({ &b""[0..] })))
);
#[derive(Debug, Clone)]
pub enum SelectResponse {
Ok(SelectResponseOk),
Bad(SelectResponseBad),
}
#[derive(Debug, Default, Clone)]
pub struct SelectResponseOk {
pub struct SelectResponse {
pub exists: usize,
pub recent: usize,
pub flags: Flag,
@ -295,13 +289,6 @@ pub struct SelectResponseOk {
pub permanentflags: Flag,
}
#[derive(Debug, Default, Clone)]
pub struct SelectResponseBad {
pub exists: usize,
pub recent: usize,
pub flags: Flag,
}
/*
* Example: C: A142 SELECT INBOX
* S: * 172 EXISTS
@ -324,9 +311,9 @@ pub struct SelectResponseBad {
* * OK [UIDVALIDITY 1554422056] UIDs valid
* * OK [UIDNEXT 50] Predicted next UID
*/
pub fn select_response(input: &str) -> IResult<&str, SelectResponse> {
pub fn select_response(input: &str) -> Result<SelectResponse> {
if input.contains("* OK") {
let mut ret = SelectResponseOk::default();
let mut ret = SelectResponse::default();
for l in input.split("\r\n") {
if l.starts_with("* ") && l.ends_with(" EXISTS") {
ret.exists = usize::from_str(&l["* ".len()..l.len() - " EXISTS".len()]).unwrap();
@ -354,23 +341,10 @@ pub fn select_response(input: &str) -> IResult<&str, SelectResponse> {
debug!("select response: {}", l);
}
}
IResult::Done(&""[0..], SelectResponse::Ok(ret))
Ok(ret)
} else {
let mut ret = SelectResponseBad::default();
for l in input.split("\r\n") {
if l.starts_with("* ") && l.ends_with(" EXISTS") {
ret.exists = usize::from_str(&l["* ".len()..l.len() - " EXISTS".len()]).unwrap();
} else if l.starts_with("* ") && l.ends_with(" RECENT") {
ret.recent = usize::from_str(&l["* ".len()..l.len() - " RECENT".len()]).unwrap();
} else if l.starts_with("* FLAGS (") {
ret.flags = flags(&l["* FLAGS (".len()..l.len() - ")".len()])
.to_full_result()
.unwrap();
} else if !l.is_empty() {
debug!("select response: {}", l);
}
}
IResult::Done(&""[0..], SelectResponse::Bad(ret))
debug!("BAD/NO response in select: {}", input);
Err(MeliError::new(input))
}
}

View File

@ -26,8 +26,7 @@ pub struct ImapWatchKit {
pub conn: ImapConnection,
pub is_online: Arc<Mutex<bool>>,
pub main_conn: Arc<Mutex<ImapConnection>>,
pub hash_index: Arc<Mutex<FnvHashMap<EnvelopeHash, (UID, FolderHash)>>>,
pub uid_index: Arc<Mutex<FnvHashMap<usize, EnvelopeHash>>>,
pub uid_store: Arc<UIDStore>,
pub folders: Arc<Mutex<FnvHashMap<FolderHash, ImapFolder>>>,
pub sender: RefreshEventConsumer,
pub work_context: WorkContext,
@ -53,8 +52,7 @@ pub fn poll_with_examine(kit: ImapWatchKit) -> Result<()> {
is_online,
mut conn,
main_conn,
hash_index,
uid_index,
uid_store,
folders,
sender,
work_context,
@ -82,14 +80,7 @@ pub fn poll_with_examine(kit: ImapWatchKit) -> Result<()> {
format!("examining `{}` for updates...", folder.path()),
))
.unwrap();
examine_updates(
folder,
&sender,
&mut conn,
&hash_index,
&uid_index,
&work_context,
)?;
examine_updates(folder, &sender, &mut conn, &uid_store, &work_context)?;
}
let mut main_conn = main_conn.lock().unwrap();
main_conn.send_command(b"NOOP").unwrap();
@ -106,8 +97,7 @@ pub fn idle(kit: ImapWatchKit) -> Result<()> {
mut conn,
is_online,
main_conn,
hash_index,
uid_index,
uid_store,
folders,
sender,
work_context,
@ -139,15 +129,37 @@ pub fn idle(kit: ImapWatchKit) -> Result<()> {
debug!("select response {}", &response);
{
let mut prev_exists = folder.exists.lock().unwrap();
*prev_exists = match protocol_parser::select_response(&response)
.to_full_result()
.map_err(MeliError::from)
{
Ok(SelectResponse::Bad(bad)) => {
debug!(bad);
panic!("could not select mailbox");
}
Ok(SelectResponse::Ok(ok)) => {
*prev_exists = match protocol_parser::select_response(&response) {
Ok(ok) => {
{
let mut uidvalidities = uid_store.uidvalidity.lock().unwrap();
if let Some(v) = uidvalidities.get_mut(&folder_hash) {
if *v != ok.uidvalidity {
sender.send(RefreshEvent {
hash: folder_hash,
kind: RefreshEventKind::Rescan,
});
uid_store.uid_index.lock().unwrap().clear();
uid_store.hash_index.lock().unwrap().clear();
uid_store.byte_cache.lock().unwrap().clear();
*v = ok.uidvalidity;
}
} else {
sender.send(RefreshEvent {
hash: folder_hash,
kind: RefreshEventKind::Rescan,
});
sender.send(RefreshEvent {
hash: folder_hash,
kind: RefreshEventKind::Failure(MeliError::new(format!(
"Unknown mailbox: {} {}",
folder.path(),
folder_hash
))),
});
}
}
debug!(&ok);
ok.exists
}
@ -216,14 +228,7 @@ pub fn idle(kit: ImapWatchKit) -> Result<()> {
format!("examining `{}` for updates...", folder.path()),
))
.unwrap();
examine_updates(
folder,
&sender,
&mut iter.conn,
&hash_index,
&uid_index,
&work_context,
);
examine_updates(folder, &sender, &mut iter.conn, &uid_store, &work_context);
}
work_context
.set_status
@ -299,11 +304,12 @@ pub fn idle(kit: ImapWatchKit) -> Result<()> {
.unwrap();
if let Ok(env) = Envelope::from_bytes(&b, flags) {
ctr += 1;
hash_index
uid_store
.hash_index
.lock()
.unwrap()
.insert(env.hash(), (uid, folder_hash));
uid_index.lock().unwrap().insert(uid, env.hash());
uid_store.uid_index.lock().unwrap().insert(uid, env.hash());
debug!(
"Create event {} {} {}",
env.hash(),
@ -407,17 +413,18 @@ pub fn idle(kit: ImapWatchKit) -> Result<()> {
format!("parsing {}/{} envelopes..", ctr, len),
))
.unwrap();
if uid_index.lock().unwrap().contains_key(&uid) {
if uid_store.uid_index.lock().unwrap().contains_key(&uid) {
ctr += 1;
continue;
}
if let Ok(env) = Envelope::from_bytes(&b, flags) {
ctr += 1;
hash_index
uid_store
.hash_index
.lock()
.unwrap()
.insert(env.hash(), (uid, folder_hash));
uid_index.lock().unwrap().insert(uid, env.hash());
uid_store.uid_index.lock().unwrap().insert(uid, env.hash());
debug!(
"Create event {} {} {}",
env.hash(),
@ -476,8 +483,7 @@ fn examine_updates(
folder: &ImapFolder,
sender: &RefreshEventConsumer,
conn: &mut ImapConnection,
hash_index: &Arc<Mutex<FnvHashMap<EnvelopeHash, (UID, FolderHash)>>>,
uid_index: &Arc<Mutex<FnvHashMap<usize, EnvelopeHash>>>,
uid_store: &Arc<UIDStore>,
work_context: &WorkContext,
) -> Result<()> {
let thread_id: std::thread::ThreadId = std::thread::current().id();
@ -492,16 +498,39 @@ fn examine_updates(
conn.send_command(format!("EXAMINE {}", folder.path()).as_bytes())
conn.read_response(&mut response)
);
match protocol_parser::select_response(&response)
.to_full_result()
.map_err(MeliError::from)
{
Ok(SelectResponse::Bad(bad)) => {
debug!(bad);
panic!("could not select mailbox");
}
Ok(SelectResponse::Ok(ok)) => {
match protocol_parser::select_response(&response) {
Ok(ok) => {
debug!(&ok);
{
let mut uidvalidities = uid_store.uidvalidity.lock().unwrap();
if let Some(v) = uidvalidities.get_mut(&folder_hash) {
if *v != ok.uidvalidity {
sender.send(RefreshEvent {
hash: folder_hash,
kind: RefreshEventKind::Rescan,
});
uid_store.uid_index.lock().unwrap().clear();
uid_store.hash_index.lock().unwrap().clear();
uid_store.byte_cache.lock().unwrap().clear();
*v = ok.uidvalidity;
}
} else {
// FIXME: Handle this case in ui/src/conf/accounts.rs
sender.send(RefreshEvent {
hash: folder_hash,
kind: RefreshEventKind::Rescan,
});
sender.send(RefreshEvent {
hash: folder_hash,
kind: RefreshEventKind::Failure(MeliError::new(format!(
"Unknown mailbox: {} {}",
folder.path(),
folder_hash
))),
});
}
}
let mut prev_exists = folder.exists.lock().unwrap();
let n = ok.exists;
if ok.recent > 0 {
@ -542,11 +571,16 @@ fn examine_updates(
Ok(v) => {
for (uid, flags, b) in v {
if let Ok(env) = Envelope::from_bytes(&b, flags) {
hash_index
uid_store
.hash_index
.lock()
.unwrap()
.insert(env.hash(), (uid, folder_hash));
uid_index.lock().unwrap().insert(uid, env.hash());
uid_store
.uid_index
.lock()
.unwrap()
.insert(uid, env.hash());
debug!(
"Create event {} {} {}",
env.hash(),
@ -600,11 +634,12 @@ fn examine_updates(
Ok(v) => {
for (uid, flags, b) in v {
if let Ok(env) = Envelope::from_bytes(&b, flags) {
hash_index
uid_store
.hash_index
.lock()
.unwrap()
.insert(env.hash(), (uid, folder_hash));
uid_index.lock().unwrap().insert(uid, env.hash());
uid_store.uid_index.lock().unwrap().insert(uid, env.hash());
debug!(
"Create event {} {} {}",
env.hash(),