Browse Source

melib/imap: add ImapCache trait

memfd
Manos Pitsidianakis 1 year ago
parent
commit
b4fe34eacf
Signed by untrusted user: epilys GPG Key ID: 73627C2F690DF710
  1. 24
      Cargo.lock
  2. 2
      melib/Cargo.toml
  3. 10
      melib/src/backends.rs
  4. 57
      melib/src/backends/imap.rs
  5. 469
      melib/src/backends/imap/cache.rs
  6. 81
      melib/src/backends/imap/cache/sync.rs
  7. 28
      melib/src/backends/imap/connection.rs
  8. 15
      melib/src/backends/imap/mailbox.rs
  9. 14
      melib/src/backends/imap/operations.rs
  10. 87
      melib/src/backends/imap/protocol_parser.rs
  11. 26
      melib/src/backends/imap/untagged.rs
  12. 28
      melib/src/backends/imap/watch.rs
  13. 14
      melib/src/error.rs

24
Cargo.lock

@ -814,9 +814,9 @@ dependencies = [
[[package]]
name = "libsqlite3-sys"
version = "0.16.0"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e5b95e89c330291768dc840238db7f9e204fd208511ab6319b56193a7f2ae25"
checksum = "e3a245984b1b06c291f46e27ebda9f369a94a1ab8461d0e845e23f9ced01f5db"
dependencies = [
"pkg-config",
"vcpkg",
@ -1476,9 +1476,9 @@ dependencies = [
[[package]]
name = "rusqlite"
version = "0.20.0"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a194373ef527035645a1bc21b10dc2125f73497e6e155771233eb187aedd051"
checksum = "4c78c3275d9d6eb684d2db4b2388546b32fdae0586c20a82f3905d21ea78b9ef"
dependencies = [
"bitflags",
"fallible-iterator",
@ -1486,7 +1486,7 @@ dependencies = [
"libsqlite3-sys",
"lru-cache",
"memchr",
"time",
"smallvec",
]
[[package]]
@ -1795,13 +1795,6 @@ dependencies = [
"redox_termios",
]
[[package]]
name = "testing"
version = "0.4.1"
dependencies = [
"melib",
]
[[package]]
name = "textwrap"
version = "0.11.0"
@ -1840,6 +1833,13 @@ dependencies = [
"serde",
]
[[package]]
name = "tools"
version = "0.4.1"
dependencies = [
"melib",
]
[[package]]
name = "tracing"
version = "0.1.18"

2
melib/Cargo.toml

@ -40,7 +40,7 @@ isahc = { version = "0.9.7", optional = true, default-features = false, features
serde_json = { version = "1.0", optional = true, features = ["raw_value",] }
smallvec = { version = "^1.4.0", features = ["serde", ] }
nix = "0.17.0"
rusqlite = {version = "0.20.0", optional = true }
rusqlite = {version = "0.24.0", optional = true }
libloading = "0.6.2"
futures = "0.3.5"

10
melib/src/backends.rs

@ -226,6 +226,16 @@ pub enum BackendEvent {
//Job(Box<Future<Output = Result<()>> + Send + 'static>)
}
impl From<MeliError> for BackendEvent {
fn from(val: MeliError) -> BackendEvent {
BackendEvent::Notice {
description: val.summary.as_ref().map(|s| s.to_string()),
content: val.to_string(),
level: crate::LoggingLevel::ERROR,
}
}
}
#[derive(Debug, Clone)]
pub enum RefreshEventKind {
Update(EnvelopeHash, Box<Envelope>),

57
melib/src/backends/imap.rs

@ -51,12 +51,17 @@ use futures::stream::Stream;
use std::collections::{hash_map::DefaultHasher, BTreeMap};
use std::collections::{BTreeSet, HashMap, HashSet};
use std::convert::TryFrom;
use std::convert::TryInto;
use std::hash::Hasher;
use std::pin::Pin;
use std::str::FromStr;
use std::sync::{Arc, Mutex, RwLock};
use std::time::{Duration, Instant};
pub type UID = usize;
pub type ImapNum = usize;
pub type UID = ImapNum;
pub type UIDVALIDITY = UID;
pub type MessageSequenceNumber = ImapNum;
pub static SUPPORTED_CAPABILITIES: &[&str] = &[
#[cfg(feature = "deflate_compression")]
@ -202,7 +207,6 @@ pub struct ImapType {
connection: Arc<FutureMutex<ImapConnection>>,
server_conf: ImapServerConf,
uid_store: Arc<UIDStore>,
can_create_flags: Arc<Mutex<bool>>,
}
impl MailBackend for ImapType {
@ -299,7 +303,6 @@ impl MailBackend for ImapType {
},
connection: self.connection.clone(),
mailbox_hash,
can_create_flags: self.can_create_flags.clone(),
uid_store: self.uid_store.clone(),
};
@ -742,11 +745,7 @@ impl MailBackend for ImapType {
}
fn tags(&self) -> Option<Arc<RwLock<BTreeMap<u64, String>>>> {
if *self.can_create_flags.lock().unwrap() {
Some(self.uid_store.tag_index.clone())
} else {
None
}
Some(self.uid_store.tag_index.clone())
}
fn as_any(&self) -> &dyn Any {
@ -1111,7 +1110,7 @@ impl MailBackend for ImapType {
l["* SEARCH".len()..]
.trim()
.split_whitespace()
.map(usize::from_str)
.map(UID::from_str)
.filter_map(std::result::Result::ok)
.filter_map(|uid| uid_index.get(&(mailbox_hash, uid)))
.copied(),
@ -1157,7 +1156,17 @@ impl ImapType {
let use_starttls = use_tls && get_conf_val!(s["use_starttls"], !(server_port == 993))?;
let danger_accept_invalid_certs: bool =
get_conf_val!(s["danger_accept_invalid_certs"], false)?;
#[cfg(feature = "sqlite3")]
let keep_offline_cache = get_conf_val!(s["offline_cache"], true)?;
#[cfg(not(feature = "sqlite3"))]
let keep_offline_cache = get_conf_val!(s["offline_cache"], false)?;
#[cfg(not(feature = "sqlite3"))]
if keep_offline_cache {
return Err(MeliError::new(format!(
"({}) keep_offline_cache is true but melib is not compiled with sqlite3",
s.name,
)));
}
let server_conf = ImapServerConf {
server_hostname: server_hostname.to_string(),
server_username: server_username.to_string(),
@ -1190,7 +1199,6 @@ impl ImapType {
Ok(Box::new(ImapType {
server_conf,
is_subscribed: Arc::new(IsSubscribedFn(is_subscribed)),
can_create_flags: Arc::new(Mutex::new(false)),
connection: Arc::new(FutureMutex::new(connection)),
uid_store,
}))
@ -1199,14 +1207,18 @@ impl ImapType {
pub fn shell(&mut self) {
let mut conn = ImapConnection::new_connection(&self.server_conf, self.uid_store.clone());
futures::executor::block_on(timeout(Duration::from_secs(3), conn.connect())).unwrap();
futures::executor::block_on(timeout(Duration::from_secs(3), conn.connect()))
.unwrap()
.unwrap();
let mut res = String::with_capacity(8 * 1024);
futures::executor::block_on(timeout(Duration::from_secs(3), conn.send_command(b"NOOP")))
.unwrap()
.unwrap();
futures::executor::block_on(timeout(
Duration::from_secs(3),
conn.read_response(&mut res, RequiredResponses::empty()),
))
.unwrap()
.unwrap();
let mut input = String::new();
@ -1220,11 +1232,13 @@ impl ImapType {
Duration::from_secs(3),
conn.send_command(input.as_bytes()),
))
.unwrap()
.unwrap();
futures::executor::block_on(timeout(
Duration::from_secs(3),
conn.read_lines(&mut res, String::new()),
))
.unwrap()
.unwrap();
if input.trim().eq_ignore_ascii_case("logout") {
break;
@ -1368,7 +1382,18 @@ impl ImapType {
)));
}
get_conf_val!(s["danger_accept_invalid_certs"], false)?;
#[cfg(feature = "sqlite3")]
get_conf_val!(s["offline_cache"], true)?;
#[cfg(not(feature = "sqlite3"))]
{
let keep_offline_cache = get_conf_val!(s["offline_cache"], false)?;
if keep_offline_cache {
return Err(MeliError::new(format!(
"({}) keep_offline_cache is true but melib is not compiled with sqlite3",
s.name,
)));
}
}
get_conf_val!(s["use_idle"], true)?;
get_conf_val!(s["use_condstore"], true)?;
#[cfg(feature = "deflate_compression")]
@ -1399,7 +1424,7 @@ enum FetchStage {
InitialFresh,
InitialCache,
ResyncCache,
FreshFetch { max_uid: usize },
FreshFetch { max_uid: UID },
Finished,
}
@ -1408,7 +1433,6 @@ struct FetchState {
stage: FetchStage,
connection: Arc<FutureMutex<ImapConnection>>,
mailbox_hash: MailboxHash,
can_create_flags: Arc<Mutex<bool>>,
uid_store: Arc<UIDStore>,
}
@ -1423,7 +1447,6 @@ async fn fetch_hlpr(state: &mut FetchState) -> Result<Vec<Envelope>> {
.await
.init_mailbox(state.mailbox_hash)
.await?;
*state.can_create_flags.lock().unwrap() = select_response.can_create_flags;
if select_response.exists == 0 {
state.stage = FetchStage::Finished;
return Ok(Vec::new());
@ -1481,7 +1504,6 @@ async fn fetch_hlpr(state: &mut FetchState) -> Result<Vec<Envelope>> {
ref mut stage,
ref connection,
mailbox_hash,
can_create_flags: _,
ref uid_store,
} = state;
let mailbox_hash = *mailbox_hash;
@ -1565,8 +1587,9 @@ async fn fetch_hlpr(state: &mut FetchState) -> Result<Vec<Envelope>> {
}
}
}
#[cfg(feature = "sqlite3")]
if uid_store.keep_offline_cache {
let mut cache_handle = cache::CacheHandle::get(uid_store.clone())?;
let mut cache_handle = cache::Sqlite3Cache::get(uid_store.clone())?;
debug!(cache_handle
.insert_envelopes(mailbox_hash, &v)
.chain_err_summary(|| {
@ -1601,7 +1624,7 @@ async fn fetch_hlpr(state: &mut FetchState) -> Result<Vec<Envelope>> {
.unwrap()
.entry(mailbox_hash)
.or_default()
.insert(message_sequence_number - 1, uid);
.insert((message_sequence_number - 1).try_into().unwrap(), uid);
uid_store
.hash_index
.lock()

469
melib/src/backends/imap/cache.rs

@ -26,7 +26,6 @@ use crate::{
email::{Envelope, EnvelopeHash},
error::*,
};
use std::convert::TryFrom;
#[derive(Debug, PartialEq, Hash, Eq, Ord, PartialOrd, Copy, Clone)]
@ -55,10 +54,36 @@ pub struct CachedEnvelope {
pub modsequence: Option<ModSequence>,
}
pub struct CacheHandle {
#[cfg(feature = "sqlite3")]
connection: crate::sqlite3::Connection,
uid_store: Arc<UIDStore>,
pub trait ImapCache: Send {
fn mailbox_state(&mut self, mailbox_hash: MailboxHash) -> Result<Option<()>>;
fn find_envelope(
&mut self,
identifier: std::result::Result<UID, EnvelopeHash>,
mailbox_hash: MailboxHash,
) -> Result<Option<CachedEnvelope>>;
fn update(
&mut self,
mailbox_hash: MailboxHash,
refresh_events: &[(UID, RefreshEvent)],
) -> Result<()>;
fn insert_envelopes(
&mut self,
mailbox_hash: MailboxHash,
fetches: &[FetchResponse<'_>],
) -> Result<()>;
fn envelopes(&mut self, mailbox_hash: MailboxHash) -> Result<Option<Vec<EnvelopeHash>>>;
fn clear(&mut self, mailbox_hash: MailboxHash, select_response: &SelectResponse) -> Result<()>;
fn rfc822(
&mut self,
identifier: std::result::Result<UID, EnvelopeHash>,
mailbox_hash: MailboxHash,
) -> Result<Option<Vec<u8>>>;
}
#[cfg(feature = "sqlite3")]
@ -71,28 +96,43 @@ mod sqlite3_m {
FromSql, FromSqlError, FromSqlResult, ToSql, ToSqlOutput,
};
use crate::sqlite3::{self, DatabaseDescription};
type Sqlite3UID = i32;
pub struct Sqlite3Cache {
connection: crate::sqlite3::Connection,
loaded_mailboxes: BTreeSet<MailboxHash>,
uid_store: Arc<UIDStore>,
}
const DB_DESCRIPTION: DatabaseDescription = DatabaseDescription {
name: "header_cache.db",
init_script: Some("PRAGMA foreign_keys = true;
init_script: Some(
"PRAGMA foreign_keys = true;
PRAGMA encoding = 'UTF-8';
CREATE TABLE IF NOT EXISTS envelopes (
mailbox_hash INTEGER,
uid INTEGER,
hash INTEGER NOT NULL,
mailbox_hash INTEGER NOT NULL,
uid INTEGER NOT NULL,
modsequence INTEGER,
rfc822 BLOB,
envelope BLOB NOT NULL,
PRIMARY KEY (mailbox_hash, uid),
FOREIGN KEY (mailbox_hash) REFERENCES uidvalidity(mailbox_hash) ON DELETE CASCADE
FOREIGN KEY (mailbox_hash) REFERENCES mailbox(mailbox_hash) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS uidvalidity (
uid INTEGER UNIQUE,
CREATE TABLE IF NOT EXISTS mailbox (
mailbox_hash INTEGER UNIQUE,
uidvalidity INTEGER,
flags BLOB NOT NULL,
highestmodseq INTEGER,
PRIMARY KEY (mailbox_hash, uid)
PRIMARY KEY (mailbox_hash)
);
CREATE INDEX IF NOT EXISTS envelope_idx ON envelopes(mailbox_hash);
CREATE INDEX IF NOT EXISTS uidvalidity_idx ON uidvalidity(mailbox_hash);"),
version: 1,
CREATE INDEX IF NOT EXISTS envelope_uid_idx ON envelopes(mailbox_hash, uid);
CREATE INDEX IF NOT EXISTS envelope_idx ON envelopes(hash);
CREATE INDEX IF NOT EXISTS mailbox_idx ON mailbox(mailbox_hash);",
),
version: 1,
};
impl ToSql for ModSequence {
@ -111,47 +151,108 @@ mod sqlite3_m {
}
}
impl CacheHandle {
pub fn get(uid_store: Arc<UIDStore>) -> Result<Self> {
Ok(Self {
impl Sqlite3Cache {
pub fn get(uid_store: Arc<UIDStore>) -> Result<Box<dyn ImapCache>> {
Ok(Box::new(Self {
connection: sqlite3::open_or_create_db(
&DB_DESCRIPTION,
Some(uid_store.account_name.as_str()),
)?,
loaded_mailboxes: BTreeSet::default(),
uid_store,
})
}))
}
pub fn mailbox_state(
&self,
mailbox_hash: MailboxHash,
) -> Result<Option<(UID, Option<ModSequence>)>> {
fn max_uid(&self, mailbox_hash: MailboxHash) -> Result<UID> {
let mut stmt = self
.connection
.prepare("SELECT uid, highestmodseq FROM uidvalidity WHERE mailbox_hash = ?1;")?;
.prepare("SELECT MAX(uid) FROM envelopes WHERE mailbox_hash = ?1;")?;
let mut ret: Vec<UID> = stmt
.query_map(sqlite3::params![mailbox_hash as i64], |row| {
Ok(row.get(0).map(|i: Sqlite3UID| i as UID)?)
})?
.collect::<std::result::Result<_, _>>()?;
Ok(ret.pop().unwrap_or(0))
}
}
impl ImapCache for Sqlite3Cache {
fn mailbox_state(&mut self, mailbox_hash: MailboxHash) -> Result<Option<()>> {
if self.loaded_mailboxes.contains(&mailbox_hash) {
return Ok(Some(()));
}
debug!("loading mailbox state {} from cache", mailbox_hash);
let mut stmt = self.connection.prepare(
"SELECT uidvalidity, flags, highestmodseq FROM mailbox WHERE mailbox_hash = ?1;",
)?;
let mut ret = stmt.query_map(sqlite3::params![mailbox_hash as i64], |row| {
Ok((row.get(0).map(|u: i64| u as usize)?, row.get(1)?))
Ok((
row.get(0).map(|u: Sqlite3UID| u as UID)?,
row.get(1)?,
row.get(2)?,
))
})?;
if let Some(row_res) = ret.next() {
Ok(Some(row_res?))
if let Some(v) = ret.next() {
let (uidvalidity, flags, highestmodseq): (
UIDVALIDITY,
Vec<u8>,
Option<ModSequence>,
) = v?;
debug!(
"mailbox state {} in cache uidvalidity {}",
mailbox_hash, uidvalidity
);
debug!(
"mailbox state {} in cache highestmodseq {:?}",
mailbox_hash, &highestmodseq
);
debug!(
"mailbox state {} inserting flags: {:?}",
mailbox_hash,
to_str!(&flags)
);
self.uid_store
.highestmodseqs
.lock()
.unwrap()
.entry(mailbox_hash)
.and_modify(|entry| *entry = highestmodseq.ok_or(()))
.or_insert(highestmodseq.ok_or(()));
self.uid_store
.uidvalidity
.lock()
.unwrap()
.entry(mailbox_hash)
.and_modify(|entry| *entry = uidvalidity)
.or_insert(uidvalidity);
let mut tag_lck = self.uid_store.tag_index.write().unwrap();
for f in to_str!(&flags).split('\0') {
let hash = tag_hash!(f);
//debug!("hash {} flag {}", hash, &f);
if !tag_lck.contains_key(&hash) {
tag_lck.insert(hash, f.to_string());
}
}
self.loaded_mailboxes.insert(mailbox_hash);
Ok(Some(()))
} else {
debug!("mailbox state {} not in cache", mailbox_hash);
Ok(None)
}
}
pub fn clear(
&self,
fn clear(
&mut self,
mailbox_hash: MailboxHash,
new_uidvalidity: UID,
highestmodseq: Option<ModSequence>,
select_response: &SelectResponse,
) -> Result<()> {
debug!("clear mailbox_hash {}", mailbox_hash);
debug!(new_uidvalidity);
debug!(&highestmodseq);
debug!("clear mailbox_hash {} {:?}", mailbox_hash, select_response);
self.loaded_mailboxes.remove(&mailbox_hash);
self.connection
.execute(
"DELETE FROM uidvalidity WHERE mailbox_hash = ?1",
"DELETE FROM mailbox WHERE mailbox_hash = ?1",
sqlite3::params![mailbox_hash as i64],
)
.chain_err_summary(|| {
@ -161,20 +262,38 @@ mod sqlite3_m {
)
})?;
self.connection.execute(
"INSERT OR IGNORE INTO uidvalidity (uid, highestmodseq, mailbox_hash) VALUES (?1, ?2, ?3)",
sqlite3::params![new_uidvalidity as i64, highestmodseq, mailbox_hash as i64],
if let Some(Ok(highestmodseq)) = select_response.highestmodseq {
self.connection.execute(
"INSERT OR IGNORE INTO mailbox (uidvalidity, flags, highestmodseq, mailbox_hash) VALUES (?1, ?2, ?3, ?4)",
sqlite3::params![select_response.uidvalidity as Sqlite3UID, select_response.flags.1.iter().map(|s| s.as_str()).collect::<Vec<&str>>().join("\0").as_bytes(), highestmodseq, mailbox_hash as i64],
)
.chain_err_summary(|| {
format!(
"Could not insert uidvalidity {} in header_cache of account {}",
new_uidvalidity, self.uid_store.account_name
select_response.uidvalidity, self.uid_store.account_name
)
})?;
} else {
self.connection
.execute(
"INSERT OR IGNORE INTO mailbox (uidvalidity, flags, mailbox_hash) VALUES (?1, ?2, ?3)",
sqlite3::params![
select_response.uidvalidity as Sqlite3UID,
select_response.flags.1.iter().map(|s| s.as_str()).collect::<Vec<&str>>().join("\0").as_bytes(),
mailbox_hash as i64
],
)
.chain_err_summary(|| {
format!(
"Could not insert mailbox {} in header_cache of account {}",
select_response.uidvalidity, self.uid_store.account_name
)
})?;
}
Ok(())
}
pub fn envelopes(&self, mailbox_hash: MailboxHash) -> Result<Option<Vec<EnvelopeHash>>> {
fn envelopes(&mut self, mailbox_hash: MailboxHash) -> Result<Option<Vec<EnvelopeHash>>> {
debug!("envelopes mailbox_hash {}", mailbox_hash);
if debug!(self.mailbox_state(mailbox_hash)?.is_none()) {
return Ok(None);
@ -187,7 +306,7 @@ mod sqlite3_m {
let ret: Vec<(UID, Envelope, Option<ModSequence>)> = stmt
.query_map(sqlite3::params![mailbox_hash as i64], |row| {
Ok((
row.get(0).map(|i: i64| i as usize)?,
row.get(0).map(|i: Sqlite3UID| i as UID)?,
row.get(1)?,
row.get(2)?,
))
@ -221,7 +340,7 @@ mod sqlite3_m {
Ok(Some(env_hashes))
}
pub fn insert_envelopes(
fn insert_envelopes(
&mut self,
mailbox_hash: MailboxHash,
fetches: &[FetchResponse<'_>],
@ -231,35 +350,22 @@ mod sqlite3_m {
mailbox_hash,
fetches.len()
);
let mut max_uid = self
.uid_store
.max_uids
.lock()
.unwrap()
.get(&mailbox_hash)
.cloned()
.unwrap_or_default();
if self.mailbox_state(mailbox_hash)?.is_none() {
debug!(self.mailbox_state(mailbox_hash)?.is_none());
let uidvalidity = self
.uid_store
.uidvalidity
.lock()
.unwrap()
.get(&mailbox_hash)
.cloned();
let highestmodseq = self
.uid_store
.highestmodseqs
.lock()
.unwrap()
.get(&mailbox_hash)
.cloned();
debug!(&uidvalidity);
debug!(&highestmodseq);
if let Some(uidvalidity) = uidvalidity {
debug!(self.clear(
mailbox_hash,
uidvalidity,
highestmodseq.and_then(|v| v.ok()),
))?;
}
return Err(MeliError::new("Mailbox is not in cache").set_kind(ErrorKind::Bug));
}
let Self {
ref mut connection,
ref uid_store,
loaded_mailboxes: _,
} = self;
let tx = connection.transaction()?;
for item in fetches {
@ -272,17 +378,23 @@ mod sqlite3_m {
envelope: Some(envelope),
} = item
{
max_uid = std::cmp::max(max_uid, *uid);
tx.execute(
"INSERT OR REPLACE INTO envelopes (uid, mailbox_hash, modsequence, envelope) VALUES (?1, ?2, ?3, ?4)",
sqlite3::params![*uid as i64, mailbox_hash as i64, modseq, &envelope],
"INSERT OR REPLACE INTO envelopes (hash, uid, mailbox_hash, modsequence, envelope) VALUES (?1, ?2, ?3, ?4, ?5)",
sqlite3::params![envelope.hash() as i64, *uid as Sqlite3UID, mailbox_hash as i64, modseq, &envelope],
).chain_err_summary(|| format!("Could not insert envelope {} {} in header_cache of account {}", envelope.message_id(), envelope.hash(), uid_store.account_name))?;
}
}
tx.commit()?;
self.uid_store
.max_uids
.lock()
.unwrap()
.insert(mailbox_hash, max_uid);
Ok(())
}
pub fn update(
fn update(
&mut self,
mailbox_hash: MailboxHash,
refresh_events: &[(UID, RefreshEvent)],
@ -294,33 +406,12 @@ mod sqlite3_m {
);
if self.mailbox_state(mailbox_hash)?.is_none() {
debug!(self.mailbox_state(mailbox_hash)?.is_none());
let uidvalidity = self
.uid_store
.uidvalidity
.lock()
.unwrap()
.get(&mailbox_hash)
.cloned();
let highestmodseq = self
.uid_store
.highestmodseqs
.lock()
.unwrap()
.get(&mailbox_hash)
.cloned();
debug!(&uidvalidity);
debug!(&highestmodseq);
if let Some(uidvalidity) = uidvalidity {
debug!(self.clear(
mailbox_hash,
uidvalidity,
highestmodseq.and_then(|v| v.ok()),
))?;
}
return Err(MeliError::new("Mailbox is not in cache").set_kind(ErrorKind::Bug));
}
let Self {
ref mut connection,
ref uid_store,
loaded_mailboxes: _,
} = self;
let tx = connection.transaction()?;
let mut hash_index_lck = uid_store.hash_index.lock().unwrap();
@ -330,7 +421,7 @@ mod sqlite3_m {
hash_index_lck.remove(&env_hash);
tx.execute(
"DELETE FROM envelopes WHERE mailbox_hash = ?1 AND uid = ?2;",
sqlite3::params![mailbox_hash as i64, *uid as i64],
sqlite3::params![mailbox_hash as i64, *uid as Sqlite3UID],
)
.chain_err_summary(|| {
format!(
@ -345,9 +436,10 @@ mod sqlite3_m {
)?;
let mut ret: Vec<Envelope> = stmt
.query_map(sqlite3::params![mailbox_hash as i64, *uid as i64], |row| {
Ok(row.get(0)?)
})?
.query_map(
sqlite3::params![mailbox_hash as i64, *uid as Sqlite3UID],
|row| Ok(row.get(0)?),
)?
.collect::<std::result::Result<_, _>>()?;
if let Some(mut env) = ret.pop() {
env.set_flags(*flags);
@ -355,7 +447,7 @@ mod sqlite3_m {
env.labels_mut().extend(tags.iter().map(|t| tag_hash!(t)));
tx.execute(
"UPDATE envelopes SET envelope = ?1 WHERE mailbox_hash = ?2 AND uid = ?3;",
sqlite3::params![&env, mailbox_hash as i64, *uid as i64],
sqlite3::params![&env, mailbox_hash as i64, *uid as Sqlite3UID],
)
.chain_err_summary(|| {
format!(
@ -377,48 +469,108 @@ mod sqlite3_m {
}
}
tx.commit()?;
let new_max_uid = self.max_uid(mailbox_hash)?;
self.uid_store
.max_uids
.lock()
.unwrap()
.insert(mailbox_hash, new_max_uid);
Ok(())
}
}
}
#[cfg(not(feature = "sqlite3"))]
pub use filesystem_m::*;
#[cfg(not(feature = "sqlite3"))]
mod filesystem_m {
use super::*;
impl CacheHandle {
pub fn get(uid_store: Arc<UIDStore>) -> Result<Self> {
Ok(Self { uid_store })
}
pub fn mailbox_state(
&self,
_mailbox_hash: MailboxHash,
) -> Result<Option<(UID, Option<ModSequence>)>> {
Ok(None)
}
fn find_envelope(
&mut self,
identifier: std::result::Result<UID, EnvelopeHash>,
mailbox_hash: MailboxHash,
) -> Result<Option<CachedEnvelope>> {
let mut ret: Vec<(UID, Envelope, Option<ModSequence>)> = match identifier {
Ok(uid) => {
let mut stmt = self.connection.prepare(
"SELECT uid, envelope, modsequence FROM envelopes WHERE mailbox_hash = ?1 AND uid = ?2;",
)?;
pub fn clear(
&self,
_mailbox_hash: MailboxHash,
_new_uidvalidity: UID,
_highestmodseq: Option<ModSequence>,
) -> Result<()> {
Ok(())
}
let x = stmt
.query_map(
sqlite3::params![mailbox_hash as i64, uid as Sqlite3UID],
|row| {
Ok((
row.get(0).map(|u: Sqlite3UID| u as UID)?,
row.get(1)?,
row.get(2)?,
))
},
)?
.collect::<std::result::Result<_, _>>()?;
x
}
Err(env_hash) => {
let mut stmt = self.connection.prepare(
"SELECT uid, envelope, modsequence FROM envelopes WHERE mailbox_hash = ?1 AND hash = ?2;",
)?;
pub fn envelopes(&self, _mailbox_hash: MailboxHash) -> Result<Option<Vec<EnvelopeHash>>> {
Ok(None)
let x = stmt
.query_map(
sqlite3::params![mailbox_hash as i64, env_hash as i64],
|row| {
Ok((
row.get(0).map(|u: Sqlite3UID| u as UID)?,
row.get(1)?,
row.get(2)?,
))
},
)?
.collect::<std::result::Result<_, _>>()?;
x
}
};
if ret.len() != 1 {
return Ok(None);
}
let (uid, inner, modsequence) = ret.pop().unwrap();
return Ok(Some(CachedEnvelope {
inner,
uid,
mailbox_hash,
modsequence,
}));
}
pub fn insert_envelopes(
fn rfc822(
&mut self,
_mailbox_hash: MailboxHash,
_fetches: &[FetchResponse<'_>],
) -> Result<()> {
Ok(())
identifier: std::result::Result<UID, EnvelopeHash>,
mailbox_hash: MailboxHash,
) -> Result<Option<Vec<u8>>> {
let mut ret: Vec<Option<Vec<u8>>> = match identifier {
Ok(uid) => {
let mut stmt = self.connection.prepare(
"SELECT rfc822 FROM envelopes WHERE mailbox_hash = ?1 AND uid = ?2;",
)?;
let x = stmt
.query_map(
sqlite3::params![mailbox_hash as i64, uid as Sqlite3UID],
|row| Ok(row.get(0)?),
)?
.collect::<std::result::Result<_, _>>()?;
x
}
Err(env_hash) => {
let mut stmt = self.connection.prepare(
"SELECT rfc822 FROM envelopes WHERE mailbox_hash = ?1 AND hash = ?2;",
)?;
let x = stmt
.query_map(
sqlite3::params![mailbox_hash as i64, env_hash as i64],
|row| Ok(row.get(0)?),
)?
.collect::<std::result::Result<_, _>>()?;
x
}
};
if ret.len() != 1 {
return Ok(None);
}
Ok(ret.pop().unwrap())
}
}
}
@ -428,7 +580,6 @@ pub(super) async fn fetch_cached_envs(state: &mut FetchState) -> Result<Option<V
stage: _,
ref mut connection,
mailbox_hash,
can_create_flags: _,
ref uid_store,
} = state;
debug!(uid_store.keep_offline_cache);
@ -481,3 +632,59 @@ pub(super) async fn fetch_cached_envs(state: &mut FetchState) -> Result<Option<V
}
}
}
#[cfg(not(feature = "sqlite3"))]
pub use default_m::*;
#[cfg(not(feature = "sqlite3"))]
mod default_m {
pub struct DefaultCache;
impl DefaultCache {
pub fn get(_uid_store: Arc<UIDStore>) -> Result<Box<dyn ImapCache>> {
Ok(Box::new(Self))
}
}
impl ImapCache for DefaultCache {
fn mailbox_state(&mut self, _mailbox_hash: MailboxHash) -> Result<Option<()>> {
Err(MeliError::new("melib is not built with any imap cache").set_kind(ErrorKind::Bug))
}
fn clear(
&mut self,
_mailbox_hash: MailboxHash,
_select_response: &SelectResponse,
) -> Result<()> {
Err(MeliError::new("melib is not built with any imap cache").set_kind(ErrorKind::Bug))
}
fn envelopes(&mut self, _mailbox_hash: MailboxHash) -> Result<Option<Vec<EnvelopeHash>>> {
Err(MeliError::new("melib is not built with any imap cache").set_kind(ErrorKind::Bug))
}
fn insert_envelopes(
&mut self,
_mailbox_hash: MailboxHash,
_fetches: &[FetchResponse<'_>],
) -> Result<()> {
Err(MeliError::new("melib is not built with any imap cache").set_kind(ErrorKind::Bug))
}
fn update(
&mut self,
_mailbox_hash: MailboxHash,
_refresh_events: &[(UID, RefreshEvent)],
) -> Result<()> {
Err(MeliError::new("melib is not built with any imap cache").set_kind(ErrorKind::Bug))
}
fn find_envelope(
&mut self,
_identifier: std::result::Result<UID, EnvelopeHash>,
_mailbox_hash: MailboxHash,
) -> Result<Option<CachedEnvelope>> {
Err(MeliError::new("melib is not built with any imap cache").set_kind(ErrorKind::Bug))
}
}
}

81
melib/src/backends/imap/cache/sync.rs

@ -29,7 +29,10 @@ impl ImapConnection {
return Ok(None);
}
let cache_handle = CacheHandle::get(self.uid_store.clone())?;
#[cfg(not(feature = "sqlite3"))]
let mut cache_handle = DefaultCache::get(self.uid_store.clone())?;
#[cfg(feature = "sqlite3")]
let mut cache_handle = Sqlite3Cache::get(self.uid_store.clone())?;
if cache_handle.mailbox_state(mailbox_hash)?.is_none() {
return Ok(None);
}
@ -52,29 +55,23 @@ impl ImapConnection {
mailbox_hash: MailboxHash,
) -> Option<Result<Vec<EnvelopeHash>>> {
debug!("load_cache {}", mailbox_hash);
let cache_handle = match CacheHandle::get(self.uid_store.clone()) {
#[cfg(not(feature = "sqlite3"))]
let mut cache_handle = match DefaultCache::get(self.uid_store.clone()) {
Ok(v) => v,
Err(err) => return Some(Err(err)),
};
let (uidvalidity, highestmodseq) = match debug!(cache_handle.mailbox_state(mailbox_hash)) {
#[cfg(feature = "sqlite3")]
let mut cache_handle = match Sqlite3Cache::get(self.uid_store.clone()) {
Ok(v) => v,
Err(err) => return Some(Err(err)),
};
match debug!(cache_handle.mailbox_state(mailbox_hash)) {
Err(err) => return Some(Err(err)),
Ok(Some(v)) => v,
Ok(Some(())) => {}
Ok(None) => {
return None;
}
};
self.uid_store
.uidvalidity
.lock()
.unwrap()
.entry(mailbox_hash)
.or_insert(uidvalidity);
self.uid_store
.highestmodseqs
.lock()
.unwrap()
.entry(mailbox_hash)
.or_insert(highestmodseq.ok_or(()));
match debug!(cache_handle.envelopes(mailbox_hash)) {
Ok(Some(envs)) => Some(Ok(envs)),
Ok(None) => None,
@ -84,7 +81,7 @@ impl ImapConnection {
pub async fn build_cache(
&mut self,
cache_handle: &mut CacheHandle,
cache_handle: &mut Box<dyn ImapCache>,
mailbox_hash: MailboxHash,
) -> Result<()> {
debug!("build_cache {}", mailbox_hash);
@ -111,11 +108,7 @@ impl ImapConnection {
.unwrap()
.insert(mailbox_hash, v);
}
cache_handle.clear(
mailbox_hash,
select_response.uidvalidity,
select_response.highestmodseq.and_then(|i| i.ok()),
)?;
cache_handle.clear(mailbox_hash, &select_response)?;
self.send_command(b"UID FETCH 1:* (UID FLAGS ENVELOPE BODYSTRUCTURE)")
.await?;
self.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
@ -128,18 +121,11 @@ impl ImapConnection {
//rfc4549_Synchronization_Operations_for_Disconnected_IMAP4_Clients
pub async fn resync_basic(
&mut self,
mut cache_handle: CacheHandle,
mut cache_handle: Box<dyn ImapCache>,
mailbox_hash: MailboxHash,
) -> Result<Option<Vec<Envelope>>> {
let mut payload = vec![];
debug!("resync_basic");
debug!(self
.uid_store
.uidvalidity
.lock()
.unwrap()
.get(&mailbox_hash));
debug!(self.uid_store.max_uids.lock().unwrap().get(&mailbox_hash));
let mut response = String::with_capacity(8 * 1024);
let cached_uidvalidity = self
.uid_store
@ -182,11 +168,7 @@ impl ImapConnection {
);
// 1. check UIDVALIDITY. If fail, discard cache and rebuild
if select_response.uidvalidity != current_uidvalidity {
cache_handle.clear(
mailbox_hash,
select_response.uidvalidity,
select_response.highestmodseq.and_then(|i| i.ok()),
)?;
cache_handle.clear(mailbox_hash, &select_response)?;
return Ok(None);
}
@ -234,7 +216,6 @@ impl ImapConnection {
}
}
{
let mut cache_handle = cache::CacheHandle::get(self.uid_store.clone())?;
debug!(cache_handle
.insert_envelopes(mailbox_hash, &v)
.chain_err_summary(|| {
@ -363,18 +344,11 @@ impl ImapConnection {
//Section 6.1
pub async fn resync_condstore(
&mut self,
mut cache_handle: CacheHandle,
mut cache_handle: Box<dyn ImapCache>,
mailbox_hash: MailboxHash,
) -> Result<Option<Vec<Envelope>>> {
let mut payload = vec![];
debug!("resync_condstore");
debug!(self
.uid_store
.uidvalidity
.lock()
.unwrap()
.get(&mailbox_hash));
debug!(self.uid_store.max_uids.lock().unwrap().get(&mailbox_hash));
let mut response = String::with_capacity(8 * 1024);
let cached_uidvalidity = self
.uid_store
@ -397,6 +371,9 @@ impl ImapConnection {
.unwrap()
.get(&mailbox_hash)
.cloned();
debug!(&cached_uidvalidity);
debug!(&cached_max_uid);
debug!(&cached_highestmodseq);
if cached_uidvalidity.is_none()
|| cached_max_uid.is_none()
|| cached_highestmodseq.is_none()
@ -444,11 +421,7 @@ impl ImapConnection {
// mailbox (note that this doesn't affect actions performed on
// client-generated fake UIDs; see Section 5); and
// * skip steps 1b and 2-II;
cache_handle.clear(
mailbox_hash,
select_response.uidvalidity,
select_response.highestmodseq.and_then(|i| i.ok()),
)?;
cache_handle.clear(mailbox_hash, &select_response)?;
return Ok(None);
}
if select_response.highestmodseq.is_none()
@ -525,7 +498,6 @@ impl ImapConnection {
}
}
{
let mut cache_handle = cache::CacheHandle::get(self.uid_store.clone())?;
debug!(cache_handle
.insert_envelopes(mailbox_hash, &v)
.chain_err_summary(|| {
@ -668,7 +640,7 @@ impl ImapConnection {
//rfc7162_Quick Flag Changes Resynchronization (CONDSTORE)_and Quick Mailbox Resynchronization (QRESYNC)
pub async fn resync_condstoreqresync(
&mut self,
_cache_handle: CacheHandle,
_cache_handle: Box<dyn ImapCache>,
_mailbox_hash: MailboxHash,
) -> Result<Option<Vec<Envelope>>> {
Ok(None)
@ -706,6 +678,13 @@ impl ImapConnection {
.or_insert(select_response.uidvalidity);
*v = select_response.uidvalidity;
}
{
if let Some(highestmodseq) = select_response.highestmodseq {
let mut highestmodseqs = self.uid_store.highestmodseqs.lock().unwrap();
let v = highestmodseqs.entry(mailbox_hash).or_insert(highestmodseq);
*v = highestmodseq;
}
}
let mut permissions = permissions.lock().unwrap();
permissions.create_messages = !select_response.read_only;
permissions.remove_messages = !select_response.read_only;

28
melib/src/backends/imap/connection.rs

@ -762,6 +762,34 @@ impl ImapConnection {
.await?;
debug!("select response {}", ret);
let select_response = protocol_parser::select_response(&ret)?;
{
if self.uid_store.keep_offline_cache {
#[cfg(not(feature = "sqlite3"))]
let mut cache_handle = super::cache::DefaultCache::get(self.uid_store.clone())?;
#[cfg(feature = "sqlite3")]
let mut cache_handle = super::cache::Sqlite3Cache::get(self.uid_store.clone())?;
if let Err(err) = cache_handle.mailbox_state(mailbox_hash).and_then(|r| {
if r.is_none() {
cache_handle.clear(mailbox_hash, &select_response)
} else {
Ok(())
}
}) {
(self.uid_store.event_consumer)(
self.uid_store.account_hash,
crate::backends::BackendEvent::from(err),
);
}
}
self.uid_store
.mailboxes
.lock()
.await
.entry(mailbox_hash)
.and_modify(|entry| {
*entry.select.write().unwrap() = Some(select_response.clone());
});
}
{
let mut permissions = permissions.lock().unwrap();
permissions.create_messages = !select_response.read_only;

15
melib/src/backends/imap/mailbox.rs

@ -18,6 +18,8 @@
* You should have received a copy of the GNU General Public License
* along with meli. If not, see <http://www.gnu.org/licenses/>.
*/
use super::protocol_parser::SelectResponse;
use crate::backends::{
BackendMailbox, Mailbox, MailboxHash, MailboxPermissions, SpecialUsageMailbox,
};
@ -95,14 +97,15 @@ fn test_lazy_count_set() {
#[derive(Debug, Default, Clone)]
pub struct ImapMailbox {
pub(super) hash: MailboxHash,
pub(super) imap_path: String,
pub(super) path: String,
pub(super) name: String,
pub(super) parent: Option<MailboxHash>,
pub(super) children: Vec<MailboxHash>,
pub hash: MailboxHash,
pub imap_path: String,
pub path: String,
pub name: String,
pub parent: Option<MailboxHash>,
pub children: Vec<MailboxHash>,
pub separator: u8,
pub usage: Arc<RwLock<SpecialUsageMailbox>>,
pub select: Arc<RwLock<Option<SelectResponse>>>,
pub no_select: bool,
pub is_subscribed: bool,

14
melib/src/backends/imap/operations.rs

@ -29,7 +29,7 @@ use std::sync::Arc;
/// `BackendOp` implementor for Imap
#[derive(Debug, Clone)]
pub struct ImapOp {
uid: usize,
uid: UID,
mailbox_hash: MailboxHash,
connection: Arc<FutureMutex<ImapConnection>>,
uid_store: Arc<UIDStore>,
@ -37,7 +37,7 @@ pub struct ImapOp {
impl ImapOp {
pub fn new(
uid: usize,
uid: UID,
mailbox_hash: MailboxHash,
connection: Arc<FutureMutex<ImapConnection>>,
uid_store: Arc<UIDStore>,
@ -80,12 +80,20 @@ impl BackendOp for ImapOp {
response.len(),
response.lines().count()
);
let mut results = protocol_parser::fetch_responses(&response)?.1;
if results.len() != 1 {
return Err(MeliError::new(format!(
"Invalid/unexpected response: {:?}",
response
))
.set_summary(format!("message with UID {} was not found?", uid)));
}
let FetchResponse {
uid: _uid,
flags: _flags,
body,
..
} = protocol_parser::fetch_response(&response)?.1;
} = results.pop().unwrap();
let _uid = _uid.unwrap();
assert_eq!(_uid, uid);
assert!(body.is_some());

87
melib/src/backends/imap/protocol_parser.rs

@ -185,7 +185,7 @@ pub enum ResponseCode {
/// Followed by a decimal number, indicates the unique identifier validity value. Refer to section 2.3.1.1 for more information.
Uidvalidity(UID),
/// Followed by a decimal number, indicates the number of the first message without the \Seen flag set.
Unseen(usize),
Unseen(ImapNum),
}
impl std::fmt::Display for ResponseCode {
@ -452,7 +452,7 @@ pub fn list_mailbox_result(input: &[u8]) -> IResult<&[u8], ImapMailbox> {
#[derive(Debug, Clone, PartialEq)]
pub struct FetchResponse<'a> {
pub uid: Option<UID>,
pub message_sequence_number: usize,
pub message_sequence_number: MessageSequenceNumber,
pub modseq: Option<ModSequence>,
pub flags: Option<(Flag, Vec<String>)>,
pub body: Option<&'a [u8]>,
@ -516,7 +516,7 @@ pub fn fetch_response(input: &str) -> ImapParseResult<FetchResponse<'_>> {
while input.as_bytes()[i].is_ascii_digit() {
let b: u8 = input.as_bytes()[i] - 0x30;
ret.message_sequence_number *= 10;
ret.message_sequence_number += b as usize;
ret.message_sequence_number += b as MessageSequenceNumber;
i += 1;
bounds!();
}
@ -537,7 +537,7 @@ pub fn fetch_response(input: &str) -> ImapParseResult<FetchResponse<'_>> {
{
i += input.len() - i - rest.len();
ret.uid =
Some(usize::from_str(unsafe { std::str::from_utf8_unchecked(uid) }).unwrap());
Some(UID::from_str(unsafe { std::str::from_utf8_unchecked(uid) }).unwrap());
} else {
return debug!(Err(MeliError::new(format!(