melib/imap: fix imap cache without sqlite3 not compiling

RFC3501 seems to prefer BODY attributes as the more modern equivalent
to RFC822. For example, this also allows us to use BODY.PEEK. Since the
fetch methods in melib are tightly coupled with the parser, we have to
add this here if we want to use BODY attributes during fetch.

contrib/oauth2.py

@@ -55,8 +55,8 @@ option.
   oauth2 --generate_oauth2_string --user=xxx@gmail.com \
     --access_token=ya29.AGy[...]ezLg
 
-The output of this mode will be a base64-encoded string. To use it, connect to a
-IMAPFE and pass it as the second argument to the AUTHENTICATE command.
+To use the output of this mode, connect to a IMAPFE and pass the base64
+encoding of this string as the second argument to the AUTHENTICATE command.
 
   a AUTHENTICATE XOAUTH2 a9sha9sfs[...]9dfja929dk==
 """
@@ -240,7 +240,7 @@ def RefreshToken(client_id, client_secret, refresh_token):
   return json.loads(response)
 
 
-def GenerateOAuth2String(username, access_token, base64_encode=True):
+def GenerateOAuth2String(username, access_token):
   """Generates an IMAP OAuth2 authentication string.
 
   See https://developers.google.com/google-apps/gmail/oauth2_overview
@@ -248,14 +248,11 @@ def GenerateOAuth2String(username, access_token, base64_encode=True):
   Args:
     username: the username (email address) of the account to authenticate
     access_token: An OAuth2 access token.
-    base64_encode: Whether to base64-encode the output.
 
   Returns:
     The SASL argument for the OAuth2 mechanism.
   """
   auth_string = 'user=%s\1auth=Bearer %s\1\1' % (username, access_token)
-  if base64_encode:
-    auth_string = base64.b64encode(bytes(auth_string, 'utf-8'))
   return auth_string
 
 
@@ -331,13 +328,11 @@ def main(argv):
   elif options.test_imap_authentication:
     RequireOptions(options, 'user', 'access_token')
     TestImapAuthentication(options.user,
-        GenerateOAuth2String(options.user, options.access_token,
-                             base64_encode=False))
+        GenerateOAuth2String(options.user, options.access_token))
   elif options.test_smtp_authentication:
     RequireOptions(options, 'user', 'access_token')
     TestSmtpAuthentication(options.user,
-        GenerateOAuth2String(options.user, options.access_token,
-                             base64_encode=False))
+        GenerateOAuth2String(options.user, options.access_token))
   else:
     options_parser.print_help()
     print('Nothing to do, exiting.')

melib/src/backends.rs

@@ -467,7 +467,7 @@ pub trait MailBackend: ::std::fmt::Debug + Send + Sync {
 /// let operation = Box::new(FooOp {});
 /// ```
 pub trait BackendOp: ::std::fmt::Debug + ::std::marker::Send {
-    fn as_bytes(&mut self) -> ResultFuture<Vec<u8>>;
+    fn as_bytes(&self) -> ResultFuture<Vec<u8>>;
     fn fetch_flags(&self) -> ResultFuture<Flag>;
 }
 
@@ -487,7 +487,7 @@ impl ReadOnlyOp {
 }
 
 impl BackendOp for ReadOnlyOp {
-    fn as_bytes(&mut self) -> ResultFuture<Vec<u8>> {
+    fn as_bytes(&self) -> ResultFuture<Vec<u8>> {
         self.op.as_bytes()
     }
     fn fetch_flags(&self) -> ResultFuture<Flag> {

melib/src/backends/imap.rs

@@ -32,7 +32,7 @@ pub use connection::*;
 mod watch;
 pub use watch::*;
 mod cache;
-use cache::{ImapCacheReset, ModSequence};
+pub use cache::{ImapCacheReset, ModSequence};
 pub mod managesieve;
 mod untagged;
 
@@ -51,6 +51,7 @@ use futures::stream::Stream;
 use std::collections::hash_map::DefaultHasher;
 use std::collections::{BTreeSet, HashMap, HashSet};
 use std::convert::TryFrom;
+use std::convert::TryInto;
 use std::hash::Hasher;
 use std::pin::Pin;
 use std::str::FromStr;
@@ -58,8 +59,9 @@ use std::sync::{Arc, Mutex};
 use std::time::{Duration, SystemTime};
 
 pub type ImapNum = usize;
-pub type UID = ImapNum;
-pub type UIDVALIDITY = UID;
+// Data type for UID and UIDVALIDITY defined in RFC 3501 2.3.1.1
+pub type UID = u32;
+pub type UIDVALIDITY = u32;
 pub type MessageSequenceNumber = ImapNum;
 
 pub static SUPPORTED_CAPABILITIES: &[&str] = &[
@@ -156,7 +158,7 @@ pub struct UIDStore {
 }
 
 impl UIDStore {
-    fn new(
+    pub fn new(
         account_hash: AccountHash,
         account_name: Arc<String>,
         event_consumer: BackendEventConsumer,
@@ -469,12 +471,9 @@ impl MailBackend for ImapType {
                 ImapProtocol::IMAP {
                     extension_use: ImapExtensionUse { idle, .. },
                 } => {
-                    idle && uid_store
-                        .capabilities
-                        .lock()
-                        .unwrap()
-                        .iter()
-                        .any(|cap| cap.eq_ignore_ascii_case(b"IDLE"))
+                    let main_conn_lck = timeout(uid_store.timeout, main_conn.lock()).await?;
+
+                    idle && main_conn_lck.has_capability("IDLE".to_string())
                 }
                 _ => false,
             };
@@ -1260,14 +1259,45 @@ impl ImapType {
         let server_username = get_conf_val!(s["server_username"])?;
         let use_oauth2: bool = get_conf_val!(s["use_oauth2"], false)?;
 
-        if use_oauth2 && !s.extra.contains_key("server_password_command") {
-            return Err(Error::new(format!(
-                "({}) `use_oauth2` use requires `server_password_command` set with a command that returns an OAUTH2 token. Consult documentation for guidance.",
-                s.name,
-            )));
+        if use_oauth2 {
+            if !s.extra.contains_key("server_password_command")
+                && !s.extra.contains_key("server_password")
+            {
+                return Err(Error::new(format!(
+                    "({}) `use_oauth2` use requires either `server_password` set or `server_password_command` set with a command that returns an OAUTH2 token. Consult documentation for guidance.",
+                    s.name,
+                )));
+            } else if s.extra.contains_key("server_password_command")
+                && s.extra.contains_key("server_password")
+            {
+                return Err(Error::new(format!(
+                    "Configuration error ({}): both server_password and server_password_command are set, cannot choose",
+                    s.name,
+                )));
+            }
         }
 
-        let server_password = s.server_password()?;
+        let server_password = if !s.extra.contains_key("server_password_command") {
+            get_conf_val!(s["server_password"])?.to_string()
+        } else {
+            let invocation = get_conf_val!(s["server_password_command"])?;
+            let output = std::process::Command::new("sh")
+                .args(&["-c", invocation])
+                .stdin(std::process::Stdio::piped())
+                .stdout(std::process::Stdio::piped())
+                .stderr(std::process::Stdio::piped())
+                .output()?;
+            if !output.status.success() {
+                return Err(Error::new(format!(
+                    "({}) server_password_command `{}` returned {}: {}",
+                    s.name,
+                    get_conf_val!(s["server_password_command"])?,
+                    output.status,
+                    String::from_utf8_lossy(&output.stderr)
+                )));
+            }
+            std::str::from_utf8(&output.stdout)?.trim_end().to_string()
+        };
         let server_port = get_conf_val!(s["server_port"], 143)?;
         let use_tls = get_conf_val!(s["use_tls"], true)?;
         let use_starttls = use_tls && get_conf_val!(s["use_starttls"], server_port != 993)?;
@@ -1521,21 +1551,31 @@ impl ImapType {
         get_conf_val!(s["server_username"])?;
         let use_oauth2: bool = get_conf_val!(s["use_oauth2"], false)?;
         keys.insert("server_password_command");
-        if !s.extra.contains_key("server_password_command") {
-            if use_oauth2 {
+        if use_oauth2 {
+            if !s.extra.contains_key("server_password_command")
+                && !s.extra.contains_key("server_password")
+            {
                 return Err(Error::new(format!(
-                    "({}) `use_oauth2` use requires `server_password_command` set with a command that returns an OAUTH2 token. Consult documentation for guidance.",
+                    "({}) `use_oauth2` use requires either `server_password` set or `server_password_command` set with a command that returns an OAUTH2 token. Consult documentation for guidance.",
                     s.name,
                 )));
-            }
-            get_conf_val!(s["server_password"])?;
-        } else if s.extra.contains_key("server_password") {
-            return Err(Error::new(format!(
+            } else if s.extra.contains_key("server_password_command")
+                && s.extra.contains_key("server_password")
+            {
+                return Err(Error::new(format!(
+                    "Configuration error ({}): both server_password and server_password_command are set, cannot choose",
+                    s.name)));
+            } else if s.extra.contains_key("server_password") {
+                return Err(Error::new(format!(
                 "Configuration error ({}): both server_password and server_password_command are set, cannot choose",
                 s.name.as_str(),
             )));
+            }
         }
+        let _ = get_conf_val!(s["server_password"]);
         let _ = get_conf_val!(s["server_password_command"]);
+        get_conf_val!(s["server_port"], 143)?;
+
         get_conf_val!(s["server_port"], 143)?;
         let use_tls = get_conf_val!(s["use_tls"], true)?;
         let use_starttls = get_conf_val!(s["use_starttls"], false)?;
@@ -1735,9 +1775,9 @@ async fn fetch_hlpr(state: &mut FetchState) -> Result<Vec<Envelope>> {
                 let mut conn = connection.lock().await;
                 let mut response = Vec::with_capacity(8 * 1024);
                 let max_uid_left = max_uid;
-                let chunk_size = 250;
+                let chunk_size: UID = 250;
 
-                let mut envelopes = Vec::with_capacity(chunk_size);
+                let mut envelopes = Vec::with_capacity(chunk_size.try_into().unwrap_or_default());
                 conn.examine_mailbox(mailbox_hash, &mut response, false)
                     .await?;
                 if max_uid_left > 0 {

melib/src/backends/imap/cache.rs

@@ -684,20 +684,20 @@ mod default_m {
     pub struct DefaultCache;
 
     impl DefaultCache {
-        pub fn get(_uid_store: Arc<UIDStore>) -> Result<Box<dyn ImapCache>> {
+        pub fn get(_: Arc<UIDStore>) -> Result<Box<dyn ImapCache>> {
             Ok(Box::new(Self))
         }
     }
 
     impl ImapCacheReset for DefaultCache {
-        fn reset_db(uid_store: &UIDStore) -> Result<()> {
+        fn reset_db(_: &UIDStore) -> Result<()> {
             Err(Error::new("melib is not built with any imap cache").set_kind(ErrorKind::Bug))
         }
     }
 
     impl ImapCache for DefaultCache {
         fn reset(&mut self) -> Result<()> {
-            DefaultCache::reset_db(&self.uid_store)
+            Err(Error::new("melib is not built with any imap cache").set_kind(ErrorKind::Bug))
         }
 
         fn mailbox_state(&mut self, _mailbox_hash: MailboxHash) -> Result<Option<()>> {

melib/src/backends/imap/connection.rs

@@ -20,7 +20,7 @@
  */
 
 use super::protocol_parser::{ImapLineSplit, ImapResponse, RequiredResponses, SelectResponse};
-use crate::backends::{MailboxHash, RefreshEvent};
+use crate::backends::{AccountHash, BackendEventConsumer, MailboxHash, RefreshEvent};
 use crate::connections::{lookup_ipv4, timeout, Connection};
 use crate::email::parser::BytesExt;
 use crate::error::*;
@@ -33,9 +33,11 @@ use std::convert::TryFrom;
 use std::future::Future;
 use std::iter::FromIterator;
 use std::pin::Pin;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 use std::time::{Duration, Instant, SystemTime};
 
+use data_encoding::BASE64;
+
 const IMAP_PROTOCOL_TIMEOUT: Duration = Duration::from_secs(60 * 28);
 
 use super::protocol_parser;
@@ -110,6 +112,12 @@ pub struct ImapConnection {
     pub server_conf: ImapServerConf,
     pub sync_policy: SyncPolicy,
     pub uid_store: Arc<UIDStore>,
+
+    pub account_hash: AccountHash,
+    pub account_name: Arc<String>,
+    pub capabilities: Arc<Mutex<Capabilities>>,
+    pub is_online: Arc<Mutex<(SystemTime, Result<()>)>>,
+    pub event_consumer: BackendEventConsumer,
 }
 
 impl ImapStream {
@@ -268,7 +276,6 @@ impl ImapStream {
             timeout: server_conf.timeout,
         };
         if let ImapProtocol::ManageSieve = server_conf.protocol {
-            use data_encoding::BASE64;
             ret.read_response(&mut res).await?;
             ret.send_command(
                 format!(
@@ -357,7 +364,11 @@ impl ImapStream {
                     )));
                 }
                 ret.send_command(
-                    format!("AUTHENTICATE XOAUTH2 {}", &server_conf.server_password).as_bytes(),
+                    format!(
+                        "AUTHENTICATE XOAUTH2 {}",
+                        BASE64.encode(server_conf.server_password.as_bytes())
+                    )
+                    .as_bytes(),
                 )
                 .await?;
             }
@@ -560,20 +571,24 @@ impl ImapConnection {
                 SyncPolicy::None
             },
             uid_store,
+            account_hash: AccountHash(0),
+            account_name: Arc::new("".to_string()),
+            capabilities: Default::default(),
+            is_online: Arc::new(Mutex::new((
+                SystemTime::now(),
+                Err(Error::new("Account is uninitialised.")),
+            ))),
+            event_consumer: BackendEventConsumer::new(Arc::new(|_, _| {})),
         }
     }
 
     pub fn connect<'a>(&'a mut self) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'a>> {
         Box::pin(async move {
-            if let (time, ref mut status @ Ok(())) = *self.uid_store.is_online.lock().unwrap() {
+            if let (time, ref mut status @ Ok(())) = *self.is_online.lock().unwrap() {
                 if SystemTime::now().duration_since(time).unwrap_or_default()
                     >= IMAP_PROTOCOL_TIMEOUT
                 {
-                    let err = Error::new(format!(
-                        "Connection timed out after {} seconds",
-                        IMAP_PROTOCOL_TIMEOUT.as_secs()
-                    ))
-                    .set_kind(ErrorKind::Timeout);
+                    let err = Error::new("Connection timed out").set_kind(ErrorKind::Timeout);
                     *status = Err(err.clone());
                     self.stream = Err(err);
                 }
@@ -598,9 +613,9 @@ impl ImapConnection {
             }
             let new_stream = ImapStream::new_connection(&self.server_conf, &self.uid_store).await;
             if let Err(err) = new_stream.as_ref() {
-                self.uid_store.is_online.lock().unwrap().1 = Err(err.clone());
+                self.is_online.lock().unwrap().1 = Err(err.clone());
             } else {
-                *self.uid_store.is_online.lock().unwrap() = (SystemTime::now(), Ok(()));
+                *self.is_online.lock().unwrap() = (SystemTime::now(), Ok(()));
             }
             let (capabilities, stream) = new_stream?;
             self.stream = Ok(stream);
@@ -672,7 +687,7 @@ impl ImapConnection {
                 }
                 ImapProtocol::ManageSieve => {}
             }
-            *self.uid_store.capabilities.lock().unwrap() = capabilities;
+            *self.capabilities.lock().unwrap() = capabilities;
             Ok(())
         })
     }
@@ -686,7 +701,7 @@ impl ImapConnection {
             let mut response = Vec::new();
             ret.clear();
             self.stream.as_mut()?.read_response(&mut response).await?;
-            *self.uid_store.is_online.lock().unwrap() = (SystemTime::now(), Ok(()));
+            *self.is_online.lock().unwrap() = (SystemTime::now(), Ok(()));
 
             match self.server_conf.protocol {
                 ImapProtocol::IMAP { .. } => {
@@ -715,8 +730,8 @@ impl ImapConnection {
                                 response_code,
                                 String::from_utf8_lossy(&response)
                             );
-                            (self.uid_store.event_consumer)(
-                                self.uid_store.account_hash,
+                            (self.event_consumer)(
+                                self.account_hash,
                                 crate::backends::BackendEvent::Notice {
                                     description: response_code.to_string(),
                                     content: None,
@@ -732,8 +747,8 @@ impl ImapConnection {
                                 response_code,
                                 String::from_utf8_lossy(&response)
                             );
-                            (self.uid_store.event_consumer)(
-                                self.uid_store.account_hash,
+                            (self.event_consumer)(
+                                self.account_hash,
                                 crate::backends::BackendEvent::Notice {
                                     description: response_code.to_string(),
                                     content: None,
@@ -795,7 +810,7 @@ impl ImapConnection {
             }
             Err(err)
         } else {
-            *self.uid_store.is_online.lock().unwrap() = (SystemTime::now(), Ok(()));
+            *self.is_online.lock().unwrap() = (SystemTime::now(), Ok(()));
             Ok(())
         }
     }
@@ -825,93 +840,52 @@ impl ImapConnection {
         }
     }
 
+    pub fn has_capability(&self, capability: String) -> bool {
+        self.capabilities
+            .lock()
+            .unwrap()
+            .iter()
+            .any(|cap| cap.eq_ignore_ascii_case(&capability.as_bytes()))
+    }
+
+    pub async fn select(&mut self, imap_path: String) -> Result<SelectResponse> {
+        self.send_command(format!("SELECT \"{}\"", imap_path).as_bytes())
+            .await?;
+
+        let mut ret = Vec::new();
+        self.read_response(&mut ret, RequiredResponses::SELECT_REQUIRED)
+            .await?;
+
+        let select_response = protocol_parser::select_response(&ret).chain_err_summary(|| {
+            format!("Could not parse select response for mailbox {}", imap_path)
+        })?;
+
+        Ok(select_response)
+    }
+
+    pub async fn uid_fetch<'a>(
+        &mut self,
+        range: String,
+        items_to_fetch: String,
+        response: &'a mut Vec<u8>, //@TODO ideally instead of having to pass response from the outside, it's better if FetchResponse doesn't borrow response so we can return it by from a function
+    ) -> Result<Vec<protocol_parser::FetchResponse<'a>>> {
+        self.send_command(format!("UID FETCH {} {}", range, items_to_fetch).as_bytes())
+            .await?;
+        self.read_response(response, RequiredResponses::FETCH_REQUIRED)
+            .await?;
+
+        let (_, mut envelopes_list, _) = protocol_parser::fetch_responses(response)?;
+
+        Ok(envelopes_list)
+    }
+
     pub async fn select_mailbox(
         &mut self,
         mailbox_hash: MailboxHash,
         ret: &mut Vec<u8>,
         force: bool,
     ) -> Result<Option<SelectResponse>> {
-        if !force && self.stream.as_ref()?.current_mailbox == MailboxSelection::Select(mailbox_hash)
-        {
-            return Ok(None);
-        }
-        let (imap_path, no_select, permissions) = {
-            let m = &self.uid_store.mailboxes.lock().await[&mailbox_hash];
-            (
-                m.imap_path().to_string(),
-                m.no_select,
-                m.permissions.clone(),
-            )
-        };
-        if no_select {
-            return Err(Error::new(format!(
-                "Trying to select a \\NoSelect mailbox: {}",
-                &imap_path
-            ))
-            .set_kind(crate::error::ErrorKind::Bug));
-        }
-        self.send_command(format!("SELECT \"{}\"", imap_path).as_bytes())
-            .await?;
-        self.read_response(ret, RequiredResponses::SELECT_REQUIRED)
-            .await?;
-        debug!(
-            "{} select response {}",
-            imap_path,
-            String::from_utf8_lossy(ret)
-        );
-        let select_response = protocol_parser::select_response(ret).chain_err_summary(|| {
-            format!("Could not parse select response for mailbox {}", imap_path)
-        })?;
-        {
-            if self.uid_store.keep_offline_cache {
-                #[cfg(not(feature = "sqlite3"))]
-                let mut cache_handle = super::cache::DefaultCache::get(self.uid_store.clone())?;
-                #[cfg(feature = "sqlite3")]
-                let mut cache_handle = super::cache::Sqlite3Cache::get(self.uid_store.clone())?;
-                if let Err(err) = cache_handle.mailbox_state(mailbox_hash).and_then(|r| {
-                    if r.is_none() {
-                        cache_handle.clear(mailbox_hash, &select_response)
-                    } else {
-                        Ok(())
-                    }
-                }) {
-                    (self.uid_store.event_consumer)(
-                        self.uid_store.account_hash,
-                        crate::backends::BackendEvent::from(err),
-                    );
-                }
-            }
-            self.uid_store
-                .mailboxes
-                .lock()
-                .await
-                .entry(mailbox_hash)
-                .and_modify(|entry| {
-                    *entry.select.write().unwrap() = Some(select_response.clone());
-                });
-        }
-        {
-            let mut permissions = permissions.lock().unwrap();
-            permissions.create_messages = !select_response.read_only;
-            permissions.remove_messages = !select_response.read_only;
-            permissions.set_flags = !select_response.read_only;
-            permissions.rename_messages = !select_response.read_only;
-            permissions.delete_messages = !select_response.read_only;
-        }
-        self.stream.as_mut()?.current_mailbox = MailboxSelection::Select(mailbox_hash);
-        if self
-            .uid_store
-            .msn_index
-            .lock()
-            .unwrap()
-            .get(&mailbox_hash)
-            .map(|i| i.is_empty())
-            .unwrap_or(true)
-        {
-            self.create_uid_msn_cache(mailbox_hash, 1, &select_response)
-                .await?;
-        }
-        Ok(Some(select_response))
+        Ok(None)
     }
 
     pub async fn examine_mailbox(
@@ -920,44 +894,7 @@ impl ImapConnection {
         ret: &mut Vec<u8>,
         force: bool,
     ) -> Result<Option<SelectResponse>> {
-        if !force
-            && self.stream.as_ref()?.current_mailbox == MailboxSelection::Examine(mailbox_hash)
-        {
-            return Ok(None);
-        }
-        let (imap_path, no_select) = {
-            let m = &self.uid_store.mailboxes.lock().await[&mailbox_hash];
-            (m.imap_path().to_string(), m.no_select)
-        };
-        if no_select {
-            return Err(Error::new(format!(
-                "Trying to examine a \\NoSelect mailbox: {}",
-                &imap_path
-            ))
-            .set_kind(crate::error::ErrorKind::Bug));
-        }
-        self.send_command(format!("EXAMINE \"{}\"", &imap_path).as_bytes())
-            .await?;
-        self.read_response(ret, RequiredResponses::EXAMINE_REQUIRED)
-            .await?;
-        debug!("examine response {}", String::from_utf8_lossy(ret));
-        let select_response = protocol_parser::select_response(ret).chain_err_summary(|| {
-            format!("Could not parse select response for mailbox {}", imap_path)
-        })?;
-        self.stream.as_mut()?.current_mailbox = MailboxSelection::Examine(mailbox_hash);
-        if !self
-            .uid_store
-            .msn_index
-            .lock()
-            .unwrap()
-            .get(&mailbox_hash)
-            .map(|i| i.is_empty())
-            .unwrap_or(true)
-        {
-            self.create_uid_msn_cache(mailbox_hash, 1, &select_response)
-                .await?;
-        }
-        Ok(Some(select_response))
+        Ok(None)
     }
 
     pub async fn unselect(&mut self) -> Result<()> {
@@ -965,7 +902,6 @@ impl ImapConnection {
             MailboxSelection::Examine(_) | MailboxSelection::Select(_) => {
                 let mut response = Vec::with_capacity(8 * 1024);
                 if self
-                    .uid_store
                     .capabilities
                     .lock()
                     .unwrap()
@@ -980,13 +916,8 @@ impl ImapConnection {
                      * functionality (via a SELECT command with a nonexistent mailbox name or
                      * reselecting the same mailbox with EXAMINE command)[..]
                      */
-                    let mut nonexistent = "blurdybloop".to_string();
-                    {
-                        let mailboxes = self.uid_store.mailboxes.lock().await;
-                        while mailboxes.values().any(|m| m.imap_path() == nonexistent) {
-                            nonexistent.push('p');
-                        }
-                    }
+                    let nonexistent = "blurdybloop".to_string();
+
                     self.send_command(format!("SELECT \"{}\"", nonexistent).as_bytes())
                         .await?;
                     self.read_response(&mut response, RequiredResponses::NO_REQUIRED)
@@ -999,8 +930,8 @@ impl ImapConnection {
     }
 
     pub fn add_refresh_event(&mut self, ev: RefreshEvent) {
-        (self.uid_store.event_consumer)(
-            self.uid_store.account_hash,
+        (self.event_consumer)(
+            self.account_hash,
             crate::backends::BackendEvent::Refresh(ev),
         );
     }
@@ -1011,16 +942,6 @@ impl ImapConnection {
         low: usize,
         _select_response: &SelectResponse,
     ) -> Result<()> {
-        debug_assert!(low > 0);
-        let mut response = Vec::new();
-        self.send_command(format!("UID SEARCH {}:*", low).as_bytes())
-            .await?;
-        self.read_response(&mut response, RequiredResponses::SEARCH)
-            .await?;
-        let mut msn_index_lck = self.uid_store.msn_index.lock().unwrap();
-        let msn_index = msn_index_lck.entry(mailbox_hash).or_default();
-        let _ = msn_index.drain(low - 1..);
-        msn_index.extend(protocol_parser::search_results(&response)?.1.into_iter());
         Ok(())
     }
 }

melib/src/backends/imap/operations.rs

@@ -52,7 +52,7 @@ impl ImapOp {
 }
 
 impl BackendOp for ImapOp {
-    fn as_bytes(&mut self) -> ResultFuture<Vec<u8>> {
+    fn as_bytes(&self) -> ResultFuture<Vec<u8>> {
         let connection = self.connection.clone();
         let mailbox_hash = self.mailbox_hash;
         let uid = self.uid;

melib/src/backends/imap/protocol_parser.rs

@@ -624,8 +624,8 @@ pub fn fetch_response(input: &[u8]) -> ImapParseResult<FetchResponse<'_>> {
                     String::from_utf8_lossy(input)
                 ))));
             }
-        } else if input[i..].starts_with(b"RFC822 {") {
-            i += b"RFC822 ".len();
+        } else if input[i..].starts_with(b"BODY[] {") {
+            i += b"BODY[] ".len();
             if let Ok((rest, body)) =
                 length_data::<_, _, (&[u8], nom::error::ErrorKind), _>(delimited(
                     tag("{"),
@@ -1008,20 +1008,20 @@ fn test_imap_fetch_response() {
     );
 }
 
-pub fn search_results<'a>(input: &'a [u8]) -> IResult<&'a [u8], Vec<ImapNum>> {
+pub fn search_results<'a>(input: &'a [u8]) -> IResult<&'a [u8], Vec<UID>> {
     alt((
-        |input: &'a [u8]| -> IResult<&'a [u8], Vec<ImapNum>> {
+        |input: &'a [u8]| -> IResult<&'a [u8], Vec<UID>> {
             let (input, _) = tag("* SEARCH ")(input)?;
             let (input, list) = separated_list1(
                 tag(b" "),
                 map_res(is_not(" \r\n"), |s: &[u8]| {
-                    ImapNum::from_str(unsafe { std::str::from_utf8_unchecked(s) })
+                    UID::from_str(unsafe { std::str::from_utf8_unchecked(s) })
                 }),
             )(input)?;
             let (input, _) = tag("\r\n")(input)?;
             Ok((input, list))
         },
-        |input: &'a [u8]| -> IResult<&'a [u8], Vec<ImapNum>> {
+        |input: &'a [u8]| -> IResult<&'a [u8], Vec<UID>> {
             let (input, _) = tag("* SEARCH\r\n")(input)?;
             Ok((input, vec![]))
         },
@@ -1759,7 +1759,7 @@ fn ctl(input: &[u8]) -> IResult<&[u8], u8> {
 
 pub fn generate_envelope_hash(mailbox_path: &str, uid: &UID) -> EnvelopeHash {
     let mut h = DefaultHasher::new();
-    h.write_usize(*uid);
+    h.write_u32(*uid);
     h.write(mailbox_path.as_bytes());
     EnvelopeHash(h.finish())
 }

melib/src/backends/jmap/operations.rs

@@ -45,7 +45,7 @@ impl JmapOp {
 }
 
 impl BackendOp for JmapOp {
-    fn as_bytes(&mut self) -> ResultFuture<Vec<u8>> {
+    fn as_bytes(&self) -> ResultFuture<Vec<u8>> {
         {
             let byte_lck = self.store.byte_cache.lock().unwrap();
             if byte_lck.contains_key(&self.hash) && byte_lck[&self.hash].bytes.is_some() {

melib/src/backends/maildir.rs

@@ -31,6 +31,7 @@ use crate::email::Flag;
 use crate::error::{Error, Result};
 use crate::shellexpand::ShellExpandTrait;
 use futures::stream::Stream;
+use std::cell::RefCell;
 use std::collections::hash_map::DefaultHasher;
 use std::fs;
 use std::hash::{Hash, Hasher};
@@ -44,7 +45,7 @@ pub struct MaildirOp {
     hash_index: HashIndexes,
     mailbox_hash: MailboxHash,
     hash: EnvelopeHash,
-    slice: Option<Vec<u8>>,
+    slice: RefCell<Option<Vec<u8>>>,
 }
 
 impl Clone for MaildirOp {
@@ -53,7 +54,7 @@ impl Clone for MaildirOp {
             hash_index: self.hash_index.clone(),
             mailbox_hash: self.mailbox_hash,
             hash: self.hash,
-            slice: None,
+            slice: RefCell::new(None),
         }
     }
 }
@@ -64,7 +65,7 @@ impl MaildirOp {
             hash_index,
             mailbox_hash,
             hash,
-            slice: None,
+            slice: RefCell::new(None),
         }
     }
     fn path(&self) -> Result<PathBuf> {
@@ -91,8 +92,10 @@ impl MaildirOp {
 }
 
 impl<'a> BackendOp for MaildirOp {
-    fn as_bytes(&mut self) -> ResultFuture<Vec<u8>> {
-        if self.slice.is_none() {
+    fn as_bytes(&self) -> ResultFuture<Vec<u8>> {
+        let ret = if let Some(bytes) = self.slice.borrow().as_ref() {
+            bytes.clone()
+        } else {
             let file = std::fs::OpenOptions::new()
                 .read(true)
                 .write(false)
@@ -100,10 +103,10 @@ impl<'a> BackendOp for MaildirOp {
             let mut buf_reader = BufReader::new(file);
             let mut contents = Vec::new();
             buf_reader.read_to_end(&mut contents)?;
-            self.slice = Some(contents);
-        }
-        let ret = Ok(self.slice.as_ref().unwrap().as_slice().to_vec());
-        Ok(Box::pin(async move { ret }))
+            *self.slice.borrow_mut() = Some(contents.clone());
+            contents
+        };
+        Ok(Box::pin(async move { Ok(ret) }))
     }
 
     fn fetch_flags(&self) -> ResultFuture<Flag> {

melib/src/backends/mbox.rs

@@ -291,8 +291,8 @@ impl MboxOp {
 }
 
 impl BackendOp for MboxOp {
-    fn as_bytes(&mut self) -> ResultFuture<Vec<u8>> {
-        if self.slice.get_mut().is_none() {
+    fn as_bytes(&self) -> ResultFuture<Vec<u8>> {
+        if self.slice.borrow().is_none() {
             let file = std::fs::OpenOptions::new()
                 .read(true)
                 .write(true)
@@ -301,9 +301,9 @@ impl BackendOp for MboxOp {
             let mut buf_reader = BufReader::new(file);
             let mut contents = Vec::new();
             buf_reader.read_to_end(&mut contents)?;
-            *self.slice.get_mut() = Some(contents);
+            *self.slice.borrow_mut() = Some(contents);
         }
-        let ret = Ok(self.slice.get_mut().as_ref().unwrap().as_slice()
+        let ret = Ok(self.slice.borrow().as_ref().unwrap().as_slice()
             [self.offset..self.offset + self.length]
             .to_vec());
         Ok(Box::pin(async move { ret }))

melib/src/backends/nntp/operations.rs

@@ -52,7 +52,7 @@ impl NntpOp {
 }
 
 impl BackendOp for NntpOp {
-    fn as_bytes(&mut self) -> ResultFuture<Vec<u8>> {
+    fn as_bytes(&self) -> ResultFuture<Vec<u8>> {
         let mailbox_hash = self.mailbox_hash;
         let uid = self.uid;
         let uid_store = self.uid_store.clone();

melib/src/backends/notmuch.rs

@@ -25,6 +25,7 @@ use crate::error::{Error, Result};
 use crate::shellexpand::ShellExpandTrait;
 use crate::{backends::*, Collection};
 use smallvec::SmallVec;
+use std::cell::RefCell;
 use std::collections::{hash_map::HashMap, BTreeMap};
 use std::ffi::{CStr, CString, OsStr};
 use std::io::Read;
@@ -768,7 +769,7 @@ impl MailBackend for NotmuchDb {
             lib: self.lib.clone(),
             hash,
             index: self.index.clone(),
-            bytes: None,
+            bytes: RefCell::new(None),
         }))
     }
 
@@ -1029,21 +1030,26 @@ struct NotmuchOp {
     hash: EnvelopeHash,
     index: Arc<RwLock<HashMap<EnvelopeHash, CString>>>,
     database: Arc<DbConnection>,
-    bytes: Option<Vec<u8>>,
+    bytes: RefCell<Option<Vec<u8>>>,
     #[allow(dead_code)]
     lib: Arc<libloading::Library>,
 }
 
 impl BackendOp for NotmuchOp {
-    fn as_bytes(&mut self) -> ResultFuture<Vec<u8>> {
-        let index_lck = self.index.write().unwrap();
-        let message = Message::find_message(&self.database, &index_lck[&self.hash])?;
-        let mut f = std::fs::File::open(message.get_filename())?;
-        let mut response = Vec::new();
-        f.read_to_end(&mut response)?;
-        self.bytes = Some(response);
-        let ret = Ok(self.bytes.as_ref().unwrap().to_vec());
-        Ok(Box::pin(async move { ret }))
+    fn as_bytes(&self) -> ResultFuture<Vec<u8>> {
+        let ret = if let Some(bytes) = self.bytes.borrow().as_ref() {
+            bytes.to_vec()
+        } else {
+            let index_lck = self.index.write().unwrap();
+            let message = Message::find_message(&self.database, &index_lck[&self.hash])?;
+            let mut f = std::fs::File::open(message.get_filename())?;
+            let mut response = Vec::new();
+            f.read_to_end(&mut response)?;
+            *self.bytes.borrow_mut() = Some(response.clone());
+            response
+        };
+
+        Ok(Box::pin(async move { Ok(ret) }))
     }
 
     fn fetch_flags(&self) -> ResultFuture<Flag> {

melib/src/email.rs

@@ -28,7 +28,7 @@
  * structure. Addresses in `To`, `From` fields etc are parsed into [`Address`](crate::email::Address) types.
  *
  * ```
- * use melib::{Attachment, Envelope};
+ * use melib::{Attachment, Envelope, email::attachment_types::Text};
  *
  * let raw_mail = r#"From: "some name" <some@example.com>
  * To: "me" <myself@example.com>
@@ -80,7 +80,7 @@
  * let body = envelope.body_bytes(raw_mail.as_bytes());
  * assert_eq!(body.content_type().to_string().as_str(), "multipart/mixed");
  *
- * let body_text = body.text();
+ * let body_text = body.text(Text::Plain);
  * assert_eq!(body_text.as_str(), "hello world.");
  *
  * let subattachments: Vec<Attachment> = body.attachments();

melib/src/email/attachments.rs

@@ -367,7 +367,7 @@ pub struct Attachment {
 impl fmt::Debug for Attachment {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         let mut text = Vec::with_capacity(4096);
-        self.get_text_recursive(&mut text);
+        self.get_text_recursive(&Text::Plain, &mut text);
         f.debug_struct("Attachment")
             .field("content_type", &self.content_type)
             .field("content_transfer_encoding", &self.content_transfer_encoding)
@@ -561,7 +561,7 @@ impl Attachment {
         false
     }
 
-    fn get_text_recursive(&self, text: &mut Vec<u8>) {
+    fn get_text_recursive(&self, kind: &Text, text: &mut Vec<u8>) {
         match self.content_type {
             ContentType::Text { .. } | ContentType::PGPSignature | ContentType::CMSSignature => {
                 text.extend(self.decode(Default::default()));
@@ -577,15 +577,13 @@ impl Attachment {
                     .find_map(|(k, v)| if k == b"type" { Some(v) } else { None })
                     .and_then(|t| parts.iter().find(|a| a.content_type == t.as_slice()))
                 {
-                    main_attachment.get_text_recursive(text);
+                    main_attachment.get_text_recursive(kind, text);
                 } else {
                     for a in parts {
                         if a.content_disposition.kind.is_inline() {
-                            if let ContentType::Text {
-                                kind: Text::Plain, ..
-                            } = a.content_type
+                            if matches!(&a.content_type, ContentType::Text { kind: a_kind, .. } if a_kind == kind)
                             {
-                                a.get_text_recursive(text);
+                                a.get_text_recursive(kind, text);
                                 break;
                             }
                         }
@@ -599,11 +597,9 @@ impl Attachment {
             } => {
                 for a in parts {
                     if a.content_disposition.kind.is_inline() {
-                        if let ContentType::Text {
-                            kind: Text::Plain, ..
-                        } = a.content_type
+                        if matches!(&a.content_type, ContentType::Text { kind: a_kind, .. } if a_kind == kind)
                         {
-                            a.get_text_recursive(text);
+                            a.get_text_recursive(kind, text);
                             break;
                         }
                     }
@@ -614,7 +610,7 @@ impl Attachment {
             } => {
                 for a in parts {
                     if a.content_disposition.kind.is_inline() {
-                        a.get_text_recursive(text);
+                        a.get_text_recursive(kind, text);
                     }
                 }
             }
@@ -622,9 +618,9 @@ impl Attachment {
         }
     }
 
-    pub fn text(&self) -> String {
+    pub fn text(&self, kind: Text) -> String {
         let mut text = Vec::with_capacity(self.body.length);
-        self.get_text_recursive(&mut text);
+        self.get_text_recursive(&kind, &mut text);
         String::from_utf8_lossy(text.as_slice()).into()
     }
 

melib/src/email/compose.rs

@@ -22,7 +22,7 @@
 /*! Compose a `Draft`, with MIME and attachment support */
 use super::*;
 use crate::email::attachment_types::{
-    Charset, ContentTransferEncoding, ContentType, MultipartType,
+    Charset, ContentTransferEncoding, ContentType, MultipartType, Text,
 };
 use crate::email::attachments::AttachmentBuilder;
 use crate::shellexpand::ShellExpandTrait;
@@ -99,13 +99,13 @@ impl FromStr for Draft {
 }
 
 impl Draft {
-    pub fn edit(envelope: &Envelope, bytes: &[u8]) -> Result<Self> {
+    pub fn edit(envelope: &Envelope, bytes: &[u8], kind: Text) -> Result<Self> {
         let mut ret = Draft::default();
         for (k, v) in envelope.headers(bytes).unwrap_or_else(|_| Vec::new()) {
             ret.headers.insert(k.try_into()?, v.into());
         }
 
-        ret.body = envelope.body_bytes(bytes).text();
+        ret.body = envelope.body_bytes(bytes).text(kind);
 
         Ok(ret)
     }
@@ -383,10 +383,9 @@ fn build_multipart(
 }
 
 fn print_attachment(ret: &mut String, a: AttachmentBuilder) {
-    use ContentType::*;
     match a.content_type {
         ContentType::Text {
-            kind: crate::email::attachment_types::Text::Plain,
+            kind: Text::Plain,
             charset: Charset::UTF8,
             parameters: ref v,
         } if v.is_empty() => {
@@ -396,13 +395,13 @@ fn print_attachment(ret: &mut String, a: AttachmentBuilder) {
                 ret.push_str("\r\n");
             }
         }
-        Text { .. } => {
+        ContentType::Text { .. } => {
             for line in a.build().into_raw().lines() {
                 ret.push_str(line);
                 ret.push_str("\r\n");
             }
         }
-        Multipart {
+        ContentType::Multipart {
             boundary: _,
             kind,
             parts,
@@ -418,7 +417,7 @@ fn print_attachment(ret: &mut String, a: AttachmentBuilder) {
                     .collect::<Vec<AttachmentBuilder>>(),
             );
         }
-        MessageRfc822 => {
+        ContentType::MessageRfc822 => {
             ret.push_str(&format!(
                 "Content-Type: {}; charset=\"utf-8\"\r\n",
                 a.content_type
@@ -430,7 +429,7 @@ fn print_attachment(ret: &mut String, a: AttachmentBuilder) {
                 ret.push_str("\r\n");
             }
         }
-        PGPSignature => {
+        ContentType::PGPSignature => {
             ret.push_str(&format!(
                 "Content-Type: {}; charset=\"utf-8\"; name=\"signature.asc\"\r\n",
                 a.content_type

src/components/mail/compose.rs

@@ -202,7 +202,7 @@ impl Composer {
         let mut ret = Composer::with_account(account_hash, context);
         let envelope: EnvelopeRef = context.accounts[&account_hash].collection.get_env(env_hash);
 
-        ret.draft = Draft::edit(&envelope, bytes)?;
+        ret.draft = Draft::edit(&envelope, bytes, Text::Plain)?;
 
         ret.account_hash = account_hash;
         Ok(ret)

src/components/mail/listing.rs

@@ -608,7 +608,7 @@ pub trait MailListingTrait: ListingTrait {
 
                 let futures: Result<Vec<_>> = envs_to_set
                     .iter()
-                    .map(|&env_hash| account.operation(env_hash).and_then(|mut op| op.as_bytes()))
+                    .map(|&env_hash| account.operation(env_hash).and_then(|op| op.as_bytes()))
                     .collect::<Result<Vec<_>>>();
                 let path_ = path.to_path_buf();
                 let format = (*format).unwrap_or_default();

src/components/mail/view.rs

@@ -282,7 +282,7 @@ impl MailView {
             {
                 match account
                     .operation(self.coordinates.2)
-                    .and_then(|mut op| op.as_bytes())
+                    .and_then(|op| op.as_bytes())
                 {
                     Ok(fut) => {
                         let mut handle = account.job_executor.spawn_specialized(fut);
@@ -1435,7 +1435,7 @@ impl Component for MailView {
                             self.subview = Some(subview);
                             self.mode = ViewMode::Subview;
                         } else {
-                            text.push_str(&attachment.text());
+                            text.push_str(&attachment.text(Text::Plain));
                             let colors = crate::conf::value(context, "mail.view.body");
                             self.pager =
                                 Pager::from_string(text, Some(context), Some(0), None, colors);
@@ -1997,7 +1997,9 @@ impl Component for MailView {
                 let (sender, mut receiver) = crate::jobs::oneshot::channel();
                 let operation = context.accounts[&account_hash].operation(env_hash);
                 let bytes_job = async move {
-                    let _ = sender.send(operation?.as_bytes()?.await);
+                    let bytes_fut = operation?.as_bytes()?;
+                    let bytes = bytes_fut.await;
+                    let _ = sender.send(bytes);
                     Ok(())
                 };
                 let handle = if context.accounts[&account_hash]

src/components/mail/view/envelope.rs

@@ -143,7 +143,7 @@ impl EnvelopeView {
             ViewMode::Raw => String::from_utf8_lossy(body.body()).into_owned(),
             ViewMode::Url => {
                 let mut t = body_text;
-                for (lidx, l) in finder.links(&body.text()).enumerate() {
+                for (lidx, l) in finder.links(&body.text(Text::Plain)).enumerate() {
                     let offset = if lidx < 10 {
                         lidx * 3
                     } else if lidx < 100 {
@@ -170,7 +170,7 @@ impl EnvelopeView {
             ViewMode::Attachment(aidx) => {
                 let attachments = body.attachments();
                 let mut ret = "Viewing attachment. Press `r` to return \n".to_string();
-                ret.push_str(&attachments[aidx].text());
+                ret.push_str(&attachments[aidx].text(Text::Plain));
                 ret
             }
         }
@@ -475,7 +475,7 @@ impl Component for EnvelopeView {
                     .push_back(UIEvent::StatusEvent(StatusEvent::BufClear));
                 let url = {
                     let finder = LinkFinder::new();
-                    let t = self.mail.body().text();
+                    let t = self.mail.body().text(Text::Plain);
                     let links: Vec<Link> = finder.links(&t).collect();
                     if let Some(u) = links.get(lidx) {
                         u.as_str().to_string()

src/conf.rs

@@ -362,6 +362,7 @@ impl FileSettings {
             }
             #[cfg(test)]
             return Ok(FileSettings::default());
+            #[cfg(not(test))]
             return Err(Error::new("No configuration file found."));
         }
 

src/plugins/backend.rs

@@ -279,7 +279,7 @@ struct PluginOp {
 }
 
 impl BackendOp for PluginOp {
-    fn as_bytes(&mut self) -> ResultFuture<Vec<u8>> {
+    fn as_bytes(&self) -> ResultFuture<Vec<u8>> {
         let hash = self.hash;
         let channel = self.channel.clone();
         Ok(Box::pin(async move {

src/sqlite3.rs

@@ -28,7 +28,7 @@ use melib::search::{
 };
 use melib::{
     backends::{MailBackend, ResultFuture},
-    email::{Envelope, EnvelopeHash},
+    email::{attachment_types::Text, Envelope, EnvelopeHash},
     log,
     sqlite3::{self as melib_sqlite3, rusqlite::params, DatabaseDescription},
     thread::{SortField, SortOrder},
@@ -156,7 +156,7 @@ pub async fn insert(
         .as_bytes()?;
 
     let body = match op.await.map(|bytes| envelope.body_bytes(&bytes)) {
-        Ok(body) => body.text(),
+        Ok(body) => body.text(Text::Plain),
         Err(err) => {
             debug!(
                 "{}",
@@ -305,14 +305,14 @@ pub fn index(context: &mut crate::state::Context, account_index: usize) -> Resul
         for chunk in env_hashes.chunks(200) {
             ctr += chunk.len();
             for env_hash in chunk {
-                let mut op = backend_mutex.read().unwrap().operation(*env_hash)?;
-                let bytes = op
-                    .as_bytes()?
+                let op = backend_mutex.read().unwrap().operation(*env_hash)?;
+                let bytes_fut = op.as_bytes()?;
+                let bytes = bytes_fut
                     .await
                     .chain_err_summary(|| format!("Failed to open envelope {}", env_hash))?;
                 let envelopes_lck = acc_mutex.read().unwrap();
                 if let Some(e) = envelopes_lck.get(env_hash) {
-                    let body = e.body_bytes(&bytes).text().replace('\0', "");
+                    let body = e.body_bytes(&bytes).text(Text::Plain).replace('\0', "");
                     conn.execute("INSERT OR REPLACE INTO envelopes (account_id, hash, date, _from, _to, cc, bcc, subject, message_id, in_reply_to, _references, flags, has_attachments, body_text, timestamp)
               VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)",
               params![account_id, e.hash().to_be_bytes().to_vec(), e.date_as_str(), e.field_from_to_string(), e.field_to_to_string(), e.field_cc_to_string(), e.field_bcc_to_string(), e.subject().into_owned().trim_end_matches('\u{0}'), e.message_id_display().to_string(), e.in_reply_to_display().map(|f| f.to_string()).unwrap_or_default(), e.field_references_to_string(), i64::from(e.flags().bits()), if e.has_attachments() { 1 } else { 0 }, body, e.date().to_be_bytes().to_vec()],

tools/src/embed.rs

@@ -244,7 +244,7 @@ impl Component for EmbedContainer {
                         match embed_guard.is_active() {
                             Ok(WaitStatus::Exited(_, exit_code)) => {
                                 drop(embed_guard);
-                                let embed = self.embed.take();
+                                _ = self.embed.take();
                                 if exit_code != 0 {
                                     context.replies.push_back(UIEvent::Notification(
                                         None,