Browse Source

imap: remove blocking imap backend, replace with async

tags/alpha-0.6.0
parent
commit
931863436d
WARNING! Although there is a key with this ID in the database it does not verify this commit! This commit is SUSPICIOUS. GPG Key ID: 73627C2F690DF710
20 changed files with 1298 additions and 6842 deletions
  1. +8
    -20
      melib/src/backends.rs
  2. +750
    -670
      melib/src/backends/imap.rs
  3. +320
    -307
      melib/src/backends/imap/connection.rs
  4. +1
    -0
      melib/src/backends/imap/managesieve.rs
  5. +82
    -56
      melib/src/backends/imap/operations.rs
  6. +42
    -22
      melib/src/backends/imap/protocol_parser.rs
  7. +15
    -15
      melib/src/backends/imap/untagged.rs
  8. +75
    -213
      melib/src/backends/imap/watch.rs
  9. +0
    -1303
      melib/src/backends/imap_async.rs
  10. +0
    -174
      melib/src/backends/imap_async/cache.rs
  11. +0
    -777
      melib/src/backends/imap_async/connection.rs
  12. +0
    -172
      melib/src/backends/imap_async/mailbox.rs
  13. +0
    -139
      melib/src/backends/imap_async/managesieve.rs
  14. +0
    -252
      melib/src/backends/imap_async/operations.rs
  15. +0
    -1544
      melib/src/backends/imap_async/protocol_parser.rs
  16. +0
    -332
      melib/src/backends/imap_async/untagged.rs
  17. +0
    -770
      melib/src/backends/imap_async/watch.rs
  18. +5
    -3
      src/managesieve.rs
  19. +0
    -4
      testing/Cargo.toml
  20. +0
    -69
      testing/src/managesieve.rs

+ 8
- 20
melib/src/backends.rs View File

@@ -33,25 +33,20 @@ macro_rules! tag_hash {

#[cfg(feature = "imap_backend")]
pub mod imap;
#[cfg(feature = "imap_backend")]
pub mod imap_async;
//#[cfg(feature = "imap_backend")]
//pub mod imap2;
#[cfg(feature = "maildir_backend")]
pub mod maildir;
#[cfg(feature = "mbox_backend")]
pub mod mbox;
#[cfg(feature = "notmuch_backend")]
pub mod notmuch;
#[cfg(feature = "notmuch_backend")]
pub use self::notmuch::NotmuchDb;
#[cfg(feature = "jmap_backend")]
pub mod jmap;
#[cfg(feature = "jmap_backend")]
pub use self::jmap::JmapType;

#[cfg(feature = "maildir_backend")]
pub mod maildir;
#[cfg(feature = "mbox_backend")]
pub mod mbox;
#[cfg(feature = "imap_backend")]
pub use self::imap::ImapType;
#[cfg(feature = "jmap_backend")]
pub use self::jmap::JmapType;
use crate::async_workers::*;
use crate::conf::AccountSettings;
use crate::error::{MeliError, Result};
@@ -145,15 +140,8 @@ impl Backends {
b.register(
"imap".to_string(),
Backend {
create_fn: Box::new(|| Box::new(|f, i| ImapType::new(f, i))),
validate_conf_fn: Box::new(ImapType::validate_config),
},
);
b.register(
"imap_async".to_string(),
Backend {
create_fn: Box::new(|| Box::new(|f, i| imap_async::ImapType::new(f, i))),
validate_conf_fn: Box::new(imap_async::ImapType::validate_config),
create_fn: Box::new(|| Box::new(|f, i| imap::ImapType::new(f, i))),
validate_conf_fn: Box::new(imap::ImapType::validate_config),
},
);
}


+ 750
- 670
melib/src/backends/imap.rs
File diff suppressed because it is too large
View File


+ 320
- 307
melib/src/backends/imap/connection.rs View File

@@ -21,15 +21,18 @@

use super::protocol_parser::{ImapLineSplit, ImapResponse, RequiredResponses};
use crate::backends::MailboxHash;
use crate::connections::Connection;
use crate::email::parser::BytesExt;
use crate::error::*;
use std::io::Read;
use std::io::Write;
extern crate native_tls;
use futures::io::{AsyncReadExt, AsyncWriteExt};
use native_tls::TlsConnector;
pub use smol::Async as AsyncWrapper;
use std::collections::HashSet;
use std::future::Future;
use std::iter::FromIterator;
use std::net::SocketAddr;
use std::pin::Pin;
use std::sync::Arc;
use std::time::Instant;

@@ -45,7 +48,7 @@ pub enum ImapProtocol {
#[derive(Debug)]
pub struct ImapStream {
cmd_id: usize,
stream: native_tls::TlsStream<std::net::TcpStream>,
stream: AsyncWrapper<Connection>,
protocol: ImapProtocol,
}

@@ -61,117 +64,136 @@ impl MailboxSelection {
std::mem::replace(self, MailboxSelection::None)
}
}

#[derive(Debug)]
pub struct ImapConnection {
pub stream: Result<ImapStream>,
pub server_conf: ImapServerConf,
pub capabilities: Capabilities,
pub uid_store: Arc<UIDStore>,
pub current_mailbox: MailboxSelection,
}

impl Drop for ImapStream {
fn drop(&mut self) {
self.send_command(b"LOGOUT").ok().take();
//self.send_command(b"LOGOUT").ok().take();
}
}

impl ImapStream {
pub fn new_connection(server_conf: &ImapServerConf) -> Result<(Capabilities, ImapStream)> {
use std::io::prelude::*;
pub async fn new_connection(
server_conf: &ImapServerConf,
) -> Result<(Capabilities, ImapStream)> {
use std::net::TcpStream;
let path = &server_conf.server_hostname;

let mut connector = TlsConnector::builder();
if server_conf.danger_accept_invalid_certs {
connector.danger_accept_invalid_certs(true);
}
let connector = connector.build()?;

let addr = if let Ok(a) = lookup_ipv4(path, server_conf.server_port) {
a
} else {
return Err(MeliError::new(format!(
"Could not lookup address {}",
&path
)));
};

let mut socket = TcpStream::connect_timeout(&addr, std::time::Duration::new(4, 0))?;
socket.set_read_timeout(Some(std::time::Duration::new(4, 0)))?;
socket.set_write_timeout(Some(std::time::Duration::new(4, 0)))?;
let cmd_id = 1;
if server_conf.use_starttls {
let mut buf = vec![0; 1024];
match server_conf.protocol {
ImapProtocol::IMAP => {
socket.write_all(format!("M{} STARTTLS\r\n", cmd_id).as_bytes())?
}
ImapProtocol::ManageSieve => {
let len = socket.read(&mut buf)?;
debug!(unsafe { std::str::from_utf8_unchecked(&buf[0..len]) });
debug!(socket.write_all(b"STARTTLS\r\n")?);
}
let stream = if server_conf.use_tls {
let mut connector = TlsConnector::builder();
if server_conf.danger_accept_invalid_certs {
connector.danger_accept_invalid_certs(true);
}
let mut response = String::with_capacity(1024);
let mut broken = false;
let now = std::time::Instant::now();
let connector = connector.build()?;

while now.elapsed().as_secs() < 3 {
let len = socket.read(&mut buf)?;
response.push_str(unsafe { std::str::from_utf8_unchecked(&buf[0..len]) });
let addr = if let Ok(a) = lookup_ipv4(path, server_conf.server_port) {
a
} else {
return Err(MeliError::new(format!(
"Could not lookup address {}",
&path
)));
};

let mut socket = AsyncWrapper::new(Connection::Tcp(TcpStream::connect_timeout(
&addr,
std::time::Duration::new(4, 0),
)?))?;
if server_conf.use_starttls {
let mut buf = vec![0; 1024];
match server_conf.protocol {
ImapProtocol::IMAP => {
if response.starts_with("* OK ") && response.find("\r\n").is_some() {
if let Some(pos) = response.as_bytes().find(b"\r\n") {
response.drain(0..pos + 2);
}
}
socket
.write_all(format!("M{} STARTTLS\r\n", cmd_id).as_bytes())
.await?
}
ImapProtocol::ManageSieve => {
if response.starts_with("OK ") && response.find("\r\n").is_some() {
response.clear();
broken = true;
break;
socket.read(&mut buf).await?;
socket.write_all(b"STARTTLS\r\n").await?;
}
}
let mut response = String::with_capacity(1024);
let mut broken = false;
let now = std::time::Instant::now();

while now.elapsed().as_secs() < 3 {
let len = socket.read(&mut buf).await?;
response.push_str(unsafe { std::str::from_utf8_unchecked(&buf[0..len]) });
match server_conf.protocol {
ImapProtocol::IMAP => {
if response.starts_with("* OK ") && response.find("\r\n").is_some() {
if let Some(pos) = response.as_bytes().find(b"\r\n") {
response.drain(0..pos + 2);
}
}
}
ImapProtocol::ManageSieve => {
if response.starts_with("OK ") && response.find("\r\n").is_some() {
response.clear();
broken = true;
break;
}
}
}
if response.starts_with("M1 OK") {
broken = true;
break;
}
}
if response.starts_with("M1 OK") {
broken = true;
break;
if !broken {
return Err(MeliError::new(format!(
"Could not initiate TLS negotiation to {}.",
path
)));
}
}
if !broken {
return Err(MeliError::new(format!(
"Could not initiate TLS negotiation to {}.",
path
)));
}
}

socket
.set_nonblocking(true)
.expect("set_nonblocking call failed");
let stream = {
let mut conn_result = connector.connect(path, socket);
if let Err(native_tls::HandshakeError::WouldBlock(midhandshake_stream)) = conn_result {
let mut midhandshake_stream = Some(midhandshake_stream);
loop {
match midhandshake_stream.take().unwrap().handshake() {
Ok(r) => {
conn_result = Ok(r);
break;
}
Err(native_tls::HandshakeError::WouldBlock(stream)) => {
midhandshake_stream = Some(stream);
}
p => {
p?;
{
// FIXME: This is blocking
let socket = socket.into_inner()?;
let mut conn_result = connector.connect(path, socket);
if let Err(native_tls::HandshakeError::WouldBlock(midhandshake_stream)) =
conn_result
{
let mut midhandshake_stream = Some(midhandshake_stream);
loop {
match midhandshake_stream.take().unwrap().handshake() {
Ok(r) => {
conn_result = Ok(r);
break;
}
Err(native_tls::HandshakeError::WouldBlock(stream)) => {
midhandshake_stream = Some(stream);
}
p => {
p?;
}
}
}
}
AsyncWrapper::new(Connection::Tls(conn_result?))?
}
conn_result?
} else {
let addr = if let Ok(a) = lookup_ipv4(path, server_conf.server_port) {
a
} else {
return Err(MeliError::new(format!(
"Could not lookup address {}",
&path
)));
};
AsyncWrapper::new(Connection::Tcp(TcpStream::connect_timeout(
&addr,
std::time::Duration::new(4, 0),
)?))?
};
let mut res = String::with_capacity(8 * 1024);
let mut ret = ImapStream {
@@ -181,7 +203,7 @@ impl ImapStream {
};
if let ImapProtocol::ManageSieve = server_conf.protocol {
use data_encoding::BASE64;
ret.read_response(&mut res)?;
ret.read_response(&mut res).await?;
ret.send_command(
format!(
"AUTHENTICATE \"PLAIN\" \"{}\"",
@@ -194,13 +216,14 @@ impl ImapStream {
)
)
.as_bytes(),
)?;
ret.read_response(&mut res)?;
)
.await?;
ret.read_response(&mut res).await?;
return Ok((Default::default(), ret));
}

ret.send_command(b"CAPABILITY")?;
ret.read_response(&mut res)?;
ret.send_command(b"CAPABILITY").await?;
ret.read_response(&mut res).await?;
let capabilities: std::result::Result<Vec<&[u8]>, _> = res
.split_rn()
.find(|l| l.starts_with("* CAPABILITY"))
@@ -244,11 +267,12 @@ impl ImapStream {
&server_conf.server_username, &server_conf.server_password
)
.as_bytes(),
)?;
)
.await?;
let tag_start = format!("M{} ", (ret.cmd_id - 1));

loop {
ret.read_lines(&mut res, &String::new(), false)?;
ret.read_lines(&mut res, &String::new(), false).await?;
let mut should_break = false;
for l in res.split_rn() {
if l.starts_with("* CAPABILITY") {
@@ -274,29 +298,31 @@ impl ImapStream {
}
}

if let Some(capabilities) = capabilities {
Ok((capabilities, ret))
} else {
if capabilities.is_none() {
/* sending CAPABILITY after LOGIN automatically is an RFC recommendation, so check
* for lazy servers */
drop(capabilities);
ret.send_command(b"CAPABILITY")?;
ret.read_response(&mut res).unwrap();
ret.send_command(b"CAPABILITY").await?;
ret.read_response(&mut res).await.unwrap();
let capabilities = protocol_parser::capabilities(res.as_bytes())?.1;
let capabilities = HashSet::from_iter(capabilities.into_iter().map(|s| s.to_vec()));
Ok((capabilities, ret))
} else {
let capabilities = capabilities.unwrap();
Ok((capabilities, ret))
}
}

pub fn read_response(&mut self, ret: &mut String) -> Result<()> {
pub async fn read_response(&mut self, ret: &mut String) -> Result<()> {
let id = match self.protocol {
ImapProtocol::IMAP => format!("M{} ", self.cmd_id - 1),
ImapProtocol::ManageSieve => String::new(),
};
self.read_lines(ret, &id, true)
self.read_lines(ret, &id, true).await?;
Ok(())
}

pub fn read_lines(
pub async fn read_lines(
&mut self,
ret: &mut String,
termination_string: &str,
@@ -306,7 +332,7 @@ impl ImapStream {
ret.clear();
let mut last_line_idx: usize = 0;
loop {
match self.stream.read(&mut buf) {
match self.stream.read(&mut buf).await {
Ok(0) => break,
Ok(b) => {
ret.push_str(unsafe { std::str::from_utf8_unchecked(&buf[0..b]) });
@@ -317,10 +343,10 @@ impl ImapStream {
if let Some(prev_line) =
ret[last_line_idx..pos + last_line_idx].rfind("\r\n")
{
last_line_idx += prev_line + 2;
pos -= prev_line + 2;
last_line_idx += prev_line + "\r\n".len();
pos -= prev_line + "\r\n".len();
}
if pos + "\r\n".len() == ret[last_line_idx..].len() {
if Some(pos + "\r\n".len()) == ret.get(last_line_idx..).map(|r| r.len()) {
if !termination_string.is_empty()
&& ret[last_line_idx..].starts_with(termination_string)
{
@@ -333,7 +359,7 @@ impl ImapStream {
break;
}
}
last_line_idx += pos + 2;
last_line_idx += pos + "\r\n".len();
}
}
Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock => {
@@ -344,29 +370,33 @@ impl ImapStream {
}
}
}
//debug!("returning IMAP response:\n{:?}", &ret);
Ok(())
}

pub fn wait_for_continuation_request(&mut self) -> Result<()> {
pub async fn wait_for_continuation_request(&mut self) -> Result<()> {
let term = "+ ".to_string();
let mut ret = String::new();
self.read_lines(&mut ret, &term, false)
self.read_lines(&mut ret, &term, false).await?;
Ok(())
}

pub fn send_command(&mut self, command: &[u8]) -> Result<()> {
pub async fn send_command(&mut self, command: &[u8]) -> Result<()> {
let command = command.trim();
match self.protocol {
ImapProtocol::IMAP => {
self.stream.write_all(b"M")?;
self.stream.write_all(self.cmd_id.to_string().as_bytes())?;
self.stream.write_all(b" ")?;
self.stream.write_all(b"M").await?;
self.stream
.write_all(self.cmd_id.to_string().as_bytes())
.await?;
self.stream.write_all(b" ").await?;
self.cmd_id += 1;
}
ImapProtocol::ManageSieve => {}
}

self.stream.write_all(command)?;
self.stream.write_all(b"\r\n")?;
self.stream.write_all(command).await?;
self.stream.write_all(b"\r\n").await?;
match self.protocol {
ImapProtocol::IMAP => {
debug!("sent: M{} {}", self.cmd_id - 1, unsafe {
@@ -378,20 +408,15 @@ impl ImapStream {
Ok(())
}

pub fn send_literal(&mut self, data: &[u8]) -> Result<()> {
self.stream.write_all(data)?;
self.stream.write_all(b"\r\n")?;
pub async fn send_literal(&mut self, data: &[u8]) -> Result<()> {
self.stream.write_all(data).await?;
self.stream.write_all(b"\r\n").await?;
Ok(())
}

pub fn send_raw(&mut self, raw: &[u8]) -> Result<()> {
self.stream.write_all(raw)?;
self.stream.write_all(b"\r\n")?;
Ok(())
}

pub fn set_nonblocking(&mut self, val: bool) -> Result<()> {
self.stream.get_mut().set_nonblocking(val)?;
pub async fn send_raw(&mut self, raw: &[u8]) -> Result<()> {
self.stream.write_all(raw).await?;
self.stream.write_all(b"\r\n").await?;
Ok(())
}
}
@@ -404,13 +429,12 @@ impl ImapConnection {
ImapConnection {
stream: Err(MeliError::new("Offline".to_string())),
server_conf: server_conf.clone(),
capabilities: Capabilities::default(),
uid_store,
current_mailbox: MailboxSelection::None,
}
}

pub fn connect(&mut self) -> Result<()> {
pub async fn connect(&mut self) -> Result<()> {
if let (instant, ref mut status @ Ok(())) = *self.uid_store.is_online.lock().unwrap() {
if Instant::now().duration_since(instant) >= std::time::Duration::new(60 * 30, 0) {
*status = Err(MeliError::new("Connection timed out"));
@@ -421,7 +445,7 @@ impl ImapConnection {
self.uid_store.is_online.lock().unwrap().0 = Instant::now();
return Ok(());
}
let new_stream = ImapStream::new_connection(&self.server_conf);
let new_stream = debug!(ImapStream::new_connection(&self.server_conf).await);
if new_stream.is_err() {
*self.uid_store.is_online.lock().unwrap() = (
Instant::now(),
@@ -432,117 +456,94 @@ impl ImapConnection {
}
let (capabilities, stream) = new_stream?;
self.stream = Ok(stream);
self.capabilities = capabilities;
*self.uid_store.capabilities.lock().unwrap() = capabilities;
Ok(())
}

pub fn read_response(
&mut self,
ret: &mut String,
pub fn read_response<'a>(
&'a mut self,
ret: &'a mut String,
required_responses: RequiredResponses,
) -> Result<()> {
let mut response = String::new();
ret.clear();
self.try_send(|s| s.read_response(&mut response))?;
) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'a>> {
Box::pin(async move {
let mut response = String::new();
ret.clear();
self.stream.as_mut()?.read_response(&mut response).await?;

match self.server_conf.protocol {
ImapProtocol::IMAP => {
let r: ImapResponse = ImapResponse::from(&response);
match r {
ImapResponse::Bye(ref response_code) => {
self.stream = Err(MeliError::new(format!(
"Offline: received BYE: {:?}",
response_code
)));
ret.push_str(&response);
}
ImapResponse::No(ref response_code) => {
debug!("Received NO response: {:?} {:?}", response_code, response);
ret.push_str(&response);
}
ImapResponse::Bad(ref response_code) => {
debug!("Received BAD response: {:?} {:?}", response_code, response);
ret.push_str(&response);
}
_ => {
/*debug!(
"check every line for required_responses: {:#?}",
&required_responses
);*/
for l in response.split_rn() {
/*debug!("check line: {}", &l);*/
if required_responses.check(l) || !self.process_untagged(l)? {
ret.push_str(l);
match self.server_conf.protocol {
ImapProtocol::IMAP => {
let r: ImapResponse = ImapResponse::from(&response);
match r {
ImapResponse::Bye(ref response_code) => {
self.stream = Err(MeliError::new(format!(
"Offline: received BYE: {:?}",
response_code
)));
ret.push_str(&response);
}
ImapResponse::No(ref response_code) => {
debug!("Received NO response: {:?} {:?}", response_code, response);
ret.push_str(&response);
}
ImapResponse::Bad(ref response_code) => {
debug!("Received BAD response: {:?} {:?}", response_code, response);
ret.push_str(&response);
}
_ => {
/*debug!(
"check every line for required_responses: {:#?}",
&required_responses
);*/
for l in response.split_rn() {
/*debug!("check line: {}", &l);*/
if required_responses.check(l) || !self.process_untagged(l).await? {
ret.push_str(l);
}
}
}
//ret.push_str(&response);
}
r.into()
}
ImapProtocol::ManageSieve => {
ret.push_str(&response);
Ok(())
}
r.into()
}
ImapProtocol::ManageSieve => {
ret.push_str(&response);
Ok(())
}
}
}

pub fn read_lines(&mut self, ret: &mut String, termination_string: String) -> Result<()> {
self.try_send(|s| s.read_lines(ret, &termination_string, false))
}

pub fn wait_for_continuation_request(&mut self) -> Result<()> {
self.try_send(|s| s.wait_for_continuation_request())
})
}

pub fn send_command(&mut self, command: &[u8]) -> Result<()> {
self.try_send(|s| s.send_command(command))
pub async fn read_lines(&mut self, ret: &mut String, termination_string: String) -> Result<()> {
self.stream
.as_mut()?
.read_lines(ret, &termination_string, false)
.await?;
Ok(())
}

pub fn send_literal(&mut self, data: &[u8]) -> Result<()> {
self.try_send(|s| s.send_literal(data))
pub async fn wait_for_continuation_request(&mut self) -> Result<()> {
self.stream
.as_mut()?
.wait_for_continuation_request()
.await?;
Ok(())
}

pub fn send_raw(&mut self, raw: &[u8]) -> Result<()> {
self.try_send(|s| s.send_raw(raw))
pub async fn send_command(&mut self, command: &[u8]) -> Result<()> {
self.stream.as_mut()?.send_command(command).await?;
Ok(())
}

pub fn set_nonblocking(&mut self, val: bool) -> Result<()> {
self.try_send(|s| s.set_nonblocking(val))
pub async fn send_literal(&mut self, data: &[u8]) -> Result<()> {
self.stream.as_mut()?.send_literal(data).await?;
Ok(())
}

pub fn try_send(
&mut self,
mut action: impl FnMut(&mut ImapStream) -> Result<()>,
) -> Result<()> {
if let (instant, ref mut status @ Ok(())) = *self.uid_store.is_online.lock().unwrap() {
if Instant::now().duration_since(instant) >= std::time::Duration::new(60 * 30, 0) {
*status = Err(MeliError::new("Connection timed out"));
self.stream = Err(MeliError::new("Connection timed out"));
}
}
if let Ok(ref mut stream) = self.stream {
if action(stream).is_ok() {
self.uid_store.is_online.lock().unwrap().0 = Instant::now();
return Ok(());
}
}
let new_stream = ImapStream::new_connection(&self.server_conf);
if new_stream.is_err() {
*self.uid_store.is_online.lock().unwrap() = (
Instant::now(),
Err(new_stream.as_ref().unwrap_err().clone()),
);
} else {
*self.uid_store.is_online.lock().unwrap() = (Instant::now(), Ok(()));
}
let (capabilities, stream) = new_stream?;
self.stream = Ok(stream);
self.capabilities = capabilities;
Err(MeliError::new("Connection timed out"))
pub async fn send_raw(&mut self, raw: &[u8]) -> Result<()> {
self.stream.as_mut()?.send_raw(raw).await?;
Ok(())
}

pub fn select_mailbox(
pub async fn select_mailbox(
&mut self,
mailbox_hash: MailboxHash,
ret: &mut String,
@@ -554,17 +555,19 @@ impl ImapConnection {
self.send_command(
format!(
"SELECT \"{}\"",
self.uid_store.mailboxes.read().unwrap()[&mailbox_hash].imap_path()
self.uid_store.mailboxes.lock().await[&mailbox_hash].imap_path()
)
.as_bytes(),
)?;
self.read_response(ret, RequiredResponses::SELECT_REQUIRED)?;
)
.await?;
self.read_response(ret, RequiredResponses::SELECT_REQUIRED)
.await?;
debug!("select response {}", ret);
self.current_mailbox = MailboxSelection::Select(mailbox_hash);
Ok(())
}

pub fn examine_mailbox(
pub async fn examine_mailbox(
&mut self,
mailbox_hash: MailboxHash,
ret: &mut String,
@@ -576,37 +579,44 @@ impl ImapConnection {
self.send_command(
format!(
"EXAMINE \"{}\"",
self.uid_store.mailboxes.read().unwrap()[&mailbox_hash].imap_path()
self.uid_store.mailboxes.lock().await[&mailbox_hash].imap_path()
)
.as_bytes(),
)?;
self.read_response(ret, RequiredResponses::EXAMINE_REQUIRED)?;
)
.await?;
self.read_response(ret, RequiredResponses::EXAMINE_REQUIRED)
.await?;
debug!("examine response {}", ret);
self.current_mailbox = MailboxSelection::Examine(mailbox_hash);
Ok(())
}

pub fn unselect(&mut self) -> Result<()> {
pub async fn unselect(&mut self) -> Result<()> {
match self.current_mailbox.take() {
MailboxSelection::Examine(mailbox_hash) | MailboxSelection::Select(mailbox_hash) => {
let mut response = String::with_capacity(8 * 1024);
if self
.capabilities
.iter()
.any(|cap| cap.eq_ignore_ascii_case(b"UNSELECT"))
{
self.send_command(b"UNSELECT")?;
self.read_response(&mut response, RequiredResponses::empty())?;
} else {
/* `RFC3691 - UNSELECT Command` states: "[..] IMAP4 provides this
* functionality (via a SELECT command with a nonexistent mailbox name or
* reselecting the same mailbox with EXAMINE command)[..]
*/

self.select_mailbox(mailbox_hash, &mut response, true)?;
self.examine_mailbox(mailbox_hash, &mut response, true)?;
}
},
MailboxSelection::Examine(mailbox_hash) |
MailboxSelection::Select(mailbox_hash) =>{
let mut response = String::with_capacity(8 * 1024);
if self
.uid_store
.capabilities
.lock()
.unwrap()
.iter()
.any(|cap| cap.eq_ignore_ascii_case(b"UNSELECT"))
{
self.send_command(b"UNSELECT").await?;
self.read_response(&mut response, RequiredResponses::empty())
.await?;
} else {
/* `RFC3691 - UNSELECT Command` states: "[..] IMAP4 provides this
* functionality (via a SELECT command with a nonexistent mailbox name or
* reselecting the same mailbox with EXAMINE command)[..]
*/
self.select_mailbox(mailbox_hash, &mut response, true).await?;
self.examine_mailbox(mailbox_hash, &mut response, true).await?;
}
},
MailboxSelection::None => {},
}
Ok(())
@@ -623,12 +633,19 @@ impl ImapConnection {
}
}

pub fn create_uid_msn_cache(&mut self, mailbox_hash: MailboxHash, low: usize) -> Result<()> {
pub async fn create_uid_msn_cache(
&mut self,
mailbox_hash: MailboxHash,
low: usize,
) -> Result<()> {
debug_assert!(low > 0);
let mut response = String::new();
self.examine_mailbox(mailbox_hash, &mut response, false)?;
self.send_command(format!("UID SEARCH {}:*", low).as_bytes())?;
self.read_response(&mut response, RequiredResponses::SEARCH)?;
self.examine_mailbox(mailbox_hash, &mut response, false)
.await?;
self.send_command(format!("UID SEARCH {}:*", low).as_bytes())
.await?;
self.read_response(&mut response, RequiredResponses::SEARCH)
.await?;
debug!("uid search response {:?}", &response);
let mut msn_index_lck = self.uid_store.msn_index.lock().unwrap();
let msn_index = msn_index_lck.entry(mailbox_hash).or_default();
@@ -651,27 +668,7 @@ pub struct ImapBlockingConnection {
}

impl From<ImapConnection> for ImapBlockingConnection {
fn from(mut conn: ImapConnection) -> Self {
conn.set_nonblocking(false)
.expect("set_nonblocking call failed");
conn.stream
.as_mut()
.map(|s| {
s.stream
.get_mut()
.set_write_timeout(Some(std::time::Duration::new(30, 0)))
.expect("set_write_timeout call failed")
})
.expect("set_write_timeout call failed");
conn.stream
.as_mut()
.map(|s| {
s.stream
.get_mut()
.set_read_timeout(Some(std::time::Duration::new(30, 0)))
.expect("set_read_timeout call failed")
})
.expect("set_read_timeout call failed");
fn from(conn: ImapConnection) -> Self {
ImapBlockingConnection {
buf: [0; 1024],
conn,
@@ -690,63 +687,79 @@ impl ImapBlockingConnection {
pub fn err(&self) -> Option<&str> {
self.err.as_ref().map(String::as_str)
}
}

impl Iterator for ImapBlockingConnection {
type Item = Vec<u8>;
fn next(&mut self) -> Option<Self::Item> {
pub fn as_stream<'a>(&'a mut self) -> impl Future<Output = Option<Vec<u8>>> + 'a {
self.result.drain(0..self.prev_res_length);
self.prev_res_length = 0;
let mut break_flag = false;
let mut prev_failure = None;
let ImapBlockingConnection {
ref mut prev_res_length,
ref mut result,
ref mut conn,
ref mut buf,
ref mut err,
} = self;
loop {
if conn.stream.is_err() {
debug!(&conn.stream);
async move {
if self.conn.stream.is_err() {
debug!(&self.conn.stream);
return None;
}
match conn.stream.as_mut().unwrap().stream.read(buf) {
Ok(0) => return None,
Ok(b) => {
result.extend_from_slice(&buf[0..b]);
debug!(unsafe { std::str::from_utf8_unchecked(result) });
if let Some(pos) = result.find(b"\r\n") {
*prev_res_length = pos + b"\r\n".len();
return Some(result[0..*prev_res_length].to_vec());
}
prev_failure = None;
loop {
if let Some(y) = read(self, &mut break_flag, &mut prev_failure).await {
return Some(y);
}
Err(e)
if e.kind() == std::io::ErrorKind::WouldBlock
|| e.kind() == std::io::ErrorKind::Interrupted =>
{
debug!(&e);
if let Some(prev_failure) = prev_failure.as_ref() {
if Instant::now().duration_since(*prev_failure)
>= std::time::Duration::new(60 * 5, 0)
{
*err = Some(e.to_string());
return None;
}
} else {
prev_failure = Some(Instant::now());
}
continue;
if break_flag {
return None;
}
Err(e) => {
debug!(&conn.stream);
debug!(&e);
}
}
}
}

async fn read(
conn: &mut ImapBlockingConnection,
break_flag: &mut bool,
prev_failure: &mut Option<std::time::Instant>,
) -> Option<Vec<u8>> {
let ImapBlockingConnection {
ref mut prev_res_length,
ref mut result,
ref mut conn,
ref mut buf,
ref mut err,
} = conn;

match conn.stream.as_mut().unwrap().stream.read(buf).await {
Ok(0) => {
*break_flag = true;
}
Ok(b) => {
result.extend_from_slice(&buf[0..b]);
debug!(unsafe { std::str::from_utf8_unchecked(result) });
if let Some(pos) = result.find(b"\r\n") {
*prev_res_length = pos + b"\r\n".len();
return Some(result[0..*prev_res_length].to_vec());
}
*prev_failure = None;
}
Err(e)
if e.kind() == std::io::ErrorKind::WouldBlock
|| e.kind() == std::io::ErrorKind::Interrupted =>
{
debug!(&e);
if let Some(prev_failure) = prev_failure.as_ref() {
if Instant::now().duration_since(*prev_failure)
>= std::time::Duration::new(60 * 5, 0)
{
*err = Some(e.to_string());
return None;
*break_flag = true;
}
} else {
*prev_failure = Some(Instant::now());
}
}
Err(e) => {
debug!(&conn.stream);
debug!(&e);
*err = Some(e.to_string());
*break_flag = true;
}
}
None
}

fn lookup_ipv4(host: &str, port: u16) -> Result<SocketAddr> {


+ 1
- 0
melib/src/backends/imap/managesieve.rs View File

@@ -97,6 +97,7 @@ pub fn new_managesieve_connection(s: &AccountSettings) -> Result<ImapConnection>
server_password: server_password.to_string(),
server_port,
use_starttls: true,
use_tls: true,
danger_accept_invalid_certs,
protocol: ImapProtocol::ManageSieve,
};


+ 82
- 56
melib/src/backends/imap/operations.rs View File

@@ -20,17 +20,18 @@
*/

use super::*;

use crate::backends::*;
use crate::email::*;
use crate::error::{MeliError, Result};
use std::sync::{Arc, Mutex};
use std::sync::Arc;

/// `BackendOp` implementor for Imap
#[derive(Debug, Clone)]
pub struct ImapOp {
uid: usize,
mailbox_hash: MailboxHash,
connection: Arc<Mutex<ImapConnection>>,
connection: Arc<FutureMutex<ImapConnection>>,
uid_store: Arc<UIDStore>,
}

@@ -38,7 +39,7 @@ impl ImapOp {
pub fn new(
uid: usize,
mailbox_hash: MailboxHash,
connection: Arc<Mutex<ImapConnection>>,
connection: Arc<FutureMutex<ImapConnection>>,
uid_store: Arc<UIDStore>,
) -> Self {
ImapOp {
@@ -52,44 +53,63 @@ impl ImapOp {

impl BackendOp for ImapOp {
fn as_bytes(&mut self) -> ResultFuture<Vec<u8>> {
let mut bytes_cache = self.uid_store.byte_cache.lock()?;
let cache = bytes_cache.entry(self.uid).or_default();
if cache.bytes.is_none() {
let mut response = String::with_capacity(8 * 1024);
{
let mut conn = try_lock(&self.connection, Some(std::time::Duration::new(2, 0)))?;
conn.examine_mailbox(self.mailbox_hash, &mut response)?;
conn.send_command(format!("UID FETCH {} (FLAGS RFC822)", self.uid).as_bytes())?;
conn.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)?;
}
debug!(
"fetch response is {} bytes and {} lines",
response.len(),
response.lines().count()
);
let UidFetchResponse {
uid, flags, body, ..
} = protocol_parser::uid_fetch_response(&response)?.1;
assert_eq!(uid, self.uid);
assert!(body.is_some());
let mut bytes_cache = self.uid_store.byte_cache.lock()?;
let cache = bytes_cache.entry(self.uid).or_default();
if let Some((flags, _)) = flags {
cache.flags = Some(flags);
let connection = self.connection.clone();
let mailbox_hash = self.mailbox_hash;
let uid = self.uid;
let uid_store = self.uid_store.clone();
Ok(Box::pin(async move {
let exists_in_cache = {
let mut bytes_cache = uid_store.byte_cache.lock()?;
let cache = bytes_cache.entry(uid).or_default();
cache.bytes.is_some()
};
if !exists_in_cache {
let mut response = String::with_capacity(8 * 1024);
{
let mut conn = connection.lock().await;
conn.examine_mailbox(mailbox_hash, &mut response, false)
.await?;
conn.send_command(format!("UID FETCH {} (FLAGS RFC822)", uid).as_bytes())
.await?;
conn.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
.await?;
}
debug!(
"fetch response is {} bytes and {} lines",
response.len(),
response.lines().collect::<Vec<&str>>().len()
);
let UidFetchResponse {
uid: _uid,
flags: _flags,
body,
..
} = protocol_parser::uid_fetch_response(&response)?.1;
assert_eq!(_uid, uid);
assert!(body.is_some());
let mut bytes_cache = uid_store.byte_cache.lock()?;
let cache = bytes_cache.entry(uid).or_default();
if let Some((_flags, _)) = _flags {
//flags.lock().await.set(Some(_flags));
cache.flags = Some(_flags);
}
cache.bytes =
Some(unsafe { std::str::from_utf8_unchecked(body.unwrap()).to_string() });
}
cache.bytes = Some(unsafe { std::str::from_utf8_unchecked(body.unwrap()).to_string() });
}
let ret = cache.bytes.clone().unwrap().into_bytes();
Ok(Box::pin(async move { Ok(ret) }))
let mut bytes_cache = uid_store.byte_cache.lock()?;
let cache = bytes_cache.entry(uid).or_default();
let ret = cache.bytes.clone().unwrap().into_bytes();
Ok(ret)
}))
}

fn fetch_flags(&self) -> ResultFuture<Flag> {
let mut response = String::with_capacity(8 * 1024);
let connection = self.connection.clone();
let mailbox_hash = self.mailbox_hash;
let uid = self.uid;
let uid_store = self.uid_store.clone();

let mut response = String::with_capacity(8 * 1024);
Ok(Box::pin(async move {
let exists_in_cache = {
let mut bytes_cache = uid_store.byte_cache.lock()?;
@@ -97,14 +117,17 @@ impl BackendOp for ImapOp {
cache.flags.is_some()
};
if !exists_in_cache {
let mut conn = try_lock(&connection, Some(std::time::Duration::new(2, 0)))?;
conn.examine_mailbox(mailbox_hash, &mut response, false)?;
conn.send_command(format!("UID FETCH {} FLAGS", uid).as_bytes())?;
conn.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)?;
let mut conn = connection.lock().await;
conn.examine_mailbox(mailbox_hash, &mut response, false)
.await?;
conn.send_command(format!("UID FETCH {} FLAGS", uid).as_bytes())
.await?;
conn.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
.await?;
debug!(
"fetch response is {} bytes and {} lines",
response.len(),
response.lines().count()
response.lines().collect::<Vec<&str>>().len()
);
let v = protocol_parser::uid_fetch_flags_response(response.as_bytes())
.map(|(_, v)| v)
@@ -121,7 +144,7 @@ impl BackendOp for ImapOp {
.set_summary(format!("message with UID {} was not found?", uid)));
}
let (_uid, (_flags, _)) = v[0];
assert_eq!(_uid, uid);
assert_eq!(uid, uid);
let mut bytes_cache = uid_store.byte_cache.lock()?;
let cache = bytes_cache.entry(uid).or_default();
cache.flags = Some(_flags);
@@ -139,21 +162,22 @@ impl BackendOp for ImapOp {

fn set_flag(
&mut self,
f: Flag,
flag: Flag,
value: bool,
) -> Result<Pin<Box<dyn Future<Output = Result<()>> + Send>>> {
let flags = self.fetch_flags()?;

let mut response = String::with_capacity(8 * 1024);
let connection = self.connection.clone();
let mailbox_hash = self.mailbox_hash;
let uid = self.uid;
let uid_store = self.uid_store.clone();

let mut response = String::with_capacity(8 * 1024);
Ok(Box::pin(async move {
let mut flags = flags.await?;
flags.set(f, value);
let mut conn = try_lock(&connection, Some(std::time::Duration::new(2, 0)))?;
conn.select_mailbox(mailbox_hash, &mut response, false)?;
flags.set(flag, value);
let mut conn = connection.lock().await;
conn.select_mailbox(mailbox_hash, &mut response, false)
.await?;
debug!(&response);
conn.send_command(
format!(
@@ -162,8 +186,10 @@ impl BackendOp for ImapOp {
flags_to_imap_list!(flags)
)
.as_bytes(),
)?;
conn.read_response(&mut response, RequiredResponses::STORE_REQUIRED)?;
)
.await?;
conn.read_response(&mut response, RequiredResponses::STORE_REQUIRED)
.await?;
debug!(&response);
match protocol_parser::uid_fetch_flags_response(response.as_bytes())
.map(|(_, v)| v)
@@ -172,13 +198,11 @@ impl BackendOp for ImapOp {
Ok(v) => {
if v.len() == 1 {
debug!("responses len is {}", v.len());
let (uid, (_flags, _)) = v[0];
assert_eq!(uid, uid);
let (_uid, (_flags, _)) = v[0];
assert_eq!(_uid, uid);
}
}
Err(e) => {
return Err(e);
}
Err(e) => Err(e)?,
}
let mut bytes_cache = uid_store.byte_cache.lock()?;
let cache = bytes_cache.entry(uid).or_default();
@@ -197,10 +221,10 @@ impl BackendOp for ImapOp {
let mailbox_hash = self.mailbox_hash;
let uid = self.uid;
let uid_store = self.uid_store.clone();

Ok(Box::pin(async move {
let mut conn = try_lock(&connection, Some(std::time::Duration::new(2, 0)))?;
conn.select_mailbox(mailbox_hash, &mut response, false)?;
let mut conn = connection.lock().await;
conn.select_mailbox(mailbox_hash, &mut response, false)
.await?;
conn.send_command(
format!(
"UID STORE {} {}FLAGS.SILENT ({})",
@@ -209,8 +233,10 @@ impl BackendOp for ImapOp {
&tag
)
.as_bytes(),
)?;
conn.read_response(&mut response, RequiredResponses::STORE_REQUIRED)?;
)
.await?;
conn.read_response(&mut response, RequiredResponses::STORE_REQUIRED)
.await?;
protocol_parser::uid_fetch_flags_response(response.as_bytes())
.map(|(_, v)| v)
.map_err(MeliError::from)?;


+ 42
- 22
melib/src/backends/imap/protocol_parser.rs View File

@@ -126,6 +126,26 @@ impl RequiredResponses {
}
}

#[test]
fn test_imap_required_responses() {
let mut ret = String::new();
let required_responses = RequiredResponses::FETCH_REQUIRED;
let response =
&"* 1040 FETCH (UID 1064 FLAGS ())\r\nM15 OK Fetch completed (0.001 + 0.299 secs).\r\n"
[0..];
for l in response.split_rn() {
/*debug!("check line: {}", &l);*/
if required_responses.check(l) {
ret.push_str(l);
}
}
assert_eq!(&ret, "* 1040 FETCH (UID 1064 FLAGS ())\r\n");
let v = protocol_parser::uid_fetch_flags_response(response.as_bytes())
.unwrap()
.1;
assert_eq!(v.len(), 1);
}

#[derive(Debug)]
pub struct Alert(String);
pub type ImapParseResult<'a, T> = Result<(&'a str, T, Option<Alert>)>;
@@ -171,18 +191,18 @@ impl std::fmt::Display for ResponseCode {
fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
use ResponseCode::*;
match self {
Alert(s)=> write!(fmt, "ALERT: {}", s),
Badcharset(None)=> write!(fmt, "Given charset is not supported by this server."),
Badcharset(Some(s))=> write!(fmt, "Given charset is not supported by this server. Supported ones are: {}", s),
Capability => write!(fmt, "Capability response"),
Parse(s) => write!(fmt, "Server error in parsing message headers: {}", s),
Permanentflags(s) => write!(fmt, "Mailbox supports these flags: {}", s),
ReadOnly=> write!(fmt, "This mailbox is selected read-only."),
ReadWrite => write!(fmt, "This mailbox is selected with read-write permissions."),
Trycreate => write!(fmt, "Failed to operate on the target mailbox because it doesn't exist. Try creating it first."),
Uidnext(uid) => write!(fmt, "Next UID value is {}", uid),
Uidvalidity(uid) => write!(fmt, "Next UIDVALIDITY value is {}", uid),
Unseen(uid) => write!(fmt, "First message without the \\Seen flag is {}", uid),
Alert(s)=> write!(fmt, "ALERT: {}", s),
Badcharset(None)=> write!(fmt, "Given charset is not supported by this server."),
Badcharset(Some(s))=> write!(fmt, "Given charset is not supported by this server. Supported ones are: {}", s),
Capability => write!(fmt, "Capability response"),
Parse(s) => write!(fmt, "Server error in parsing message headers: {}", s),
Permanentflags(s) => write!(fmt, "Mailbox supports these flags: {}", s),
ReadOnly=> write!(fmt, "This mailbox is selected read-only."),
ReadWrite => write!(fmt, "This mailbox is selected with read-write permissions."),
Trycreate => write!(fmt, "Failed to operate on the target mailbox because it doesn't exist. Try creating it first."),
Uidnext(uid) => write!(fmt, "Next UID value is {}", uid),
Uidvalidity(uid) => write!(fmt, "Next UIDVALIDITY value is {}", uid),
Unseen(uid) => write!(fmt, "First message without the \\Seen flag is {}", uid),
}
}
}
@@ -190,7 +210,7 @@ ReadWrite => write!(fmt, "This mailbox is selected with read-write permissions."
impl ResponseCode {
fn from(val: &str) -> ResponseCode {
use ResponseCode::*;
if !val.starts_with('[') {
if !val.starts_with("[") {
let msg = val.trim();
return Alert(msg.to_string());
}
@@ -235,8 +255,7 @@ pub enum ImapResponse {

impl<T: AsRef<str>> From<T> for ImapResponse {
fn from(val: T) -> ImapResponse {
let val_ref = val.as_ref();
let val: &str = val_ref.split_rn().last().unwrap_or(val_ref);
let val: &str = val.as_ref().split_rn().last().unwrap_or(val.as_ref());
debug!(&val);
let mut val = val[val.as_bytes().find(b" ").unwrap() + 1..].trim();
// M12 NO [CANNOT] Invalid mailbox name: Name must not have \'/\' characters (0.000 + 0.098 + 0.097 secs).\r\n
@@ -617,7 +636,7 @@ pub fn uid_fetch_responses(mut input: &str) -> ImapParseResult<Vec<UidFetchRespo
if let Some(el_alert) = el_alert {
match &mut alert {
Some(Alert(ref mut alert)) => {
alert.push_str(&el_alert.0);
alert.extend(el_alert.0.chars());
}
a @ None => *a = Some(el_alert),
}
@@ -685,7 +704,7 @@ pub fn uid_fetch_response_(
pub fn uid_fetch_flags_response(input: &[u8]) -> IResult<&[u8], Vec<(usize, (Flag, Vec<String>))>> {
many0(|input| -> IResult<&[u8], (usize, (Flag, Vec<String>))> {
let (input, _) = tag("* ")(input)?;
let (input, _) = take_while(is_digit)(input)?;
let (input, _msn) = take_while(is_digit)(input)?;
let (input, _) = tag(" FETCH (")(input)?;
let (input, uid_flags) = permutation((
preceded(
@@ -696,7 +715,7 @@ pub fn uid_fetch_flags_response(input: &[u8]) -> IResult<&[u8], Vec<(usize, (Fla
alt((tag("FLAGS "), tag(" FLAGS "))),
delimited(tag("("), byte_flags, tag(")")),
),
))(input.ltrim())?;
))(input)?;
let (input, _) = tag(")\r\n")(input)?;
Ok((input, (uid_flags.0, uid_flags.1)))
})(input)
@@ -832,6 +851,7 @@ pub enum UntaggedResponse {
}

pub fn untagged_responses(input: &[u8]) -> IResult<&[u8], Option<UntaggedResponse>> {
debug!("Parse untagged response from {:?}", to_str!(input));
let (input, _) = tag("* ")(input)?;
let (input, num) = map_res(digit1, |s| usize::from_str(to_str!(s)))(input)?;
let (input, _) = tag(" ")(input)?;
@@ -995,13 +1015,13 @@ pub fn flags(input: &str) -> IResult<&str, (Flag, Vec<String>)> {
let mut keywords = Vec::new();

let mut input = input;
while !input.starts_with(')') && !input.is_empty() {
if input.starts_with('\\') {
while !input.starts_with(")") && !input.is_empty() {
if input.starts_with("\\") {
input = &input[1..];
}
let mut match_end = 0;
while match_end < input.len() {
if input[match_end..].starts_with(' ') || input[match_end..].starts_with(')') {
if input[match_end..].starts_with(" ") || input[match_end..].starts_with(")") {
break;
}
match_end += 1;
@@ -1344,7 +1364,7 @@ pub fn quoted(input: &[u8]) -> IResult<&[u8], Vec<u8>> {
}

pub fn quoted_or_nil(input: &[u8]) -> IResult<&[u8], Option<Vec<u8>>> {
alt((map(tag("NIL"), |_| None), map(quoted, Some)))(input.ltrim())
alt((map(tag("NIL"), |_| None), map(quoted, |v| Some(v))))(input.ltrim())
/*
alt_complete!(map!(ws!(tag!("NIL")), |_| None) | map!(quoted, |v| Some(v))));
*/


+ 15
- 15
melib/src/backends/imap/untagged.rs View File

@@ -33,7 +33,7 @@ use crate::error::*;
use std::time::Instant;

impl ImapConnection {
pub fn process_untagged(&mut self, line: &str) -> Result<bool> {
pub async fn process_untagged(&mut self, line: &str) -> Result<bool> {
macro_rules! try_fail {
($mailbox_hash: expr, $($result:expr)+) => {
$(if let Err(err) = $result {
@@ -51,12 +51,13 @@ impl ImapConnection {
} else { Ok(()) }?;)+
};
}
//FIXME
let mailbox_hash = match self.current_mailbox {
MailboxSelection::Select(h) | MailboxSelection::Examine(h) => h,
MailboxSelection::None => return Ok(false),
};
let mailbox =
std::clone::Clone::clone(&self.uid_store.mailboxes.read().unwrap()[&mailbox_hash]);
std::clone::Clone::clone(&self.uid_store.mailboxes.lock().await[&mailbox_hash]);

let mut response = String::with_capacity(8 * 1024);
let untagged_response =
@@ -102,20 +103,19 @@ impl ImapConnection {
UntaggedResponse::Exists(n) => {
/* UID FETCH ALL UID, cross-ref, then FETCH difference headers
* */
let mut prev_exists = mailbox.exists.lock().unwrap();
debug!("exists {}", n);
if n > prev_exists.len() {
if n > mailbox.exists.lock().unwrap().len() {
try_fail!(
mailbox_hash,
self.send_command(
&[
b"FETCH",
format!("{}:{}", prev_exists.len() + 1, n).as_bytes(),
format!("{}:{}", mailbox.exists.lock().unwrap().len() + 1, n).as_bytes(),
b"(UID FLAGS RFC822)",
]
.join(&b' '),
)
self.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
).await
self.read_response(&mut response, RequiredResponses::FETCH_REQUIRED).await
);
match super::protocol_parser::uid_fetch_responses(&response) {
Ok((_, v, _)) => {
@@ -165,7 +165,7 @@ impl ImapConnection {
if !env.is_seen() {
mailbox.unseen.lock().unwrap().insert_new(env.hash());
}
prev_exists.insert_new(env.hash());
mailbox.exists.lock().unwrap().insert_new(env.hash());
self.add_refresh_event(RefreshEvent {
account_hash: self.uid_store.account_hash,
mailbox_hash,
@@ -183,8 +183,8 @@ impl ImapConnection {
UntaggedResponse::Recent(_) => {
try_fail!(
mailbox_hash,
self.send_command(b"UID SEARCH RECENT")
self.read_response(&mut response, RequiredResponses::SEARCH)
self.send_command(b"UID SEARCH RECENT").await
self.read_response(&mut response, RequiredResponses::SEARCH).await
);
match super::protocol_parser::search_results_raw(response.as_bytes())
.map(|(_, v)| v)
@@ -199,8 +199,8 @@ impl ImapConnection {
self.send_command(
&[b"UID FETCH", v, b"(FLAGS RFC822)"]
.join(&b' '),
)
self.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
).await
self.read_response(&mut response, RequiredResponses::FETCH_REQUIRED).await
);
debug!(&response);
match super::protocol_parser::uid_fetch_responses(&response) {
@@ -292,8 +292,8 @@ impl ImapConnection {
format!("{}", msg_seq).as_bytes(),
]
.join(&b' '),
)
self.read_response(&mut response, RequiredResponses::SEARCH)
).await
self.read_response(&mut response, RequiredResponses::SEARCH).await
);
debug!(&response);
match super::protocol_parser::search_results(
@@ -304,7 +304,7 @@ impl ImapConnection {
Ok(mut v) => {
if let Some(uid) = v.pop() {
let lck = self.uid_store.uid_index.lock().unwrap();
let env_hash = lck.get(&(mailbox_hash, uid)).map(|&h| h);
let env_hash = lck.get(&(mailbox_hash, uid)).copied();
drop(lck);
if let Some(env_hash) = env_hash {
if !flags.0.intersects(crate::email::Flag::SEEN) {


+ 75
- 213
melib/src/backends/imap/watch.rs View File

@@ -22,26 +22,23 @@ use super::*;
use crate::backends::SpecialUsageMailbox;
use crate::email::parser::BytesExt;
use crate::email::parser::BytesIterExt;
use std::sync::{Arc, Mutex};
use std::sync::Arc;

/// Arguments for IMAP watching functions
pub struct ImapWatchKit {
pub conn: ImapConnection,
pub main_conn: Arc<Mutex<ImapConnection>>,
pub main_conn: Arc<FutureMutex<ImapConnection>>,
pub uid_store: Arc<UIDStore>,
pub work_context: WorkContext,
}

macro_rules! exit_on_error {
($conn:expr, $mailbox_hash:ident, $work_context:ident, $thread_id:ident, $($result:expr)+) => {
($conn:expr, $mailbox_hash:ident, $($result:expr)+) => {
$(if let Err(e) = $result {
*$conn.uid_store.is_online.lock().unwrap() = (
Instant::now(),
Err(e.clone()),
);
debug!("failure: {}", e.to_string());
$work_context.set_status.send(($thread_id, e.to_string())).unwrap();
$work_context.finished.send($thread_id).unwrap();
let account_hash = $conn.uid_store.account_hash;
$conn.add_refresh_event(RefreshEvent {
account_hash,
@@ -53,50 +50,29 @@ macro_rules! exit_on_error {
};
}

pub fn poll_with_examine(kit: ImapWatchKit) -> Result<()> {
pub async fn poll_with_examine(kit: ImapWatchKit) -> Result<()> {
debug!("poll with examine");
let ImapWatchKit {
mut conn,
main_conn,
uid_store,
work_context,
} = kit;
loop {
if super::try_lock(&uid_store.is_online, Some(std::time::Duration::new(10, 0)))?
.1
.is_ok()
{
break;
}
std::thread::sleep(std::time::Duration::from_millis(100));
}
conn.connect()?;
conn.connect().await?;
let mut response = String::with_capacity(8 * 1024);
let thread_id: std::thread::ThreadId = std::thread::current().id();
loop {
work_context
.set_status
.send((thread_id, "sleeping...".to_string()))
.unwrap();
std::thread::sleep(std::time::Duration::from_millis(5 * 60 * 1000));
let mailboxes = uid_store.mailboxes.read()?;
let mailboxes = uid_store.mailboxes.lock().await;
for mailbox in mailboxes.values() {
work_context
.set_status
.send((
thread_id,
format!("examining `{}` for updates...", mailbox.path()),
))
.unwrap();
examine_updates(mailbox, &mut conn, &uid_store, &work_context)?;
examine_updates(mailbox, &mut conn, &uid_store).await?;
}
let mut main_conn = super::try_lock(&main_conn, Some(std::time::Duration::new(10, 0)))?;
main_conn.send_command(b"NOOP")?;
main_conn.read_response(&mut response, RequiredResponses::empty())?;
let mut main_conn = main_conn.lock().await;
main_conn.send_command(b"NOOP").await?;
main_conn
.read_response(&mut response, RequiredResponses::empty())
.await?;
}
}

pub fn idle(kit: ImapWatchKit) -> Result<()> {
pub async fn idle(kit: ImapWatchKit) -> Result<()> {
debug!("IDLE");
/* IDLE only watches the connection's selected mailbox. We will IDLE on INBOX and every ~5
* minutes wake up and poll the others */
@@ -104,23 +80,12 @@ pub fn idle(kit: ImapWatchKit) -> Result<()> {
mut conn,
main_conn,
uid_store,
work_context,
} = kit;
loop {
if super::try_lock(&uid_store.is_online, Some(std::time::Duration::new(10, 0)))?
.1
.is_ok()
{
break;
}
std::thread::sleep(std::time::Duration::from_millis(100));
}
conn.connect()?;
let thread_id: std::thread::ThreadId = std::thread::current().id();
conn.connect().await?;
let mailbox: ImapMailbox = match uid_store
.mailboxes
.read()
.unwrap()
.lock()
.await
.values()
.find(|f| f.parent.is_none() && (f.special_usage() == SpecialUsageMailbox::Inbox))
.map(std::clone::Clone::clone)
@@ -129,10 +94,6 @@ pub fn idle(kit: ImapWatchKit) -> Result<()> {
None => {
let err = MeliError::new("INBOX mailbox not found in local mailbox index. meli may have not parsed the IMAP mailboxes correctly");
debug!("failure: {}", err.to_string());
work_context
.set_status
.send((thread_id, err.to_string()))
.unwrap();
conn.add_refresh_event(RefreshEvent {
account_hash: uid_store.account_hash,
mailbox_hash: 0,
@@ -147,10 +108,10 @@ pub fn idle(kit: ImapWatchKit) -> Result<()> {
exit_on_error!(
conn,
mailbox_hash,
work_context,
thread_id,
conn.send_command(format!("SELECT \"{}\"", mailbox.imap_path()).as_bytes())
.await
conn.read_response(&mut response, RequiredResponses::SELECT_REQUIRED)
.await
);
debug!("select response {}", &response);
{
@@ -197,71 +158,44 @@ pub fn idle(kit: ImapWatchKit) -> Result<()> {
}
Err(e) => {
debug!("{:?}", e);
panic!("could not select mailbox");
return Err(e).chain_err_summary(|| "could not select mailbox");
}
};
}
exit_on_error!(
conn,
mailbox_hash,
work_context,
thread_id,
conn.send_command(b"IDLE")
);
work_context
.set_status
.send((thread_id, "IDLEing".to_string()))
.unwrap();
let mut iter = ImapBlockingConnection::from(conn);
exit_on_error!(conn, mailbox_hash, conn.send_command(b"IDLE").await);
let mut blockn = ImapBlockingConnection::from(conn);
let mut beat = std::time::Instant::now();
let mut watch = std::time::Instant::now();
/* duration interval to send heartbeat */
let _26_mins = std::time::Duration::from_secs(26 * 60);
const _26_MINS: std::time::Duration = std::time::Duration::from_secs(26 * 60);
/* duration interval to check other mailboxes for changes */
let _5_mins = std::time::Duration::from_secs(5 * 60);
while let Some(line) = iter.next() {
const _5_MINS: std::time::Duration = std::time::Duration::from_secs(5 * 60);
while let Some(line) = blockn.as_stream().await {
let now = std::time::Instant::now();
if now.duration_since(beat) >= _26_mins {
let mut main_conn_lck =
super::try_lock(&main_conn, Some(std::time::Duration::new(10, 0)))?;
if now.duration_since(beat) >= _26_MINS {
let mut main_conn_lck = main_conn.lock().await;
exit_on_error!(
iter.conn,
blockn.conn,
mailbox_hash,
work_context,
thread_id,
iter.conn.set_nonblocking(true)
iter.conn.send_raw(b"DONE")
iter.conn.read_response(&mut response, RequiredResponses::empty())
iter.conn.send_command(b"IDLE")
iter.conn.set_nonblocking(false)
main_conn_lck.send_command(b"NOOP")
main_conn_lck.read_response(&mut response, RequiredResponses::empty())
blockn.conn.send_raw(b"DONE").await
blockn.conn.read_response(&mut response, RequiredResponses::empty()).await
blockn.conn.send_command(b"IDLE").await
main_conn_lck.send_command(b"NOOP").await
main_conn_lck.read_response(&mut response, RequiredResponses::empty()).await
);
beat = now;
}
if now.duration_since(watch) >= _5_mins {
if now.duration_since(watch) >= _5_MINS {
/* Time to poll all inboxes */
let mut conn = try_lock(&main_conn, Some(std::time::Duration::new(10, 0)))?;
for mailbox in uid_store.mailboxes.read().unwrap().values() {
work_context
.set_status
.send((
thread_id,
format!("examining `{}` for updates...", mailbox.path()),
))
.unwrap();
let mut conn = main_conn.lock().await;
let mailboxes = uid_store.mailboxes.lock().await;
for mailbox in mailboxes.values() {
exit_on_error!(
conn,
mailbox_hash,
work_context,
thread_id,
examine_updates(mailbox, &mut conn, &uid_store, &work_context,)
examine_updates(mailbox, &mut conn, &uid_store).await
);
}
work_context
.set_status
.send((thread_id, "done examining mailboxes.".to_string()))
.unwrap();
watch = now;
}
*uid_store.is_online.lock().unwrap() = (Instant::now(), Ok(()));
@@ -269,21 +203,15 @@ pub fn idle(kit: ImapWatchKit) -> Result<()> {
.map(|(_, v)| v)
.map_err(MeliError::from)
{
Ok(Some(Recent(r))) => {
let mut conn = super::try_lock(&main_conn, Some(std::time::Duration::new(10, 0)))?;
work_context
.set_status
.send((thread_id, format!("got `{} RECENT` notification", r)))
.unwrap();
Ok(Some(Recent(_r))) => {
let mut conn = main_conn.lock().await;
/* UID SEARCH RECENT */
exit_on_error!(
conn,
mailbox_hash,
work_context,
thread_id,
conn.examine_mailbox(mailbox_hash, &mut response, false)
conn.send_command(b"UID SEARCH RECENT")
conn.read_response(&mut response, RequiredResponses::SEARCH)
conn.examine_mailbox(mailbox_hash, &mut response, false).await
conn.send_command(b"UID SEARCH RECENT").await
conn.read_response(&mut response, RequiredResponses::SEARCH).await
);
match protocol_parser::search_results_raw(response.as_bytes())
.map(|(_, v)| v)
@@ -296,31 +224,19 @@ pub fn idle(kit: ImapWatchKit) -> Result<()> {
exit_on_error!(
conn,
mailbox_hash,
work_context,
thread_id,
conn.send_command(
&[&b"UID FETCH"[..], &v.trim().split(|b| b == &b' ').join(b','), &b"(FLAGS RFC822)"[..]]
.join(&b' '),
)
conn.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
).await
conn.read_response(&mut response, RequiredResponses::FETCH_REQUIRED).await
);
debug!(&response);
match protocol_parser::uid_fetch_responses(&response) {
Ok((_, v, _)) => {
let len = v.len();
let mut ctr = 0;
for UidFetchResponse {
uid, flags, body, ..
} in v
{
work_context
.set_status
.send((
thread_id,
format!("parsing {}/{} envelopes..", ctr, len),
))
.unwrap();
ctr += 1;
if !uid_store
.uid_index
.lock()
@@ -328,6 +244,10 @@ pub fn idle(kit: ImapWatchKit) -> Result<()> {
.contains_key(&(mailbox_hash, uid))
{
if let Ok(mut env) = Envelope::from_bytes(
/* unwrap() is safe since we ask for RFC822 in the
* above FETCH, thus uid_fetch_responses() if
* returns a successful parse, it will include the
* RFC822 response */
body.unwrap(),
flags.as_ref().map(|&(f, _)| f),
) {
@@ -383,10 +303,6 @@ pub fn idle(kit: ImapWatchKit) -> Result<()> {
}
}
}
work_context
.set_status
.send((thread_id, format!("parsed {}/{} envelopes.", ctr, len)))
.unwrap();
}
Err(e) => {
debug!(e);
@@ -409,11 +325,7 @@ pub fn idle(kit: ImapWatchKit) -> Result<()> {
// immediately decremented by 1, and this decrement is reflected in
// message sequence numbers in subsequent responses (including other
// untagged EXPUNGE responses).
let mut conn = super::try_lock(&main_conn, Some(std::time::Duration::new(10, 0)))?;
work_context
.set_status
.send((thread_id, format!("got `{} EXPUNGED` notification", n)))
.unwrap();
let mut conn = main_conn.lock().await;
let deleted_uid = uid_store
.msn_index
.lock()
@@ -436,69 +348,43 @@ pub fn idle(kit: ImapWatchKit) -> Result<()> {
});
}
Ok(Some(Exists(n))) => {
let mut conn = super::try_lock(&main_conn, Some(std::time::Duration::new(10, 0)))?;
let mut conn = main_conn.lock().await;
/* UID FETCH ALL UID, cross-ref, then FETCH difference headers
* */
let mut prev_exists = mailbox.exists.lock().unwrap();
debug!("exists {}", n);
work_context
.set_status
.send((
thread_id,
format!(
"got `{} EXISTS` notification (EXISTS was previously {} for {}",
n,
prev_exists.len(),
mailbox.path()
),
))
.unwrap();
if n > prev_exists.len() {
if n > mailbox.exists.lock().unwrap().len() {
exit_on_error!(
conn,
mailbox_hash,
work_context,
thread_id,
conn.examine_mailbox(mailbox_hash, &mut response, false)
conn.examine_mailbox(mailbox_hash, &mut response, false).await
conn.send_command(
&[
b"FETCH",
format!("{}:{}", prev_exists.len() + 1, n).as_bytes(),
format!("{}:{}", mailbox.exists.lock().unwrap().len() + 1, n).as_bytes(),
b"(UID FLAGS RFC822)",
]
.join(&b' '),
)
conn.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
).await
conn.read_response(&mut response, RequiredResponses::FETCH_REQUIRED).await
);
match protocol_parser::uid_fetch_responses(&response) {
Ok((_, v, _)) => {
let len = v.len();
let mut ctr = 0;
'fetch_responses_b: for UidFetchResponse {
uid, flags, body, ..
} in v
{
work_context
.set_status
.send((
thread_id,
format!("parsing {}/{} envelopes..", ctr, len),
))
.unwrap();
if uid_store
.uid_index
.lock()
.unwrap()
.contains_key(&(mailbox_hash, uid))
{
ctr += 1;
continue 'fetch_responses_b;
}
if let Ok(mut env) = Envelope::from_bytes(
body.unwrap(),
flags.as_ref().map(|&(f, _)| f),
) {
ctr += 1;
uid_store
.hash_index
.lock()
@@ -536,7 +422,7 @@ pub fn idle(kit: ImapWatchKit) -> Result<()> {
&[(uid, &env)],
)?;
}
prev_exists.insert_new(env.hash());
mailbox.exists.lock().unwrap().insert_new(env.hash());

conn.add_refresh_event(RefreshEvent {
account_hash: uid_store.account_hash,
@@ -545,10 +431,6 @@ pub fn idle(kit: ImapWatchKit) -> Result<()> {
});
}
}
work_context
.set_status
.send((thread_id, format!("parsed {}/{} envelopes.", ctr, len)))
.unwrap();
}
Err(e) => {
debug!(e);
@@ -560,22 +442,20 @@ pub fn idle(kit: ImapWatchKit) -> Result<()> {
/* a * {msg_seq} FETCH (FLAGS ({flags})) was received, so find out UID from msg_seq
* and send update
*/
let mut conn = super::try_lock(&main_conn, Some(std::time::Duration::new(10, 0)))?;
let mut conn = main_conn.lock().await;
debug!("fetch {} {:?}", msg_seq, flags);
exit_on_error!(
conn,
mailbox_hash,
work_context,
thread_id,
conn.examine_mailbox(mailbox_hash, &mut response, false)
conn.examine_mailbox(mailbox_hash, &mut response, false).await
conn.send_command(
&[
b"UID SEARCH ",
format!("{}", msg_seq).as_bytes(),
]
.join(&b' '),
)
conn.read_response(&mut response, RequiredResponses::SEARCH)
).await
conn.read_response(&mut response, RequiredResponses::SEARCH).await
);
match search_results(response.split_rn().next().unwrap_or("").as_bytes())
.map(|(_, v)| v)
@@ -610,19 +490,10 @@ pub fn idle(kit: ImapWatchKit) -> Result<()> {
Ok(Some(Bye { .. })) => break,
Ok(None) | Err(_) => {}
}
work_context
.set_status
.send((thread_id, "IDLEing".to_string()))
.unwrap();
}
debug!("IDLE connection dropped");
let err: &str = iter.err().unwrap_or("Unknown reason.");
work_context
.set_status
.send((thread_id, "IDLE connection dropped".to_string()))
.unwrap();
work_context.finished.send(thread_id).unwrap();
main_conn.lock().unwrap().add_refresh_event(RefreshEvent {
let err: &str = blockn.err().unwrap_or("Unknown reason.");
main_conn.lock().await.add_refresh_event(RefreshEvent {
account_hash: uid_store.account_hash,
mailbox_hash,
kind: RefreshEventKind::Failure(MeliError::new(format!(
@@ -633,22 +504,19 @@ pub fn idle(kit: ImapWatchKit) -> Result<()> {
Err(MeliError::new(format!("IDLE connection dropped: {}", err)))
}

pub fn examine_updates(
pub async fn examine_updates(
mailbox: &ImapMailbox,
conn: &mut ImapConnection,
uid_store: &Arc<UIDStore>,
work_context: &WorkContext,
) -> Result<()> {
let thread_id: std::thread::ThreadId = std::thread::current().id();
let mailbox_hash = mailbox.hash();
debug!("examining mailbox {} {}", mailbox_hash, mailbox.path());
let mut response = String::with_capacity(8 * 1024);
exit_on_error!(
conn,
mailbox_hash,
work_context,
thread_id,
conn.examine_mailbox(mailbox_hash, &mut response, true)
.await
);
*uid_store.is_online.lock().unwrap() = (Instant::now(), Ok(()));
let uidvalidity;
@@ -690,7 +558,6 @@ pub fn examine_updates(
});