mredaelli
/
meli
forked from meli/meli
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add somewhat-working async IMAP backend

async
Manos Pitsidianakis 2020-06-27 21:40:46 +03:00
parent b72a1ca6d8
commit a38764f490
Signed by: Manos Pitsidianakis
GPG Key ID: 73627C2F690DF710
20 changed files with 6426 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
melib/Cargo.toml
View File

@ -43,6 +43,7 @@ rusqlite = {version = "0.20.0", optional = true }
libloading = "0.6.2"
futures = "0.3.5"
smol = "0.1.18"
[features]
default = ["unicode_algorithms", "imap_backend", "maildir_backend", "mbox_backend", "vcard", "sqlite3"]

25
melib/src/backends.rs
View File

@ -33,6 +33,8 @@ macro_rules! tag_hash {
#[cfg(feature = "imap_backend")]
pub mod imap;
#[cfg(feature = "imap_backend")]
pub mod imap_async;
//#[cfg(feature = "imap_backend")]
//pub mod imap2;
#[cfg(feature = "maildir_backend")]
@ -66,7 +68,9 @@ use std::fmt::Debug;
use std::ops::Deref;
use std::sync::{Arc, RwLock};
use core::pin::Pin;
pub use futures::stream::Stream;
use std::future::Future;
use std;
use std::collections::HashMap;
@ -146,6 +150,13 @@ impl Backends {
validate_conf_fn: Box::new(ImapType::validate_config),
},
);
b.register(
"imap_async".to_string(),
Backend {
create_fn: Box::new(|| Box::new(|f, i| imap_async::ImapType::new(f, i))),
validate_conf_fn: Box::new(imap_async::ImapType::validate_config),
},
);
}
#[cfg(feature = "notmuch_backend")]
{
@ -286,12 +297,17 @@ impl NotifyFn {
pub trait MailBackend: ::std::fmt::Debug + Send + Sync {
fn is_online(&self) -> Result<()>;
fn is_online_async(
&self,
) -> Result<Pin<Box<dyn Future<Output = Result<()>> + Send + 'static>>> {
Err(MeliError::new("Unimplemented."))
}
fn connect(&mut self) {}
fn get(&mut self, mailbox: &Mailbox) -> Async<Result<Vec<Envelope>>>;
fn get_async(
&mut self,
mailbox: &Mailbox,
) -> Result<Box<dyn Stream<Item = Result<Vec<Envelope>>>>> {
) -> Result<Pin<Box<dyn Future<Output = Result<Vec<Envelope>>> + Send + 'static>>> {
Err(MeliError::new("Unimplemented."))
}
fn refresh(
@ -301,6 +317,13 @@ pub trait MailBackend: ::std::fmt::Debug + Send + Sync {
) -> Result<Async<()>> {
Err(MeliError::new("Unimplemented."))
}
fn refresh_async(
&mut self,
_mailbox_hash: MailboxHash,
_sender: RefreshEventConsumer,
) -> Result<Pin<Box<dyn Future<Output = Result<()>> + Send + 'static>>> {
Err(MeliError::new("Unimplemented."))
}
fn watch(
&self,
sender: RefreshEventConsumer,

1815
melib/src/backends/imap_async.rs 100644
View File

File diff suppressed because it is too large Load Diff

174
melib/src/backends/imap_async/cache.rs 100644
View File

@ -0,0 +1,174 @@
/*
* meli - imap melib
*
* Copyright 2020 Manos Pitsidianakis
*
* This file is part of meli.
*
* meli is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* meli is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with meli. If not, see <http://www.gnu.org/licenses/>.
*/
use super::UID;
use crate::{
backends::{AccountHash, MailboxHash},
email::Envelope,
error::*,
};
pub type MaxUID = UID;
#[cfg(feature = "sqlite3")]
pub use sqlite3_m::*;
#[cfg(feature = "sqlite3")]
mod sqlite3_m {
use super::*;
use crate::sqlite3;
const DB_NAME: &'static str = "header_cache.db";
const INIT_SCRIPT: &'static str = "PRAGMA foreign_keys = true;
PRAGMA encoding = 'UTF-8';
CREATE TABLE IF NOT EXISTS envelopes (
mailbox_hash INTEGER,
uid INTEGER,
validity INTEGER,
envelope BLOB NOT NULL UNIQUE,
PRIMARY KEY (mailbox_hash, uid, validity),
FOREIGN KEY (mailbox_hash, validity) REFERENCES uidvalidity(mailbox_hash, uid) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS uidvalidity (
uid INTEGER UNIQUE,
mailbox_hash INTEGER UNIQUE,
PRIMARY KEY (mailbox_hash, uid)
);
CREATE INDEX IF NOT EXISTS envelope_idx ON envelopes(mailbox_hash, uid, validity);
CREATE INDEX IF NOT EXISTS uidvalidity_idx ON uidvalidity(mailbox_hash);";
pub fn get_envelopes(
account_hash: AccountHash,
mailbox_hash: MailboxHash,
uidvalidity: usize,
) -> Result<(MaxUID, Vec<(UID, Envelope)>)> {
let conn = sqlite3::open_or_create_db(
&format!("{}_{}", account_hash, DB_NAME),
Some(INIT_SCRIPT),
)?;
let mut stmt = conn
.prepare("SELECT MAX(uid) FROM envelopes WHERE mailbox_hash = ? AND validity = ?")
.unwrap();
let max_uid: usize = stmt
.query_map(
sqlite3::params![mailbox_hash as i64, uidvalidity as i64],
|row| row.get(0).map(|u: i64| u as usize),
)
.chain_err_summary(|| {
format!(
"Error while performing query {:?}",
"SELECT MAX(uid) FROM envelopes WHERE mailbox_hash = ? AND validity = ?"
)
})?
.next()
.unwrap()
.unwrap_or(0);
let mut stmt = conn
.prepare("SELECT uid, envelope FROM envelopes WHERE mailbox_hash = ? AND validity = ?")
.unwrap();
let results: Vec<(UID, Vec<u8>)> = stmt
.query_map(
sqlite3::params![mailbox_hash as i64, uidvalidity as i64],
|row| Ok((row.get::<_, i64>(0)? as usize, row.get(1)?)),
)
.chain_err_summary(|| {
format!(
"Error while performing query {:?}",
"SELECT uid, envelope FROM envelopes WHERE mailbox_hash = ? AND validity = ?",
)
})?
.collect::<std::result::Result<_, _>>()?;
debug!(
"imap cache max_uid: {} results len: {}",
max_uid,
results.len()
);
Ok((
max_uid,
results
.into_iter()
.map(|(uid, env)| {
Ok((
uid,
bincode::deserialize(&env).map_err(|e| MeliError::new(e.to_string()))?,
))
})
.collect::<Result<Vec<(UID, Envelope)>>>()?,
))
}
pub fn save_envelopes(
account_hash: AccountHash,
mailbox_hash: MailboxHash,
uidvalidity: usize,
envs: &[(UID, &Envelope)],
) -> Result<()> {
let conn =
sqlite3::open_or_create_db(&format!("{}_{}", account_hash, DB_NAME), Some(INIT_SCRIPT))
.chain_err_summary(|| {
format!(
"Could not create header_cache.db for account {}",
account_hash
)
})?;
conn.execute(
"INSERT OR REPLACE INTO uidvalidity (uid, mailbox_hash) VALUES (?1, ?2)",
sqlite3::params![uidvalidity as i64, mailbox_hash as i64],
)
.chain_err_summary(|| {
format!(
"Could not insert uidvalidity {} in header_cache of account {}",
uidvalidity, account_hash
)
})?;
for (uid, env) in envs {
conn.execute(
"INSERT OR REPLACE INTO envelopes (uid, mailbox_hash, validity, envelope) VALUES (?1, ?2, ?3, ?4)",
sqlite3::params![*uid as i64, mailbox_hash as i64, uidvalidity as i64, bincode::serialize(env).map_err(|e| MeliError::new(e.to_string()))?],
).chain_err_summary(|| format!("Could not insert envelope with hash {} in header_cache of account {}", env.hash(), account_hash))?;
}
Ok(())
}
}
#[cfg(not(feature = "sqlite3"))]
pub use filesystem_m::*;
#[cfg(not(feature = "sqlite3"))]
mod filesystem_m {
use super::*;
pub fn get_envelopes(
_account_hash: AccountHash,
_mailbox_hash: MailboxHash,
_uidvalidity: usize,
) -> Result<(MaxUID, Vec<(UID, Envelope)>)> {
Ok((0, vec![]))
}
pub fn save_envelopes(
_account_hash: AccountHash,
_mailbox_hash: MailboxHash,
_uidvalidity: usize,
_envs: &[(UID, &Envelope)],
) -> Result<()> {
Ok(())
}
}

809
melib/src/backends/imap_async/connection.rs 100644
View File

@ -0,0 +1,809 @@
/*
* meli - imap module.
*
* Copyright 2017 - 2019 Manos Pitsidianakis
*
* This file is part of meli.
*
* meli is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* meli is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with meli. If not, see <http://www.gnu.org/licenses/>.
*/
use super::protocol_parser::{ImapLineSplit, ImapResponse, RequiredResponses};
use crate::backends::MailboxHash;
use crate::connections::Connection;
use crate::email::parser::BytesExt;
use crate::error::*;
use std::io::Read;
use std::io::Write;
extern crate native_tls;
use futures::io::{AsyncReadExt, AsyncWriteExt};
use native_tls::TlsConnector;
pub use smol::Async as AsyncWrapper;
use std::collections::HashSet;
use std::iter::FromIterator;
use std::net::SocketAddr;
use std::sync::Arc;
use std::time::Instant;
use super::protocol_parser;
use super::{Capabilities, ImapServerConf, UIDStore};
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum ImapProtocol {
IMAP,
ManageSieve,
}
#[derive(Debug)]
pub struct ImapStream {
cmd_id: usize,
stream: AsyncWrapper<Connection>,
protocol: ImapProtocol,
}
#[derive(Debug)]
pub struct ImapConnection {
pub stream: Result<ImapStream>,
pub server_conf: ImapServerConf,
pub capabilities: Capabilities,
pub uid_store: Arc<UIDStore>,
pub current_mailbox: Option<MailboxHash>,
}
impl Drop for ImapStream {
fn drop(&mut self) {
//self.send_command(b"LOGOUT").ok().take();
}
}
impl ImapStream {
pub async fn new_connection(
server_conf: &ImapServerConf,
) -> Result<(Capabilities, ImapStream)> {
use std::io::prelude::*;
use std::net::TcpStream;
let path = &server_conf.server_hostname;
debug!("ImapStream::new_connection");
let cmd_id = 1;
let stream = if debug!(server_conf.use_tls) {
let mut connector = TlsConnector::builder();
if server_conf.danger_accept_invalid_certs {
connector.danger_accept_invalid_certs(true);
}
let connector = connector.build()?;
let addr = if let Ok(a) = lookup_ipv4(path, server_conf.server_port) {
a
} else {
return Err(MeliError::new(format!(
"Could not lookup address {}",
&path
)));
};
debug!("addr {:?}", addr);
let mut socket = AsyncWrapper::new(Connection::Tcp(debug!(
TcpStream::connect_timeout(&addr, std::time::Duration::new(4, 0),)
)?))?;
if debug!(server_conf.use_starttls) {
let mut buf = vec![0; 1024];
match server_conf.protocol {
ImapProtocol::IMAP => {
socket
.write_all(format!("M{} STARTTLS\r\n", cmd_id).as_bytes())
.await?
}
ImapProtocol::ManageSieve => {
let len = socket.read(&mut buf).await?;
debug!(unsafe { std::str::from_utf8_unchecked(&buf[0..len]) });
debug!(socket.write_all(b"STARTTLS\r\n").await?);
}
}
let mut response = String::with_capacity(1024);
let mut broken = false;
let now = std::time::Instant::now();
while now.elapsed().as_secs() < 3 {
let len = socket.read(&mut buf).await?;
response.push_str(unsafe { std::str::from_utf8_unchecked(&buf[0..len]) });
match server_conf.protocol {
ImapProtocol::IMAP => {
if response.starts_with("* OK ") && response.find("\r\n").is_some() {
if let Some(pos) = response.as_bytes().find(b"\r\n") {
response.drain(0..pos + 2);
}
}
}
ImapProtocol::ManageSieve => {
if response.starts_with("OK ") && response.find("\r\n").is_some() {
response.clear();
broken = true;
break;
}
}
}
if response.starts_with("M1 OK") {
broken = true;
break;
}
}
if !broken {
return Err(MeliError::new(format!(
"Could not initiate TLS negotiation to {}.",
path
)));
}
}
{
let socket = socket.into_inner()?;
let mut conn_result = debug!(connector.connect(path, socket));
if let Err(native_tls::HandshakeError::WouldBlock(midhandshake_stream)) =
conn_result
{
let mut midhandshake_stream = Some(midhandshake_stream);
loop {
match debug!(midhandshake_stream.take().unwrap().handshake()) {
Ok(r) => {
conn_result = Ok(r);
break;
}
Err(native_tls::HandshakeError::WouldBlock(stream)) => {
midhandshake_stream = Some(stream);
}
p => {
p?;
}
}
}
}
AsyncWrapper::new(Connection::Tls(conn_result?))?
}
} else {
let addr = if let Ok(a) = lookup_ipv4(path, server_conf.server_port) {
a
} else {
return Err(MeliError::new(format!(
"Could not lookup address {}",
&path
)));
};
AsyncWrapper::new(Connection::Tcp(TcpStream::connect_timeout(
&addr,
std::time::Duration::new(4, 0),
)?))?
};
let mut res = String::with_capacity(8 * 1024);
let mut ret = ImapStream {
cmd_id,
stream,
protocol: server_conf.protocol,
};
if let ImapProtocol::ManageSieve = server_conf.protocol {
use data_encoding::BASE64;
ret.read_response(&mut res).await?;
ret.send_command(
format!(
"AUTHENTICATE \"PLAIN\" \"{}\"",
BASE64.encode(
format!(
"\0{}\0{}",
&server_conf.server_username, &server_conf.server_password
)
.as_bytes()
)
)
.as_bytes(),
)
.await?;
ret.read_response(&mut res).await?;
return Ok((Default::default(), ret));
}
ret.send_command(b"CAPABILITY").await?;
ret.read_response(&mut res).await?;
let capabilities: std::result::Result<Vec<&[u8]>, _> = res
.split_rn()
.find(|l| l.starts_with("* CAPABILITY"))
.ok_or_else(|| MeliError::new(""))
.and_then(|res| {
protocol_parser::capabilities(res.as_bytes())
.map_err(|_| MeliError::new(""))
.map(|(_, v)| v)
});
if capabilities.is_err() {
return Err(MeliError::new(format!(
"Could not connect to {}: expected CAPABILITY response but got:{}",
&server_conf.server_hostname, res
)));
}
let capabilities = capabilities.unwrap();
if !capabilities
.iter()
.any(|cap| cap.eq_ignore_ascii_case(b"IMAP4rev1"))
{
return Err(MeliError::new(format!(
"Could not connect to {}: server is not IMAP4rev1 compliant",
&server_conf.server_hostname
)));
} else if capabilities
.iter()
.any(|cap| cap.eq_ignore_ascii_case(b"LOGINDISABLED"))
{
return Err(MeliError::new(format!(
"Could not connect to {}: server does not accept logins [LOGINDISABLED]",
&server_conf.server_hostname
)));
}
let mut capabilities = None;
ret.send_command(
format!(
"LOGIN \"{}\" \"{}\"",
&server_conf.server_username, &server_conf.server_password
)
.as_bytes(),
)
.await?;
let tag_start = format!("M{} ", (ret.cmd_id - 1));
loop {
ret.read_lines(&mut res, &String::new(), false).await?;
let mut should_break = false;
for l in res.split_rn() {
if l.starts_with("* CAPABILITY") {
capabilities = protocol_parser::capabilities(l.as_bytes())
.map(|(_, capabilities)| {
HashSet::from_iter(capabilities.into_iter().map(|s: &[u8]| s.to_vec()))
})
.ok();
}
if l.starts_with(tag_start.as_str()) {
if !l[tag_start.len()..].trim().starts_with("OK ") {
return Err(MeliError::new(format!(
"Could not connect. Server replied with '{}'",
l[tag_start.len()..].trim()
)));
}
should_break = true;
}
}
if should_break {
break;
}
}
if capabilities.is_none() {
/* sending CAPABILITY after LOGIN automatically is an RFC recommendation, so check
* for lazy servers */
drop(capabilities);
ret.send_command(b"CAPABILITY").await?;
ret.read_response(&mut res).await.unwrap();
let capabilities = protocol_parser::capabilities(res.as_bytes())?.1;
let capabilities = HashSet::from_iter(capabilities.into_iter().map(|s| s.to_vec()));
Ok((capabilities, ret))
} else {
let capabilities = capabilities.unwrap();
Ok((capabilities, ret))
}
}
pub async fn read_response(&mut self, ret: &mut String) -> Result<()> {
let id = match self.protocol {
ImapProtocol::IMAP => format!("M{} ", self.cmd_id - 1),
ImapProtocol::ManageSieve => String::new(),
};
self.read_lines(ret, &id, true).await?;
Ok(())
}
pub async fn read_lines(
&mut self,
ret: &mut String,
termination_string: &str,
keep_termination_string: bool,
) -> Result<()> {
let mut buf: [u8; 1024] = [0; 1024];
ret.clear();
let mut last_line_idx: usize = 0;
loop {
match self.stream.read(&mut buf).await {
Ok(0) => break,
Ok(b) => {
ret.push_str(unsafe { std::str::from_utf8_unchecked(&buf[0..b]) });
if let Some(mut pos) = ret[last_line_idx..].rfind("\r\n") {
if ret[last_line_idx..].starts_with("* BYE") {
return Err(MeliError::new("Disconnected"));
}
if let Some(prev_line) =
ret[last_line_idx..pos + last_line_idx].rfind("\r\n")
{
last_line_idx += prev_line + "\r\n".len();
pos -= prev_line + "\r\n".len();
}
if Some(pos + "\r\n".len()) == ret.get(last_line_idx..).map(|r| r.len()) {
if !termination_string.is_empty()
&& ret[last_line_idx..].starts_with(termination_string)
{
debug!(&ret[last_line_idx..]);
if !keep_termination_string {
ret.replace_range(last_line_idx.., "");
}
break;
} else if termination_string.is_empty() {
break;
}
}
last_line_idx += pos + "\r\n".len();
}
}
Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock => {
continue;
}
Err(e) => {
return Err(MeliError::from(e));
}
}
}
//debug!("returning IMAP response:\n{:?}", &ret);
Ok(())
}
pub async fn wait_for_continuation_request(&mut self) -> Result<()> {
let term = "+ ".to_string();
let mut ret = String::new();
self.read_lines(&mut ret, &term, false).await?;
Ok(())
}
pub async fn send_command(&mut self, command: &[u8]) -> Result<()> {
let command = command.trim();
match self.protocol {
ImapProtocol::IMAP => {
self.stream.write_all(b"M").await?;
self.stream
.write_all(self.cmd_id.to_string().as_bytes())
.await?;
self.stream.write_all(b" ").await?;
self.cmd_id += 1;
}
ImapProtocol::ManageSieve => {}
}
self.stream.write_all(command).await?;
self.stream.write_all(b"\r\n").await?;
match self.protocol {
ImapProtocol::IMAP => {
debug!("sent: M{} {}", self.cmd_id - 1, unsafe {
std::str::from_utf8_unchecked(command)
});
}
ImapProtocol::ManageSieve => {}
}
Ok(())
}
pub async fn send_literal(&mut self, data: &[u8]) -> Result<()> {
self.stream.write_all(data).await?;
self.stream.write_all(b"\r\n").await?;
Ok(())
}
pub async fn send_raw(&mut self, raw: &[u8]) -> Result<()> {
self.stream.write_all(raw).await?;
self.stream.write_all(b"\r\n").await?;
Ok(())
}
pub async fn set_nonblocking(&mut self, val: bool) -> Result<()> {
//self.stream.set_nonblocking(val).await?;
Ok(())
}
}
impl ImapConnection {
pub fn new_connection(
server_conf: &ImapServerConf,
uid_store: Arc<UIDStore>,
) -> ImapConnection {
ImapConnection {
stream: Err(MeliError::new("Offline".to_string())),
server_conf: server_conf.clone(),
capabilities: Capabilities::default(),
uid_store,
current_mailbox: None,
}
}
pub async fn connect(&mut self) -> Result<()> {
if let (instant, ref mut status @ Ok(())) = *self.uid_store.is_online.lock().unwrap() {
if Instant::now().duration_since(instant) >= std::time::Duration::new(60 * 30, 0) {
*status = Err(MeliError::new("Connection timed out"));
self.stream = Err(MeliError::new("Connection timed out"));
}
}
if self.stream.is_ok() {
self.uid_store.is_online.lock().unwrap().0 = Instant::now();
return Ok(());
}
let new_stream = debug!(ImapStream::new_connection(&self.server_conf).await);
if new_stream.is_err() {
*self.uid_store.is_online.lock().unwrap() = (
Instant::now(),
Err(new_stream.as_ref().unwrap_err().clone()),
);
} else {
*self.uid_store.is_online.lock().unwrap() = (Instant::now(), Ok(()));
}
let (capabilities, stream) = new_stream?;
self.stream = Ok(stream);
self.capabilities = capabilities;
Ok(())
}
pub async fn read_response(
&mut self,
ret: &mut String,
required_responses: RequiredResponses,
) -> Result<()> {
let mut response = String::new();
ret.clear();
self.stream.as_mut()?.read_response(&mut response).await?;
match self.server_conf.protocol {
ImapProtocol::IMAP => {
let r: ImapResponse = ImapResponse::from(&response);
match r {
ImapResponse::Bye(ref response_code) => {
self.stream = Err(MeliError::new(format!(
"Offline: received BYE: {:?}",
response_code
)));
ret.push_str(&response);
}
ImapResponse::No(ref response_code) => {
debug!("Received NO response: {:?} {:?}", response_code, response);
ret.push_str(&response);
}
ImapResponse::Bad(ref response_code) => {
debug!("Received BAD response: {:?} {:?}", response_code, response);
ret.push_str(&response);
}
_ => {
/*debug!(
"check every line for required_responses: {:#?}",
&required_responses
);*/
for l in response.split_rn() {
/*debug!("check line: {}", &l);*/
//if required_responses.check(l) || !self.process_untagged(l)? {
// ret.push_str(l);
//}
ret.push_str(l);
}
//ret.push_str(&response);
}
}
r.into()
}
ImapProtocol::ManageSieve => {
ret.push_str(&response);
Ok(())
}
}
}
pub async fn read_lines(&mut self, ret: &mut String, termination_string: String) -> Result<()> {
self.stream
.as_mut()?
.read_lines(ret, &termination_string, false)
.await?;
Ok(())
}
pub async fn wait_for_continuation_request(&mut self) -> Result<()> {
self.stream
.as_mut()?
.wait_for_continuation_request()
.await?;
Ok(())
}
pub async fn send_command(&mut self, command: &[u8]) -> Result<()> {
self.stream.as_mut()?.send_command(command).await?;
Ok(())
}
pub async fn send_literal(&mut self, data: &[u8]) -> Result<()> {
self.stream.as_mut()?.send_literal(data).await?;
Ok(())
}
pub async fn send_raw(&mut self, raw: &[u8]) -> Result<()> {
self.stream.as_mut()?.send_raw(raw).await?;
Ok(())
}
pub async fn set_nonblocking(&mut self, val: bool) -> Result<()> {
self.stream.as_mut()?.set_nonblocking(val).await?;
Ok(())
}
/*
pub fn try_send(
&mut self,
mut action: impl FnMut(&mut ImapStream) -> Result<()>,
) -> Result<()> {
if let (instant, ref mut status @ Ok(())) = *self.uid_store.is_online.lock().unwrap() {
if Instant::now().duration_since(instant) >= std::time::Duration::new(60 * 30, 0) {
*status = Err(MeliError::new("Connection timed out"));
self.stream = Err(MeliError::new("Connection timed out"));
}
}
if let Ok(ref mut stream) = self.stream {
if let Ok(_) = action(stream).await {
self.uid_store.is_online.lock().unwrap().0 = Instant::now();
return Ok(());
}
}
let new_stream = ImapStream::new_connection(&self.server_conf).await;
if new_stream.is_err() {
*self.uid_store.is_online.lock().unwrap() = (
Instant::now(),
Err(new_stream.as_ref().unwrap_err().clone()),
);
} else {
*self.uid_store.is_online.lock().unwrap() = (Instant::now(), Ok(()));
}
let (capabilities, stream) = new_stream?;
self.stream = Ok(stream);
self.capabilities = capabilities;
Err(MeliError::new("Connection timed out"))
}
*/
pub async fn select_mailbox(
&mut self,
mailbox_hash: MailboxHash,
ret: &mut String,
) -> Result<()> {
self.send_command(
format!(
"SELECT \"{}\"",
self.uid_store.mailboxes.read().unwrap()[&mailbox_hash].imap_path()
)
.as_bytes(),
)
.await?;
self.read_response(ret, RequiredResponses::SELECT_REQUIRED)
.await?;
debug!("select response {}", ret);
self.current_mailbox = Some(mailbox_hash);
Ok(())
}
pub async fn examine_mailbox(
&mut self,
mailbox_hash: MailboxHash,
ret: &mut String,
) -> Result<()> {
self.send_command(
format!(
"EXAMINE \"{}\"",
self.uid_store.mailboxes.read().unwrap()[&mailbox_hash].imap_path()
)
.as_bytes(),
)
.await?;
self.read_response(ret, RequiredResponses::EXAMINE_REQUIRED)
.await?;
debug!("examine response {}", ret);
self.current_mailbox = Some(mailbox_hash);
Ok(())
}
pub async fn unselect(&mut self) -> Result<()> {
if let Some(mailbox_hash) = self.current_mailbox.take() {
let mut response = String::with_capacity(8 * 1024);
if self
.capabilities
.iter()
.any(|cap| cap.eq_ignore_ascii_case(b"UNSELECT"))
{
self.send_command(b"UNSELECT").await?;
self.read_response(&mut response, RequiredResponses::empty())
.await?;
} else {
/* `RFC3691 - UNSELECT Command` states: "[..] IMAP4 provides this
* functionality (via a SELECT command with a nonexistent mailbox name or
* reselecting the same mailbox with EXAMINE command)[..]
*/
self.select_mailbox(mailbox_hash, &mut response).await?;
self.examine_mailbox(mailbox_hash, &mut response).await?;
}
}
Ok(())
}
pub fn add_refresh_event(&mut self, ev: crate::backends::RefreshEvent) {
if let Some(ref sender) = self.uid_store.sender.read().unwrap().as_ref() {
sender.send(ev);
for ev in self.uid_store.refresh_events.lock().unwrap().drain(..) {
sender.send(ev);
}
} else {
self.uid_store.refresh_events.lock().unwrap().push(ev);
}
}
pub async fn create_uid_msn_cache(
&mut self,
mailbox_hash: MailboxHash,
low: usize,
) -> Result<()> {
debug_assert!(low > 0);
let mut response = String::new();
if self
.current_mailbox
.map(|h| h != mailbox_hash)
.unwrap_or(true)
{
self.examine_mailbox(mailbox_hash, &mut response).await?;
}
self.send_command(format!("UID SEARCH {}:*", low).as_bytes())
.await?;
self.read_response(&mut response, RequiredResponses::SEARCH)
.await?;
debug!("uid search response {:?}", &response);
let mut msn_index_lck = self.uid_store.msn_index.lock().unwrap();
let msn_index = msn_index_lck.entry(mailbox_hash).or_default();
let _ = msn_index.drain(low - 1..);
msn_index.extend(
debug!(protocol_parser::search_results(response.as_bytes()))?
.1
.into_iter(),
);
Ok(())
}
}
pub struct ImapBlockingConnection {
buf: [u8; 1024],
result: Vec<u8>,
prev_res_length: usize,
pub conn: ImapConnection,
err: Option<String>,
}
impl From<ImapConnection> for ImapBlockingConnection {
fn from(mut conn: ImapConnection) -> Self {
unimplemented!()
/*
conn.set_nonblocking(false)
.expect("set_nonblocking call failed");
conn.stream
.as_mut()
.map(|s| {
s.stream
.set_write_timeout(Some(std::time::Duration::new(30, 0)))
.expect("set_write_timeout call failed")
})
.expect("set_write_timeout call failed");
conn.stream
.as_mut()
.map(|s| {
s.stream
.set_read_timeout(Some(std::time::Duration::new(30, 0)))
.expect("set_read_timeout call failed")
})
.expect("set_read_timeout call failed");
ImapBlockingConnection {
buf: [0; 1024],
conn,
prev_res_length: 0,
result: Vec::with_capacity(8 * 1024),
err: None,
}
*/
}
}
impl ImapBlockingConnection {
pub fn into_conn(self) -> ImapConnection {
self.conn
}
pub fn err(&self) -> Option<&str> {
self.err.as_ref().map(String::as_str)
}
}
impl Iterator for ImapBlockingConnection {
type Item = Vec<u8>;
fn next(&mut self) -> Option<Self::Item> {
unimplemented!()
/*
self.result.drain(0..self.prev_res_length);
self.prev_res_length = 0;
let mut prev_failure = None;
let ImapBlockingConnection {
ref mut prev_res_length,
ref mut result,
ref mut conn,
ref mut buf,
ref mut err,
} = self;
loop {
if conn.stream.is_err() {
debug!(&conn.stream);
return None;
}
match conn.stream.as_mut().unwrap().stream.read(buf) {
Ok(0) => return None,
Ok(b) => {
result.extend_from_slice(&buf[0..b]);
debug!(unsafe { std::str::from_utf8_unchecked(result) });
if let Some(pos) = result.find(b"\r\n") {
*prev_res_length = pos + b"\r\n".len();
return Some(result[0..*prev_res_length].to_vec());
}
prev_failure = None;
}
Err(e)
if e.kind() == std::io::ErrorKind::WouldBlock
|| e.kind() == std::io::ErrorKind::Interrupted =>
{
debug!(&e);
if let Some(prev_failure) = prev_failure.as_ref() {
if Instant::now().duration_since(*prev_failure)
>= std::time::Duration::new(60 * 5, 0)
{
*err = Some(e.to_string());
return None;
}
} else {
prev_failure = Some(Instant::now());
}
continue;
}
Err(e) => {
debug!(&conn.stream);
debug!(&e);
*err = Some(e.to_string());
return None;
}
}
}
*/
}
}
fn lookup_ipv4(host: &str, port: u16) -> Result<SocketAddr> {
use std::net::ToSocketAddrs;
let addrs = (host, port).to_socket_addrs()?;
for addr in addrs {
if let SocketAddr::V4(_) = addr {
return Ok(addr);
}
}
Err(MeliError::new("Cannot lookup address"))
}

172
melib/src/backends/imap_async/mailbox.rs 100644
View File

@ -0,0 +1,172 @@
/*
* meli - imap module.
*
* Copyright 2019 Manos Pitsidianakis
*
* This file is part of meli.
*
* meli is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* meli is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with meli. If not, see <http://www.gnu.org/licenses/>.
*/
use crate::backends::{
BackendMailbox, Mailbox, MailboxHash, MailboxPermissions, SpecialUsageMailbox,
};
use crate::email::EnvelopeHash;
use crate::error::*;
use std::collections::BTreeSet;
use std::sync::{Arc, Mutex, RwLock};
#[derive(Debug, Default, Clone)]
pub struct LazyCountSet {
not_yet_seen: usize,
set: BTreeSet<EnvelopeHash>,
}
impl LazyCountSet {
pub fn set_not_yet_seen(&mut self, new_val: usize) {
self.not_yet_seen = new_val;
}
pub fn insert_existing(&mut self, new_val: EnvelopeHash) -> bool {
if self.not_yet_seen == 0 {
false
} else {
self.not_yet_seen -= 1;
self.set.insert(new_val);
true
}
}
pub fn insert_existing_set(&mut self, set: BTreeSet<EnvelopeHash>) -> bool {
debug!("insert_existing_set {:?}", &set);
if self.not_yet_seen < set.len() {
false
} else {
self.not_yet_seen -= set.len();
self.set.extend(set.into_iter());
true
}
}
#[inline(always)]
pub fn len(&self) -> usize {
self.set.len() + self.not_yet_seen
}
#[inline(always)]
pub fn clear(&mut self) {
self.set.clear();
self.not_yet_seen = 0;
}
pub fn insert_new(&mut self, new_val: EnvelopeHash) {
self.set.insert(new_val);
}
pub fn insert_set(&mut self, set: BTreeSet<EnvelopeHash>) {
debug!("insert__set {:?}", &set);
self.set.extend(set.into_iter());
}
pub fn remove(&mut self, new_val: EnvelopeHash) -> bool {
self.set.remove(&new_val)
}
}
#[test]
fn test_lazy_count_set() {
let mut new = LazyCountSet::default();
new.set_not_yet_seen(10);
for i in 0..10 {
assert!(new.insert_existing(i));
}
assert!(!new.insert_existing(10));
}
#[derive(Debug, Default, Clone)]
pub struct ImapMailbox {
pub(super) hash: MailboxHash,
pub(super) imap_path: String,
pub(super) path: String,
pub(super) name: String,
pub(super) parent: Option<MailboxHash>,
pub(super) children: Vec<MailboxHash>,
pub separator: u8,
pub usage: Arc<RwLock<SpecialUsageMailbox>>,
pub no_select: bool,
pub is_subscribed: bool,
pub permissions: Arc<Mutex<MailboxPermissions>>,
pub exists: Arc<Mutex<LazyCountSet>>,
pub unseen: Arc<Mutex<LazyCountSet>>,
}
impl ImapMailbox {
pub fn imap_path(&self) -> &str {
&self.imap_path
}
}
impl BackendMailbox for ImapMailbox {
fn hash(&self) -> MailboxHash {
self.hash
}
fn name(&self) -> &str {
&self.name
}
fn path(&self) -> &str {
&self.path
}
fn change_name(&mut self, s: &str) {
self.name = s.to_string();
}
fn children(&self) -> &[MailboxHash] {
&self.children
}
fn clone(&self) -> Mailbox {
Box::new(std::clone::Clone::clone(self))
}
fn special_usage(&self) -> SpecialUsageMailbox {
*self.usage.read().unwrap()
}
fn parent(&self) -> Option<MailboxHash> {
self.parent
}
fn permissions(&self) -> MailboxPermissions {
*self.permissions.lock().unwrap()
}
fn is_subscribed(&self) -> bool {
self.is_subscribed
}
fn set_is_subscribed(&mut self, new_val: bool) -> Result<()> {
self.is_subscribed = new_val;
Ok(())
}
fn set_special_usage(&mut self, new_val: SpecialUsageMailbox) -> Result<()> {
*self.usage.write()? = new_val;
Ok(())
}
fn count(&self) -> Result<(usize, usize)> {
Ok((self.unseen.lock()?.len(), self.exists.lock()?.len()))
}
}

139
melib/src/backends/imap_async/managesieve.rs 100644
View File

@ -0,0 +1,139 @@
/*
* meli - managesieve
*
* Copyright 2020 Manos Pitsidianakis
*
* This file is part of meli.
*
* meli is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* meli is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with meli. If not, see <http://www.gnu.org/licenses/>.
*/
use super::{ImapConnection, ImapProtocol, ImapServerConf, UIDStore};
use crate::conf::AccountSettings;
use crate::error::{MeliError, Result};
use crate::get_conf_val;
use nom::{
branch::alt, bytes::complete::tag, combinator::map, error::ErrorKind,
multi::separated_nonempty_list, sequence::separated_pair, IResult,
};
use std::str::FromStr;
use std::sync::{Arc, Mutex};
use std::time::Instant;
pub fn managesieve_capabilities(input: &[u8]) -> Result<Vec<(&[u8], &[u8])>> {
let (_, ret) = separated_nonempty_list(
tag(b"\r\n"),
alt((
separated_pair(quoted_raw, tag(b" "), quoted_raw),
map(quoted_raw, |q| (q, &b""[..])),
)),
)(input)?;
Ok(ret)
}
#[test]
fn test_managesieve_capabilities() {
assert_eq!(managesieve_capabilities(b"\"IMPLEMENTATION\" \"Dovecot Pigeonhole\"\r\n\"SIEVE\" \"fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date index ihave duplicate mime foreverypart extracttext\"\r\n\"NOTIFY\" \"mailto\"\r\n\"SASL\" \"PLAIN\"\r\n\"STARTTLS\"\r\n\"VERSION\" \"1.0\"\r\n").unwrap(), vec![
(&b"IMPLEMENTATION"[..],&b"Dovecot Pigeonhole"[..]),
(&b"SIEVE"[..],&b"fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date index ihave duplicate mime foreverypart extracttext"[..]),
(&b"NOTIFY"[..],&b"mailto"[..]),
(&b"SASL"[..],&b"PLAIN"[..]),
(&b"STARTTLS"[..], &b""[..]),
(&b"VERSION"[..],&b"1.0"[..])]
);
}
// Return a byte sequence surrounded by "s and decoded if necessary
pub fn quoted_raw(input: &[u8]) -> IResult<&[u8], &[u8]> {
if input.is_empty() || input[0] != b'"' {
return Err(nom::Err::Error((input, ErrorKind::Tag)));
}
let mut i = 1;
while i < input.len() {
if input[i] == b'\"' && input[i - 1] != b'\\' {
return Ok((&input[i + 1..], &input[1..i]));
}
i += 1;
}
Err(nom::Err::Error((input, ErrorKind::Tag)))
}
pub trait ManageSieve {
fn havespace(&mut self) -> Result<()>;
fn putscript(&mut self) -> Result<()>;
fn listscripts(&mut self) -> Result<()>;
fn setactive(&mut self) -> Result<()>;
fn getscript(&mut self) -> Result<()>;
fn deletescript(&mut self) -> Result<()>;
fn renamescript(&mut self) -> Result<()>;
}
pub fn new_managesieve_connection(s: &AccountSettings) -> Result<ImapConnection> {
let server_hostname = get_conf_val!(s["server_hostname"])?;
let server_username = get_conf_val!(s["server_username"])?;
let server_password = get_conf_val!(s["server_password"])?;
let server_port = get_conf_val!(s["server_port"], 4190)?;
let danger_accept_invalid_certs: bool = get_conf_val!(s["danger_accept_invalid_certs"], false)?;
let server_conf = ImapServerConf {
server_hostname: server_hostname.to_string(),
server_username: server_username.to_string(),
server_password: server_password.to_string(),
server_port,
use_starttls: true,
use_tls: true,
danger_accept_invalid_certs,
protocol: ImapProtocol::ManageSieve,
};
let uid_store = Arc::new(UIDStore {
is_online: Arc::new(Mutex::new((
Instant::now(),
Err(MeliError::new("Account is uninitialised.")),
))),
..Default::default()
});
Ok(ImapConnection::new_connection(&server_conf, uid_store))
}
impl ManageSieve for ImapConnection {
fn havespace(&mut self) -> Result<()> {
Ok(())
}
fn putscript(&mut self) -> Result<()> {
Ok(())
}
fn listscripts(&mut self) -> Result<()> {
Ok(())
}
fn setactive(&mut self) -> Result<()> {
Ok(())
}
fn getscript(&mut self) -> Result<()> {
Ok(())
}
fn deletescript(&mut self) -> Result<()> {
Ok(())
}
fn renamescript(&mut self) -> Result<()> {
Ok(())
}
}

240
melib/src/backends/imap_async/operations.rs 100644
View File

@ -0,0 +1,240 @@
/*
* meli - imap module.
*
* Copyright 2017 - 2019 Manos Pitsidianakis
*
* This file is part of meli.
*
* meli is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* meli is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with meli. If not, see <http://www.gnu.org/licenses/>.
*/
use super::*;
use crate::backends::BackendOp;
use crate::email::*;
use crate::error::{MeliError, Result};
use std::cell::Cell;
use std::sync::{Arc, Mutex};
/// `BackendOp` implementor for Imap
#[derive(Debug, Clone)]
pub struct ImapOp {
uid: usize,
bytes: Option<String>,
headers: Option<String>,
body: Option<String>,
mailbox_path: String,
mailbox_hash: MailboxHash,
flags: Cell<Option<Flag>>,
connection: Arc<Mutex<ImapConnection>>,
uid_store: Arc<UIDStore>,
}
impl ImapOp {
pub fn new(
uid: usize,
mailbox_path: String,
mailbox_hash: MailboxHash,
connection: Arc<Mutex<ImapConnection>>,
uid_store: Arc<UIDStore>,
) -> Self {
ImapOp {
uid,
connection,
bytes: None,
headers: None,
body: None,
mailbox_path,
mailbox_hash,
flags: Cell::new(None),
uid_store,
}
}
}
impl BackendOp for ImapOp {
fn description(&self) -> String {
format!("Message in mailbox: {}", &self.mailbox_path)
}
fn as_bytes(&mut self) -> Result<&[u8]> {
if self.bytes.is_none() {
let mut bytes_cache = self.uid_store.byte_cache.lock()?;
let cache = bytes_cache.entry(self.uid).or_default();
if cache.bytes.is_some() {
self.bytes = cache.bytes.clone();
} else {
drop(cache);
drop(bytes_cache);
let mut response = String::with_capacity(8 * 1024);
{
let mut conn =
try_lock(&self.connection, Some(std::time::Duration::new(2, 0)))?;
conn.examine_mailbox(self.mailbox_hash, &mut response)?;
conn.send_command(format!("UID FETCH {} (FLAGS RFC822)", self.uid).as_bytes())?;
conn.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)?;
}
debug!(
"fetch response is {} bytes and {} lines",
response.len(),
response.lines().collect::<Vec<&str>>().len()
);
let UidFetchResponse {
uid, flags, body, ..
} = protocol_parser::uid_fetch_response(&response)?.1;
assert_eq!(uid, self.uid);
assert!(body.is_some());
let mut bytes_cache = self.uid_store.byte_cache.lock()?;
let cache = bytes_cache.entry(self.uid).or_default();
if let Some((flags, _)) = flags {
self.flags.set(Some(flags));
cache.flags = Some(flags);
}
cache.bytes =
Some(unsafe { std::str::from_utf8_unchecked(body.unwrap()).to_string() });
self.bytes = cache.bytes.clone();
}
}
Ok(self.bytes.as_ref().unwrap().as_bytes())
}
fn fetch_flags(&self) -> Flag {
macro_rules! or_return_default {
($expr:expr) => {
match $expr {
Ok(ok) => ok,
Err(_) => return Default::default(),
}
};
}
if self.flags.get().is_some() {
return self.flags.get().unwrap();
}
let mut bytes_cache = or_return_default!(self.uid_store.byte_cache.lock());
let cache = bytes_cache.entry(self.uid).or_default();
if cache.flags.is_some() {
self.flags.set(cache.flags);
} else {
let mut response = String::with_capacity(8 * 1024);
let mut conn = or_return_default!(try_lock(
&self.connection,
Some(std::time::Duration::new(2, 0))
));
or_return_default!(conn.examine_mailbox(self.mailbox_hash, &mut response));
or_return_default!(
conn.send_command(format!("UID FETCH {} FLAGS", self.uid).as_bytes())
);
or_return_default!(conn.read_response(&mut response, RequiredResponses::FETCH_REQUIRED));
debug!(
"fetch response is {} bytes and {} lines",
response.len(),
response.lines().collect::<Vec<&str>>().len()
);
match protocol_parser::uid_fetch_flags_response(response.as_bytes())
.map(|(_, v)| v)
.map_err(MeliError::from)
{
Ok(v) => {
if v.len() != 1 {
debug!("responses len is {}", v.len());
debug!(response);
/* TODO: Trigger cache invalidation here. */
debug!(format!("message with UID {} was not found", self.uid));
return Flag::default();
}
let (uid, (flags, _)) = v[0];
assert_eq!(uid, self.uid);
cache.flags = Some(flags);
self.flags.set(Some(flags));
}
Err(e) => or_return_default!(Err(e)),
}
}
self.flags.get().unwrap()
}
fn set_flag(&mut self, _envelope: &mut Envelope, f: Flag, value: bool) -> Result<()> {
let mut flags = self.fetch_flags();
flags.set(f, value);
let mut response = String::with_capacity(8 * 1024);
let mut conn = try_lock(&self.connection, Some(std::time::Duration::new(2, 0)))?;
conn.select_mailbox(self.mailbox_hash, &mut response)?;
debug!(&response);
conn.send_command(
format!(
"UID STORE {} FLAGS.SILENT ({})",
self.uid,
flags_to_imap_list!(flags)
)
.as_bytes(),
)?;
conn.read_response(&mut response, RequiredResponses::STORE_REQUIRED)?;
debug!(&response);
match protocol_parser::uid_fetch_flags_response(response.as_bytes())
.map(|(_, v)| v)
.map_err(MeliError::from)
{
Ok(v) => {
if v.len() == 1 {
debug!("responses len is {}", v.len());
let (uid, (flags, _)) = v[0];
assert_eq!(uid, self.uid);
self.flags.set(Some(flags));
}
}
Err(e) => Err(e)?,
}
let mut bytes_cache = self.uid_store.byte_cache.lock()?;
let cache = bytes_cache.entry(self.uid).or_default();
cache.flags = Some(flags);
Ok(())
}
fn set_tag(&mut self, envelope: &mut Envelope, tag: String, value: bool) -> Result<()> {
let mut response = String::with_capacity(8 * 1024);
let mut conn = try_lock(&self.connection, Some(std::time::Duration::new(2, 0)))?;
conn.select_mailbox(self.mailbox_hash, &mut response)?;
conn.send_command(
format!(
"UID STORE {} {}FLAGS.SILENT ({})",
self.uid,
if value { "+" } else { "-" },
&tag
)
.as_bytes(),
)?;
conn.read_response(&mut response, RequiredResponses::STORE_REQUIRED)?;
protocol_parser::uid_fetch_flags_response(response.as_bytes())
.map(|(_, v)| v)
.map_err(MeliError::from)?;
let hash = tag_hash!(tag);
if value {
self.uid_store.tag_index.write().unwrap().insert(hash, tag);
} else {
self.uid_store.tag_index.write().unwrap().remove(&hash);
}
if !envelope.labels().iter().any(|&h_| h_ == hash) {
if value {
envelope.labels_mut().push(hash);
}
}
if !value {
if let Some(pos) = envelope.labels().iter().position(|&h_| h_ == hash) {
envelope.labels_mut().remove(pos);
}
}
Ok(())
}
}

1522
melib/src/backends/imap_async/protocol_parser.rs 100644
View File

File diff suppressed because it is too large Load Diff

334
melib/src/backends/imap_async/untagged.rs 100644
View File

@ -0,0 +1,334 @@
/*
* meli - imap
*
* Copyright 2020 Manos Pitsidianakis
*
* This file is part of meli.
*
* meli is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* meli is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with meli. If not, see <http://www.gnu.org/licenses/>.
*/
use super::ImapConnection;
use crate::backends::imap_async::protocol_parser::{
ImapLineSplit, RequiredResponses, UidFetchResponse, UntaggedResponse,
};
use crate::backends::BackendMailbox;
use crate::backends::{
RefreshEvent,
RefreshEventKind::{self, *},
};
use crate::email::Envelope;
use crate::error::*;
use std::time::Instant;
impl ImapConnection {
pub fn process_untagged(&mut self, line: &str) -> Result<bool> {
macro_rules! try_fail {
($mailbox_hash: expr, $($result:expr)+) => {
$(if let Err(err) = $result {
*self.uid_store.is_online.lock().unwrap() = (
Instant::now(),
Err(err.clone()),
);
debug!("failure: {}", err.to_string());
self.add_refresh_event(RefreshEvent {
account_hash: self.uid_store.account_hash,
mailbox_hash: $mailbox_hash,
kind: RefreshEventKind::Failure(err.clone()),
});
Err(err)
} else { Ok(()) }?;)+
};
}
//FIXME
let mailbox_hash = if let Some(mailbox_hash) = self.current_mailbox {
mailbox_hash
} else {
return Ok(false);
};
let mailbox =
std::clone::Clone::clone(&self.uid_store.mailboxes.read().unwrap()[&mailbox_hash]);
let mut response = String::with_capacity(8 * 1024);
let untagged_response =
match super::protocol_parser::untagged_responses(line.as_bytes()).map(|(_, v)| v) {
Ok(None) | Err(_) => {
return Ok(false);
}
Ok(Some(r)) => r,
};
match untagged_response {
UntaggedResponse::Bye { reason } => {
*self.uid_store.is_online.lock().unwrap() =
(std::time::Instant::now(), Err(reason.into()));
}
UntaggedResponse::Expunge(n) => {
let deleted_uid = self
.uid_store
.msn_index
.lock()
.unwrap()
.entry(mailbox_hash)
.or_default()
.remove(n);
debug!("expunge {}, UID = {}", n, deleted_uid);
let deleted_hash: crate::email::EnvelopeHash = self
.uid_store
.uid_index
.lock()
.unwrap()
.remove(&(mailbox_hash, deleted_uid))
.unwrap();
self.uid_store
.hash_index
.lock()
.unwrap()
.remove(&deleted_hash);
self.add_refresh_event(RefreshEvent {
account_hash: self.uid_store.account_hash,
mailbox_hash,
kind: Remove(deleted_hash),
});
}
UntaggedResponse::Exists(n) => {
/* UID FETCH ALL UID, cross-ref, then FETCH difference headers
* */
let mut prev_exists = mailbox.exists.lock().unwrap();
debug!("exists {}", n);
if n > prev_exists.len() {
try_fail!(
mailbox_hash,
self.send_command(
&[
b"FETCH",
format!("{}:{}", prev_exists.len() + 1, n).as_bytes(),
b"(UID FLAGS RFC822)",
]
.join(&b' '),
)
self.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
);
match super::protocol_parser::uid_fetch_responses(&response) {
Ok((_, v, _)) => {
'fetch_responses: for UidFetchResponse {
uid, flags, body, ..
} in v
{
if self
.uid_store
.uid_index
.lock()
.unwrap()
.contains_key(&(mailbox_hash, uid))
{
continue 'fetch_responses;
}
if let Ok(mut env) = Envelope::from_bytes(
body.unwrap(),
flags.as_ref().map(|&(f, _)| f),
) {
self.uid_store
.hash_index
.lock()
.unwrap()
.insert(env.hash(), (uid, mailbox_hash));
self.uid_store
.uid_index
.lock()
.unwrap()
.insert((mailbox_hash, uid), env.hash());
if let Some((_, keywords)) = flags {
let mut tag_lck = self.uid_store.tag_index.write().unwrap();
for f in keywords {
let hash = tag_hash!(f);
if !tag_lck.contains_key(&hash) {
tag_lck.insert(hash, f);
}
env.labels_mut().push(hash);
}
}
debug!(
"Create event {} {} {}",
env.hash(),
env.subject(),
mailbox.path(),
);
if !env.is_seen() {
mailbox.unseen.lock().unwrap().insert_new(env.hash());
}
prev_exists.insert_new(env.hash());
self.add_refresh_event(RefreshEvent {
account_hash: self.uid_store.account_hash,
mailbox_hash,
kind: Create(Box::new(env)),
});
}
}
}
Err(e) => {
debug!(e);
}
}
}
}
UntaggedResponse::Recent(_) => {
try_fail!(
mailbox_hash,
self.send_command(b"UID SEARCH RECENT")
self.read_response(&mut response, RequiredResponses::SEARCH)
);
match super::protocol_parser::search_results_raw(response.as_bytes())
.map(|(_, v)| v)
.map_err(MeliError::from)
{
Ok(&[]) => {
debug!("UID SEARCH RECENT returned no results");
}
Ok(v) => {
try_fail!(
mailbox_hash,
self.send_command(
&[b"UID FETCH", v, b"(FLAGS RFC822)"]
.join(&b' '),
)
self.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
);
debug!(&response);
match super::protocol_parser::uid_fetch_responses(&response) {
Ok((_, v, _)) => {
for UidFetchResponse {
uid, flags, body, ..
} in v
{
if !self
.uid_store
.uid_index
.lock()
.unwrap()
.contains_key(&(mailbox_hash, uid))
{
if let Ok(mut env) = Envelope::from_bytes(
body.unwrap(),
flags.as_ref().map(|&(f, _)| f),
) {
self.uid_store
.hash_index
.lock()
.unwrap()
.insert(env.hash(), (uid, mailbox_hash));
self.uid_store
.uid_index
.lock()
.unwrap()
.insert((mailbox_hash, uid), env.hash());
debug!(
"Create event {} {} {}",
env.hash(),
env.subject(),
mailbox.path(),
);
if let Some((_, keywords)) = flags {
let mut tag_lck =
self.uid_store.tag_index.write().unwrap();
for f in keywords {
let hash = tag_hash!(f);
if !tag_lck.contains_key(&hash) {
tag_lck.insert(hash, f);
}
env.labels_mut().push(hash);
}
}
if !env.is_seen() {
mailbox
.unseen
.lock()
.unwrap()
.insert_new(env.hash());
}
mailbox.exists.lock().unwrap().insert_new(env.hash());
self.add_refresh_event(RefreshEvent {
account_hash: self.uid_store.account_hash,
mailbox_hash,
kind: Create(Box::new(env)),
});
}
}
}
}
Err(e) => {
debug!(e);
}
}
}
Err(e) => {
debug!(
"UID SEARCH RECENT err: {}\nresp: {}",
e.to_string(),
&response
);
}
}
}
UntaggedResponse::Fetch(msg_seq, flags) => {
/* a * {msg_seq} FETCH (FLAGS ({flags})) was received, so find out UID from msg_seq
* and send update
*/
debug!("fetch {} {:?}", msg_seq, flags);
try_fail!(
mailbox_hash,
self.send_command(
&[
b"UID SEARCH ",
format!("{}", msg_seq).as_bytes(),
]
.join(&b' '),
)
self.read_response(&mut response, RequiredResponses::SEARCH)
);
debug!(&response);
match super::protocol_parser::search_results(
response.split_rn().next().unwrap_or("").as_bytes(),
)
.map(|(_, v)| v)
{
Ok(mut v) => {
if let Some(uid) = v.pop() {
let lck = self.uid_store.uid_index.lock().unwrap();
let env_hash = lck.get(&(mailbox_hash, uid)).map(|&h| h);
drop(lck);
if let Some(env_hash) = env_hash {
if !flags.0.intersects(crate::email::Flag::SEEN) {
mailbox.unseen.lock().unwrap().insert_new(env_hash);
} else {
mailbox.unseen.lock().unwrap().remove(env_hash);
}
self.add_refresh_event(RefreshEvent {
account_hash: self.uid_store.account_hash,
mailbox_hash,
kind: NewFlags(env_hash, flags),
});
};
}
}
Err(e) => {
debug!(&response);
debug!(e);
}
}
}
}
Ok(true)
}
}

908
melib/src/backends/imap_async/watch.rs 100644
View File

@ -0,0 +1,908 @@
/*
* meli - imap module.
*
* Copyright 2019 Manos Pitsidianakis
*
* This file is part of meli.
*
* meli is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* meli is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with meli. If not, see <http://www.gnu.org/licenses/>.
*/
use super::*;
use crate::backends::SpecialUsageMailbox;
use crate::email::parser::BytesExt;
use crate::email::parser::BytesIterExt;
use std::sync::{Arc, Mutex};
/// Arguments for IMAP watching functions
pub struct ImapWatchKit {
pub conn: ImapConnection,
pub main_conn: Arc<Mutex<ImapConnection>>,
pub uid_store: Arc<UIDStore>,
pub work_context: WorkContext,
}
macro_rules! exit_on_error {
($conn:expr, $mailbox_hash:ident, $work_context:ident, $thread_id:ident, $($result:expr)+) => {
$(if let Err(e) = $result {
*$conn.uid_store.is_online.lock().unwrap() = (
Instant::now(),
Err(e.clone()),
);
debug!("failure: {}", e.to_string());
$work_context.set_status.send(($thread_id, e.to_string())).unwrap();
$work_context.finished.send($thread_id).unwrap();
let account_hash = $conn.uid_store.account_hash;
$conn.add_refresh_event(RefreshEvent {
account_hash,
mailbox_hash: $mailbox_hash,
kind: RefreshEventKind::Failure(e.clone()),
});
Err(e)
} else { Ok(()) }?;)+
};
}
pub fn poll_with_examine(kit: ImapWatchKit) -> Result<()> {
debug!("poll with examine");
let ImapWatchKit {
mut conn,
main_conn,
uid_store,
work_context,
} = kit;
loop {
if super::try_lock(&uid_store.is_online, Some(std::time::Duration::new(10, 0)))?
.1
.is_ok()
{
break;
}
std::thread::sleep(std::time::Duration::from_millis(100));
}
conn.connect()?;
let mut response = String::with_capacity(8 * 1024);
let thread_id: std::thread::ThreadId = std::thread::current().id();
loop {
work_context
.set_status
.send((thread_id, "sleeping...".to_string()))
.unwrap();
std::thread::sleep(std::time::Duration::from_millis(5 * 60 * 1000));
let mailboxes = uid_store.mailboxes.read()?;
for mailbox in mailboxes.values() {
work_context
.set_status
.send((
thread_id,
format!("examining `{}` for updates...", mailbox.path()),
))
.unwrap();
examine_updates(mailbox, &mut conn, &uid_store, &work_context)?;
}
let mut main_conn = super::try_lock(&main_conn, Some(std::time::Duration::new(10, 0)))?;
main_conn.send_command(b"NOOP")?;
main_conn.read_response(&mut response, RequiredResponses::empty())?;
}
}
pub fn idle(kit: ImapWatchKit) -> Result<()> {
debug!("IDLE");
/* IDLE only watches the connection's selected mailbox. We will IDLE on INBOX and every ~5
* minutes wake up and poll the others */
let ImapWatchKit {
mut conn,
main_conn,
uid_store,
work_context,
} = kit;
loop {
if super::try_lock(&uid_store.is_online, Some(std::time::Duration::new(10, 0)))?
.1
.is_ok()
{
break;
}
std::thread::sleep(std::time::Duration::from_millis(100));
}
conn.connect()?;
let thread_id: std::thread::ThreadId = std::thread::current().id();
let mailbox: ImapMailbox = match uid_store
.mailboxes
.read()
.unwrap()
.values()
.find(|f| f.parent.is_none() && (f.special_usage() == SpecialUsageMailbox::Inbox))
.map(std::clone::Clone::clone)
{
Some(mailbox) => mailbox,
None => {
let err = MeliError::new("INBOX mailbox not found in local mailbox index. meli may have not parsed the IMAP mailboxes correctly");
debug!("failure: {}", err.to_string());
work_context
.set_status
.send((thread_id, err.to_string()))
.unwrap();
conn.add_refresh_event(RefreshEvent {
account_hash: uid_store.account_hash,
mailbox_hash: 0,
kind: RefreshEventKind::Failure(err.clone()),
});
return Err(err);
}
};
let mailbox_hash = mailbox.hash();
let uidvalidity;
let mut response = String::with_capacity(8 * 1024);
exit_on_error!(
conn,
mailbox_hash,
work_context,
thread_id,
conn.send_command(format!("SELECT \"{}\"", mailbox.imap_path()).as_bytes())
conn.read_response(&mut response, RequiredResponses::SELECT_REQUIRED)
);
debug!("select response {}", &response);
{
let mut prev_exists = mailbox.exists.lock().unwrap();
match protocol_parser::select_response(&response) {
Ok(ok) => {
{
uidvalidity = ok.uidvalidity;
let mut uidvalidities = uid_store.uidvalidity.lock().unwrap();
if let Some(v) = uidvalidities.get_mut(&mailbox_hash) {
if *v != ok.uidvalidity {
conn.add_refresh_event(RefreshEvent {
account_hash: uid_store.account_hash,
mailbox_hash,
kind: RefreshEventKind::Rescan,
});
prev_exists.clear();
/*
uid_store.uid_index.lock().unwrap().clear();
uid_store.hash_index.lock().unwrap().clear();
uid_store.byte_cache.lock().unwrap().clear();
*/
*v = ok.uidvalidity;
}
} else {
conn.add_refresh_event(RefreshEvent {
account_hash: uid_store.account_hash,
mailbox_hash,
kind: RefreshEventKind::Rescan,
});
conn.add_refresh_event(RefreshEvent {
account_hash: uid_store.account_hash,
mailbox_hash,
kind: RefreshEventKind::Failure(MeliError::new(format!(
"Unknown mailbox: {} {}",
mailbox.path(),
mailbox_hash
))),
});
}
}
debug!(&ok);
}
Err(e) => {
debug!("{:?}", e);
panic!("could not select mailbox");
}
};
}
exit_on_error!(
conn,
mailbox_hash,
work_context,
thread_id,
conn.send_command(b"IDLE")
);
work_context
.set_status
.send((thread_id, "IDLEing".to_string()))
.unwrap();
let mut iter = ImapBlockingConnection::from(conn);
let mut beat = std::time::Instant::now();
let mut watch = std::time::Instant::now();
/* duration interval to send heartbeat */
let _26_mins = std::time::Duration::from_secs(26 * 60);
/* duration interval to check other mailboxes for changes */
let _5_mins = std::time::Duration::from_secs(5 * 60);
while let Some(line) = iter.next() {
let now = std::time::Instant::now();
if now.duration_since(beat) >= _26_mins {
let mut main_conn_lck =
super::try_lock(&main_conn, Some(std::time::Duration::new(10, 0)))?;
exit_on_error!(
iter.conn,
mailbox_hash,
work_context,
thread_id,
iter.conn.set_nonblocking(true)
iter.conn.send_raw(b"DONE")
iter.conn.read_response(&mut response, RequiredResponses::empty())
iter.conn.send_command(b"IDLE")
iter.conn.set_nonblocking(false)
main_conn_lck.send_command(b"NOOP")
main_conn_lck.read_response(&mut response, RequiredResponses::empty())
);
beat = now;
}
if now.duration_since(watch) >= _5_mins {
/* Time to poll all inboxes */
let mut conn = try_lock(&main_conn, Some(std::time::Duration::new(10, 0)))?;
for mailbox in uid_store.mailboxes.read().unwrap().values() {
work_context
.set_status
.send((
thread_id,
format!("examining `{}` for updates...", mailbox.path()),
))
.unwrap();
exit_on_error!(
conn,
mailbox_hash,
work_context,
thread_id,
examine_updates(mailbox, &mut conn, &uid_store, &work_context,)
);
}
work_context
.set_status
.send((thread_id, "done examining mailboxes.".to_string()))
.unwrap();
watch = now;
}
*uid_store.is_online.lock().unwrap() = (Instant::now(), Ok(()));
match protocol_parser::untagged_responses(line.as_slice())
.map(|(_, v)| v)
.map_err(MeliError::from)
{
Ok(Some(Recent(r))) => {
let mut conn = super::try_lock(&main_conn, Some(std::time::Duration::new(10, 0)))?;
work_context
.set_status
.send((thread_id, format!("got `{} RECENT` notification", r)))
.unwrap();
/* UID SEARCH RECENT */
exit_on_error!(
conn,
mailbox_hash,
work_context,
thread_id,
conn.examine_mailbox(mailbox_hash, &mut response)
conn.send_command(b"UID SEARCH RECENT")
conn.read_response(&mut response, RequiredResponses::SEARCH)
);
match protocol_parser::search_results_raw(response.as_bytes())
.map(|(_, v)| v)
.map_err(MeliError::from)
{
Ok(&[]) => {
debug!("UID SEARCH RECENT returned no results");
}
Ok(v) => {
exit_on_error!(
conn,
mailbox_hash,
work_context,
thread_id,
conn.send_command(
&[&b"UID FETCH"[..], &v.trim().split(|b| b == &b' ').join(b','), &b"(FLAGS RFC822)"[..]]
.join(&b' '),
)
conn.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
);
debug!(&response);
match protocol_parser::uid_fetch_responses(&response) {
Ok((_, v, _)) => {
let len = v.len();
let mut ctr = 0;
for UidFetchResponse {
uid, flags, body, ..
} in v
{
work_context
.set_status
.send((
thread_id,
format!("parsing {}/{} envelopes..", ctr, len),
))
.unwrap();
ctr += 1;
if !uid_store
.uid_index
.lock()
.unwrap()
.contains_key(&(mailbox_hash, uid))
{
if let Ok(mut env) = Envelope::from_bytes(
body.unwrap(),
flags.as_ref().map(|&(f, _)| f),
) {
uid_store
.hash_index
.lock()
.unwrap()
.insert(env.hash(), (uid, mailbox_hash));
uid_store
.uid_index
.lock()
.unwrap()
.insert((mailbox_hash, uid), env.hash());
debug!(
"Create event {} {} {}",
env.hash(),
env.subject(),
mailbox.path(),
);
if let Some((_, keywords)) = flags {
let mut tag_lck =
uid_store.tag_index.write().unwrap();
for f in keywords {
let hash = tag_hash!(f);
if !tag_lck.contains_key(&hash) {
tag_lck.insert(hash, f);
}
env.labels_mut().push(hash);
}
}
if !env.is_seen() {
mailbox
.unseen
.lock()
.unwrap()
.insert_new(env.hash());
}
if uid_store.cache_headers {
cache::save_envelopes(
uid_store.account_hash,
mailbox_hash,
uidvalidity,
&[(uid, &env)],
)?;
}
mailbox.exists.lock().unwrap().insert_new(env.hash());
conn.add_refresh_event(RefreshEvent {
account_hash: uid_store.account_hash,
mailbox_hash,
kind: Create(Box::new(env)),
});
}
}
}
work_context
.set_status
.send((thread_id, format!("parsed {}/{} envelopes.", ctr, len)))
.unwrap();
}
Err(e) => {
debug!(e);
}
}
}
Err(e) => {
debug!(
"UID SEARCH RECENT err: {}\nresp: {}",
e.to_string(),
&response
);
}
}
}
Ok(Some(Expunge(n))) => {
// The EXPUNGE response reports that the specified message sequence
// number has been permanently removed from the mailbox. The message
// sequence number for each successive message in the mailbox is
// immediately decremented by 1, and this decrement is reflected in
// message sequence numbers in subsequent responses (including other
// untagged EXPUNGE responses).
let mut conn = super::try_lock(&main_conn, Some(std::time::Duration::new(10, 0)))?;
work_context
.set_status
.send((thread_id, format!("got `{} EXPUNGED` notification", n)))
.unwrap();
let deleted_uid = uid_store
.msn_index
.lock()
.unwrap()
.entry(mailbox_hash)
.or_default()
.remove(n);
debug!("expunge {}, UID = {}", n, deleted_uid);
let deleted_hash: EnvelopeHash = uid_store
.uid_index
.lock()
.unwrap()
.remove(&(mailbox_hash, deleted_uid))
.unwrap();
uid_store.hash_index.lock().unwrap().remove(&deleted_hash);
conn.add_refresh_event(RefreshEvent {
account_hash: uid_store.account_hash,
mailbox_hash,
kind: Remove(deleted_hash),
});
}
Ok(Some(Exists(n))) => {
let mut conn = super::try_lock(&main_conn, Some(std::time::Duration::new(10, 0)))?;
/* UID FETCH ALL UID, cross-ref, then FETCH difference headers
* */
let mut prev_exists = mailbox.exists.lock().unwrap();
debug!("exists {}", n);
work_context
.set_status
.send((
thread_id,
format!(
"got `{} EXISTS` notification (EXISTS was previously {} for {}",
n,
prev_exists.len(),
mailbox.path()
),
))
.unwrap();
if n > prev_exists.len() {
exit_on_error!(
conn,
mailbox_hash,
work_context,
thread_id,
conn.examine_mailbox(mailbox_hash, &mut response)
conn.send_command(
&[
b"FETCH",
format!("{}:{}", prev_exists.len() + 1, n).as_bytes(),
b"(UID FLAGS RFC822)",
]
.join(&b' '),
)
conn.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
);
match protocol_parser::uid_fetch_responses(&response) {
Ok((_, v, _)) => {
let len = v.len();
let mut ctr = 0;
'fetch_responses_b: for UidFetchResponse {
uid, flags, body, ..
} in v
{
work_context
.set_status
.send((
thread_id,
format!("parsing {}/{} envelopes..", ctr, len),
))
.unwrap();
if uid_store
.uid_index
.lock()
.unwrap()
.contains_key(&(mailbox_hash, uid))
{
ctr += 1;
continue 'fetch_responses_b;
}
if let Ok(mut env) = Envelope::from_bytes(
body.unwrap(),
flags.as_ref().map(|&(f, _)| f),
) {
ctr += 1;
uid_store
.hash_index
.lock()
.unwrap()
.insert(env.hash(), (uid, mailbox_hash));
uid_store
.uid_index
.lock()
.unwrap()
.insert((mailbox_hash, uid), env.hash());
if let Some((_, keywords)) = flags {
let mut tag_lck = uid_store.tag_index.write().unwrap();
for f in keywords {
let hash = tag_hash!(f);
if !tag_lck.contains_key(&hash) {
tag_lck.insert(hash, f);
}
env.labels_mut().push(hash);
}
}
debug!(
"Create event {} {} {}",
env.hash(),
env.subject(),
mailbox.path(),
);
if !env.is_seen() {
mailbox.unseen.lock().unwrap().insert_new(env.hash());
}
if uid_store.cache_headers {
cache::save_envelopes(
uid_store.account_hash,
mailbox_hash,
uidvalidity,
&[(uid, &env)],
)?;
}
prev_exists.insert_new(env.hash());
conn.add_refresh_event(RefreshEvent {
account_hash: uid_store.account_hash,
mailbox_hash,
kind: Create(Box::new(env)),
});
}
}
work_context
.set_status
.send((thread_id, format!("parsed {}/{} envelopes.", ctr, len)))
.unwrap();
}
Err(e) => {
debug!(e);
}
}
}
}
Ok(Some(Fetch(msg_seq, flags))) => {
/* a * {msg_seq} FETCH (FLAGS ({flags})) was received, so find out UID from msg_seq
* and send update
*/
let mut conn = super::try_lock(&main_conn, Some(std::time::Duration::new(10, 0)))?;
debug!("fetch {} {:?}", msg_seq, flags);
exit_on_error!(
conn,
mailbox_hash,
work_context,
thread_id,
conn.examine_mailbox(mailbox_hash, &mut response)
conn.send_command(
&[
b"UID SEARCH ",
format!("{}", msg_seq).as_bytes(),
]
.join(&b' '),
)
conn.read_response(&mut response, RequiredResponses::SEARCH)
);
match search_results(response.split_rn().next().unwrap_or("").as_bytes())
.map(|(_, v)| v)
{
Ok(mut v) => {
if let Some(uid) = v.pop() {
if let Some(env_hash) = uid_store
.uid_index
.lock()
.unwrap()
.get(&(mailbox_hash, uid))
{
if !flags.0.intersects(crate::email::Flag::SEEN) {
mailbox.unseen.lock().unwrap().insert_new(*env_hash);
} else {
mailbox.unseen.lock().unwrap().remove(*env_hash);
}
conn.add_refresh_event(RefreshEvent {
account_hash: uid_store.account_hash,
mailbox_hash,
kind: NewFlags(*env_hash, flags),
});
}
}
}
Err(e) => {
debug!(&response);
debug!(e);
}
}
}
Ok(Some(Bye { .. })) => break,
Ok(None) | Err(_) => {}
}
work_context
.set_status
.send((thread_id, "IDLEing".to_string()))
.unwrap();
}
debug!("IDLE connection dropped");
let err: &str = iter.err().unwrap_or("Unknown reason.");
work_context
.set_status
.send((thread_id, "IDLE connection dropped".to_string()))
.unwrap();
work_context.finished.send(thread_id).unwrap();
main_conn.lock().unwrap().add_refresh_event(RefreshEvent {
account_hash: uid_store.account_hash,
mailbox_hash,
kind: RefreshEventKind::Failure(MeliError::new(format!(
"IDLE connection dropped: {}",
&err
))),
});
Err(MeliError::new(format!("IDLE connection dropped: {}", err)))
}
pub fn examine_updates(
mailbox: &ImapMailbox,
conn: &mut ImapConnection,
uid_store: &Arc<UIDStore>,
work_context: &WorkContext,
) -> Result<()> {
let thread_id: std::thread::ThreadId = std::thread::current().id();
let mailbox_hash = mailbox.hash();
debug!("examining mailbox {} {}", mailbox_hash, mailbox.path());
let mut response = String::with_capacity(8 * 1024);
exit_on_error!(
conn,
mailbox_hash,
work_context,
thread_id,
conn.examine_mailbox(mailbox_hash, &mut response)
);
*uid_store.is_online.lock().unwrap() = (Instant::now(), Ok(()));
let uidvalidity;
match protocol_parser::select_response(&response) {
Ok(ok) => {
uidvalidity = ok.uidvalidity;
debug!(&ok);
{
let mut uidvalidities = uid_store.uidvalidity.lock().unwrap();
if let Some(v) = uidvalidities.get_mut(&mailbox_hash) {
if *v != ok.uidvalidity {
conn.add_refresh_event(RefreshEvent {
account_hash: uid_store.account_hash,
mailbox_hash,
kind: RefreshEventKind::Rescan,
});
/*
uid_store.uid_index.lock().unwrap().clear();
uid_store.hash_index.lock().unwrap().clear();
uid_store.byte_cache.lock().unwrap().clear();
*/
*v = ok.uidvalidity;
}
} else {
conn.add_refresh_event(RefreshEvent {
account_hash: uid_store.account_hash,
mailbox_hash,
kind: RefreshEventKind::Rescan,
});
conn.add_refresh_event(RefreshEvent {
account_hash: uid_store.account_hash,
mailbox_hash,
kind: RefreshEventKind::Failure(MeliError::new(format!(
"Unknown mailbox: {} {}",
mailbox.path(),
mailbox_hash
))),
});
}
}
let mut prev_exists = mailbox.exists.lock().unwrap();
let n = ok.exists;
if ok.recent > 0 {
{
/* UID SEARCH RECENT */
exit_on_error!(
conn,
mailbox_hash,
work_context,
thread_id,
conn.send_command(b"UID SEARCH RECENT")
conn.read_response(&mut response, RequiredResponses::SEARCH)
);
match protocol_parser::search_results_raw(response.as_bytes())
.map(|(_, v)| v)
.map_err(MeliError::from)
{
Ok(&[]) => {
debug!("UID SEARCH RECENT returned no results");
}
Ok(v) => {
exit_on_error!(
conn,
mailbox_hash,
work_context,
thread_id,
conn.send_command(
&[&b"UID FETCH"[..], &v.trim().split(|b| b == &b' ').join(b','), &b"(FLAGS RFC822)"[..]]
.join(&b' '),
)
conn.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
);
debug!(&response);
match protocol_parser::uid_fetch_responses(&response) {
Ok((_, v, _)) => {
'fetch_responses_c: for UidFetchResponse {
uid,
flags,
body,
..
} in v
{
if uid_store
.uid_index
.lock()
.unwrap()
.contains_key(&(mailbox_hash, uid))
{
continue 'fetch_responses_c;
}
if let Ok(mut env) = Envelope::from_bytes(
body.unwrap(),
flags.as_ref().map(|&(f, _)| f),
) {
uid_store
.hash_index
.lock()
.unwrap()
.insert(env.hash(), (uid, mailbox_hash));
uid_store
.uid_index
.lock()
.unwrap()
.insert((mailbox_hash, uid), env.hash());
debug!(
"Create event {} {} {}",
env.hash(),
env.subject(),
mailbox.path(),
);
if let Some((_, keywords)) = flags {
let mut tag_lck =
uid_store.tag_index.write().unwrap();
for f in keywords {
let hash = tag_hash!(f);
if !tag_lck.contains_key(&hash) {
tag_lck.insert(hash, f);
}
env.labels_mut().push(hash);
}
}
if !env.is_seen() {
mailbox
.unseen
.lock()
.unwrap()
.insert_new(env.hash());
}
if uid_store.cache_headers {
cache::save_envelopes(
uid_store.account_hash,
mailbox_hash,
uidvalidity,
&[(uid, &env)],
)?;
}
prev_exists.insert_new(env.hash());
conn.add_refresh_event(RefreshEvent {
account_hash: uid_store.account_hash,
mailbox_hash,
kind: Create(Box::new(env)),
});
}
}
}
Err(e) => {
debug!(e);
}
}
}
Err(e) => {
debug!(
"UID SEARCH RECENT err: {}\nresp: {}",
e.to_string(),
&response
);
}
}
}
} else if n > prev_exists.len() {
/* UID FETCH ALL UID, cross-ref, then FETCH difference headers
* */
debug!("exists {}", n);
exit_on_error!(
conn,
mailbox_hash,
work_context,
thread_id,
conn.send_command(
&[
b"FETCH",
format!("{}:{}", prev_exists.len() + 1, n).as_bytes(),
b"(UID FLAGS RFC822)",
]
.join(&b' '),
)
conn.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
);
match protocol_parser::uid_fetch_responses(&response) {
Ok((_, v, _)) => {
'fetch_responses_a: for UidFetchResponse {
uid, flags, body, ..
} in v
{
if uid_store
.uid_index
.lock()
.unwrap()
.contains_key(&(mailbox_hash, uid))
{
continue 'fetch_responses_a;
}
if let Ok(mut env) =
Envelope::from_bytes(body.unwrap(), flags.as_ref().map(|&(f, _)| f))
{
uid_store
.hash_index
.lock()
.unwrap()
.insert(env.hash(), (uid, mailbox_hash));
uid_store
.uid_index
.lock()
.unwrap()
.insert((mailbox_hash, uid), env.hash());
if let Some((_, keywords)) = flags {
let mut tag_lck = uid_store.tag_index.write().unwrap();
for f in keywords {
let hash = tag_hash!(f);
if !tag_lck.contains_key(&hash) {
tag_lck.insert(hash, f);
}
env.labels_mut().push(hash);
}
}
debug!(
"Create event {} {} {}",
env.hash(),
env.subject(),
mailbox.path(),
);
if !env.is_seen() {
mailbox.unseen.lock().unwrap().insert_new(env.hash());
}
if uid_store.cache_headers {
cache::save_envelopes(
uid_store.account_hash,
mailbox_hash,
uidvalidity,
&[(uid, &env)],
)?;
}
prev_exists.insert_new(env.hash());
conn.add_refresh_event(RefreshEvent {
account_hash: uid_store.account_hash,
mailbox_hash,
kind: Create(Box::new(env)),
});
}
}
}
Err(e) => {
debug!(e);
}
}
}
}
Err(e) => {
debug!("{:?}", e);
panic!("could not select mailbox");
}
};
Ok(())
}

9
melib/src/backends/maildir/backend.rs
View File

@ -24,7 +24,6 @@ use super::{
RefreshEventConsumer, RefreshEventKind::*,
};
use super::{MaildirMailbox, MaildirOp};
use futures::prelude::Stream;
use crate::async_workers::*;
use crate::conf::AccountSettings;
use crate::email::{Envelope, EnvelopeHash, Flag};
@ -194,7 +193,12 @@ impl MailBackend for MaildirType {
self.multicore(4, mailbox)
}
fn get_async(&mut self, mailbox: &Mailbox) -> Result<Box<dyn Stream<Item = Result<Vec<Envelope>>>>> {
/*
fn get_async(
&mut self,
mailbox: &Mailbox,
) -> Result<core::pin::Pin<Box<dyn Future<Output = Result<Vec<Envelope>>> + Send + 'static>>>
{
let mailbox: &MaildirMailbox = &self.mailboxes[&self.owned_mailbox_idx(mailbox)];
let mailbox_hash = mailbox.hash();
let unseen = mailbox.unseen.clone();
@ -205,6 +209,7 @@ impl MailBackend for MaildirType {
let mailbox_index = self.mailbox_index.clone();
super::stream::MaildirStream::new(&self.name, mailbox_hash, unseen, total, path, root_path, map, mailbox_index)
}
*/
fn refresh(
&mut self,

1
melib/src/conf.rs
View File

@ -35,6 +35,7 @@ pub struct AccountSettings {
pub mailboxes: HashMap<String, MailboxConf>,
#[serde(default)]
pub manual_refresh: bool,
pub is_async: bool,
#[serde(flatten)]
pub extra: HashMap<String, String>,
}

6
melib/src/error.rs
View File

@ -171,9 +171,11 @@ impl<T> From<std::sync::PoisonError<T>> for MeliError {
}
#[cfg(feature = "imap_backend")]
impl From<native_tls::HandshakeError<std::net::TcpStream>> for MeliError {
impl<T: Sync + Send + 'static + core::fmt::Debug> From<native_tls::HandshakeError<T>>
for MeliError
{
#[inline]
fn from(kind: native_tls::HandshakeError<std::net::TcpStream>) -> MeliError {
fn from(kind: native_tls::HandshakeError<T>) -> MeliError {
MeliError::new(format!("{}", kind)).set_source(Some(Arc::new(kind)))
}
}

3
melib/src/lib.rs
View File

@ -113,6 +113,7 @@ pub mod error;
pub mod thread;
pub use crate::email::*;
pub use crate::thread::*;
pub mod connections;
pub mod parsec;
pub mod search;
@ -131,6 +132,8 @@ extern crate bitflags;
extern crate uuid;
pub use smallvec;
pub use smol;
pub use crate::backends::{Backends, RefreshEvent, RefreshEventConsumer, SpecialUsageMailbox};
pub use crate::collection::*;
pub use crate::conf::*;

5
src/bin.rs
View File

@ -466,6 +466,11 @@ fn run_app(opt: Opt) -> Result<()> {
},
ThreadEvent::JobFinished(id) => {
debug!("Job finished {}", id);
for account in state.context.accounts.iter_mut() {
if account.process_event(&id) {
break;
}
}
//state.new_thread(id, name);
},
}

5
src/conf.rs
View File

@ -167,6 +167,8 @@ pub struct FileAccount {
cache_type: CacheType,
#[serde(default = "false_val")]
pub manual_refresh: bool,
#[serde(default = "false_val")]
pub is_async: bool,
#[serde(default = "none")]
pub refresh_command: Option<String>,
#[serde(flatten)]
@ -197,6 +199,7 @@ impl From<FileAccount> for AccountConf {
display_name,
subscribed_mailboxes: x.subscribed_mailboxes.clone(),
mailboxes,
is_async: x.is_async,
manual_refresh: x.manual_refresh,
extra: x.extra.clone(),
};
@ -412,6 +415,7 @@ impl FileSettings {
mailboxes,
extra,
manual_refresh,
is_async,
refresh_command: _,
cache_type: _,
conf_override: _,
@ -427,6 +431,7 @@ impl FileSettings {
display_name,
subscribed_mailboxes,
manual_refresh,
is_async,
mailboxes: mailboxes
.into_iter()
.map(|(k, v)| (k, v.mailbox_conf))

213
src/conf/accounts.rs
View File

@ -24,7 +24,7 @@
*/
use super::{AccountConf, FileMailboxConf};
use crate::jobs1::JobExecutor;
use crate::jobs1::{JobExecutor, JobId};
use melib::async_workers::{Async, AsyncBuilder, AsyncStatus, WorkContext};
use melib::backends::{
AccountHash, BackendOp, Backends, MailBackend, Mailbox, MailboxHash, NotifyFn, ReadOnlyOp,
@ -42,6 +42,7 @@ use std::collections::{HashMap, HashSet};
use crate::types::UIEvent::{self, EnvelopeRemove, EnvelopeRename, EnvelopeUpdate, Notification};
use crate::{StatusEvent, ThreadEvent};
use crossbeam::Sender;
use futures::channel::oneshot;
pub use futures::stream::Stream;
use std::collections::VecDeque;
use std::fs;
@ -127,11 +128,21 @@ pub struct Account {
pub(crate) settings: AccountConf,
pub(crate) backend: Arc<RwLock<Box<dyn MailBackend>>>,
job_executor: Arc<JobExecutor>,
active_jobs: HashMap<JobId, JobRequest>,
sender: Sender<ThreadEvent>,
event_queue: VecDeque<(MailboxHash, RefreshEvent)>,
notify_fn: Arc<NotifyFn>,
}
#[derive(Debug)]
enum JobRequest {
Mailboxes(oneshot::Receiver<Result<HashMap<MailboxHash, Mailbox>>>),
Get(MailboxHash, oneshot::Receiver<Result<Vec<Envelope>>>),
IsOnline(oneshot::Receiver<Result<()>>),
Refresh(oneshot::Receiver<Result<()>>),
}
impl Drop for Account {
fn drop(&mut self) {
if let Ok(data_dir) = xdg::BaseDirectories::with_profile("meli", &self.name) {
@ -195,7 +206,7 @@ impl Account {
mut settings: AccountConf,
map: &Backends,
work_context: WorkContext,
job_executor: &JobExecutor,
job_executor: Arc<JobExecutor>,
sender: Sender<ThreadEvent>,
notify_fn: NotifyFn,
) -> Result<Self> {
@ -235,6 +246,14 @@ impl Account {
settings.conf.cache_type = crate::conf::CacheType::None;
}
let mut active_jobs = HashMap::default();
if settings.conf.is_async {
if let Ok(mailboxes_job) = backend.mailboxes_async() {
let (rcvr, job_id) = job_executor.spawn_specialized(mailboxes_job);
active_jobs.insert(job_id, JobRequest::Mailboxes(rcvr));
}
}
Ok(Account {
index,
hash,
@ -251,20 +270,18 @@ impl Account {
backend: Arc::new(RwLock::new(backend)),
notify_fn,
sender,
job_executor,
active_jobs,
event_queue: VecDeque::with_capacity(8),
})
}
fn init(&mut self) -> Result<()> {
let mut ref_mailboxes: HashMap<MailboxHash, Mailbox> =
match self.backend.read().unwrap().mailboxes() {
Ok(f) => f,
Err(err) => {
debug!(&err);
return Err(err);
}
};
fn init(&mut self, ref_mailboxes: Option<HashMap<MailboxHash, Mailbox>>) -> Result<()> {
let mut ref_mailboxes: HashMap<MailboxHash, Mailbox> = if let Some(v) = ref_mailboxes {
v
} else {
self.backend.read().unwrap().mailboxes()?
};
let mut mailbox_entries: HashMap<MailboxHash, MailboxEntry> =
HashMap::with_capacity_and_hasher(ref_mailboxes.len(), Default::default());
let mut mailboxes_order: Vec<MailboxHash> = Vec::with_capacity(ref_mailboxes.len());
@ -389,12 +406,19 @@ impl Account {
mailbox_entries.entry(*h).and_modify(|entry| {
if entry.conf.mailbox_conf.autoload {
entry.status = MailboxStatus::Parsing(0, 0);
entry.worker = Account::new_worker(
f.clone(),
&mut self.backend,
&self.work_context,
self.notify_fn.clone(),
);
if self.settings.conf.is_async {
if let Ok(mailbox_job) = self.backend.write().unwrap().get_async(&f) {
let (rcvr, job_id) = self.job_executor.spawn_specialized(mailbox_job);
self.active_jobs.insert(job_id, JobRequest::Get(*h, rcvr));
}
} else {
entry.worker = Account::new_worker(
f.clone(),
&mut self.backend,
&self.work_context,
self.notify_fn.clone(),
);
}
}
});
self.collection.new_mailbox(*h);
@ -799,17 +823,30 @@ impl Account {
Ok(())
}
MailboxStatus::None => {
let handle = Account::new_worker(
self.mailbox_entries[&mailbox_hash].ref_mailbox.clone(),
&mut self.backend,
&self.work_context,
self.notify_fn.clone(),
);
self.mailbox_entries
.entry(mailbox_hash)
.and_modify(|entry| {
entry.worker = handle;
});
if self.settings.conf.is_async {
if let Ok(mailbox_job) =
self.backend.write().unwrap().get_async(
&&self.mailbox_entries[&mailbox_hash].ref_mailbox,
)
{
let (rcvr, job_id) =
self.job_executor.spawn_specialized(mailbox_job);
self.active_jobs
.insert(job_id, JobRequest::Get(mailbox_hash, rcvr));
}
} else {
let handle = Account::new_worker(
self.mailbox_entries[&mailbox_hash].ref_mailbox.clone(),
&mut self.backend,
&self.work_context,
self.notify_fn.clone(),
);
self.mailbox_entries
.entry(mailbox_hash)
.and_modify(|entry| {
entry.worker = handle;
});
}
self.collection.new_mailbox(mailbox_hash);
Err(0)
}
@ -1154,16 +1191,34 @@ impl Account {
self.backend.write().unwrap().connect();
}
let ret = self.backend.read().unwrap().is_online();
if ret.is_ok() != self.is_online && ret.is_ok() {
self.init()?;
if self.settings.conf.is_async {
if self.is_online {
return Ok(());
}
if !self.active_jobs.values().any(|j| {
if let JobRequest::IsOnline(_) = j {
true
} else {
false
}
}) {
if let Ok(online_job) = self.backend.read().unwrap().is_online_async() {
let (rcvr, job_id) = self.job_executor.spawn_specialized(online_job);
self.active_jobs.insert(job_id, JobRequest::IsOnline(rcvr));
}
}
return self.backend.read().unwrap().is_online();
} else {
let ret = self.backend.read().unwrap().is_online();
if ret.is_ok() != self.is_online && ret.is_ok() {
//self.init()?;
}
self.is_online = ret.is_ok();
if !self.is_online {
self.backend.write().unwrap().connect();
}
ret
}
self.is_online = ret.is_ok();
if !self.is_online {
self.backend.write().unwrap().connect();
}
ret
}
pub fn search(
@ -1245,6 +1300,88 @@ impl Account {
Err(MeliError::new("Mailbox with that path not found."))
}
}
pub fn process_event(&mut self, job_id: &JobId) -> bool {
if debug!(self.active_jobs.contains_key(job_id)) {
match debug!(self.active_jobs.remove(job_id)).unwrap() {
JobRequest::Mailboxes(mut chan) => {
if let Some(mailboxes) = debug!(chan.try_recv()).unwrap() {
if mailboxes.is_err()
|| debug!(self.init(Some(mailboxes.unwrap()))).is_err()
{
if let Ok(mailboxes_job) =
self.backend.read().unwrap().mailboxes_async()
{
let (rcvr, job_id) =
self.job_executor.spawn_specialized(mailboxes_job);
self.active_jobs.insert(job_id, JobRequest::Mailboxes(rcvr));
}
}
}
}
JobRequest::Get(mailbox_hash, mut chan) => {
let payload = debug!(chan.try_recv()).unwrap().unwrap();
debug!("got payload in status for {}", mailbox_hash);
if payload.is_err() {
self.mailbox_entries
.entry(mailbox_hash)
.and_modify(|entry| {
entry.status = MailboxStatus::Failed(payload.unwrap_err());
});
self.sender
.send(ThreadEvent::UIEvent(UIEvent::StartupCheck(mailbox_hash)))
.unwrap();
return true;
}
let envelopes = payload
.unwrap()
.into_iter()
.map(|e| (e.hash(), e))
.collect::<HashMap<EnvelopeHash, Envelope>>();
if let Some(updated_mailboxes) =
self.collection
.merge(envelopes, mailbox_hash, self.sent_mailbox)
{
for f in updated_mailboxes {
self.sender
.send(ThreadEvent::UIEvent(UIEvent::StartupCheck(f)))
.unwrap();
}
}
self.sender
.send(ThreadEvent::UIEvent(UIEvent::StartupCheck(mailbox_hash)))
.unwrap();
self.mailbox_entries
.entry(mailbox_hash)
.and_modify(|entry| {
entry.status = MailboxStatus::Available;
entry.worker = None;
});
self.sender
.send(ThreadEvent::UIEvent(UIEvent::MailboxUpdate((
self.index,
mailbox_hash,
))))
.unwrap();
}
JobRequest::IsOnline(mut chan) => {
let is_online = debug!(chan.try_recv()).unwrap();
if is_online.is_some() {
self.is_online = true;
} else {
if let Ok(online_job) = self.backend.read().unwrap().is_online_async() {
let (rcvr, job_id) = self.job_executor.spawn_specialized(online_job);
self.active_jobs.insert(job_id, JobRequest::IsOnline(rcvr));
}
}
}
_ => {}
}
true
} else {
false
}
}
}
impl Index<&MailboxHash> for Account {

145
src/jobs1.rs
View File

@ -19,36 +19,35 @@
* along with meli. If not, see <http://www.gnu.org/licenses/>.
*/
use melib::smol;
use std::future::Future;
use std::panic::catch_unwind;
use std::time::Duration;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use std::thread;
use std::time::Duration;
use uuid::Uuid;
use crate::types::ThreadEvent;
use crossbeam::channel;
use crossbeam::deque::{Injector, Steal, Stealer, Worker};
use std::iter;
use crossbeam::sync::{Parker, Unparker};
use crossbeam::Sender;
use crate::types::ThreadEvent;
use futures::channel::oneshot;
use crossbeam::sync::{Unparker, Parker};
use once_cell::sync::Lazy;
use std::iter;
type AsyncTask = async_task::Task<()>;
fn find_task<T>(
local: &Worker<T>,
global: &Injector<T>,
stealers: &[Stealer<T>],
) -> Option<T> {
fn find_task<T>(local: &Worker<T>, global: &Injector<T>, stealers: &[Stealer<T>]) -> Option<T> {
// Pop a task from the local queue, if not empty.
local.pop().or_else(|| {
// Otherwise, we need to look for a task elsewhere.
iter::repeat_with(|| {
// Try stealing a batch of tasks from the global queue.
global.steal_batch_and_pop(local)
global
.steal_batch_and_pop(local)
// Or try stealing a task from one of the other threads.
.or_else(|| stealers.iter().map(|s| s.steal()).collect())
})
@ -94,6 +93,7 @@ pub struct MeliTask {
id: JobId,
}
#[derive(Debug)]
pub struct JobExecutor {
active_jobs: Vec<JobId>,
global_queue: Arc<Injector<MeliTask>>,
@ -103,52 +103,51 @@ pub struct JobExecutor {
}
impl JobExecutor {
/// A queue that holds scheduled tasks.
/// A queue that holds scheduled tasks.
pub fn new(sender: Sender<ThreadEvent>) -> Self {
// Create a queue.
// Create a queue.
let mut ret = JobExecutor {
active_jobs: vec![],
global_queue: Arc::new(Injector::new()),
workers: vec![],
parkers: vec![],
parkers: vec![],
sender,
};
let mut workers = vec![];
for _ in 0..num_cpus::get().max(1) {
let new_worker = Worker::new_fifo();
ret.workers.push(new_worker.stealer());
let p = Parker::new();
ret.parkers.push(p.unparker().clone());
workers.push((new_worker, p));
}
for _ in 0..num_cpus::get().max(1) {
let new_worker = Worker::new_fifo();
ret.workers.push(new_worker.stealer());
let p = Parker::new();
ret.parkers.push(p.unparker().clone());
workers.push((new_worker, p));
}
// Spawn executor threads the first time the queue is created.
for (i, (local, parker)) in workers.into_iter().enumerate() {
let sender = ret.sender.clone();
let global = ret.global_queue.clone();
let stealers = ret.workers.clone();
// Reactor thread
thread::Builder::new()
.name(format!("meli executor thread {}", i))
.spawn(move || {
loop {
parker.park_timeout(Duration::from_millis(100));
let task = find_task(&local, &global, stealers.as_slice());
if let Some(meli_task) = task {
let MeliTask { task, id} = meli_task;
debug!("Worker {} got task {:?}", i,id);
if let Ok(false) = catch_unwind(|| task.run()) {
debug!("Worker {} got result {:?}", i,id);
sender.send(ThreadEvent::JobFinished(id)).unwrap();
} else {
debug!("Worker {} rescheduled {:?}", i,id);
.name("meli-reactor".to_string())
.spawn(move || {
smol::run(futures::future::pending::<()>());
});
// Spawn executor threads the first time the queue is created.
for (i, (local, parker)) in workers.into_iter().enumerate() {
let sender = ret.sender.clone();
let global = ret.global_queue.clone();
let stealers = ret.workers.clone();
thread::Builder::new()
.name(format!("meli-executor-{}", i))
.spawn(move || loop {
parker.park_timeout(Duration::from_millis(100));
let task = find_task(&local, &global, stealers.as_slice());
if let Some(meli_task) = task {
let MeliTask { task, id } = meli_task;
debug!("Worker {} got task {:?}", i, id);
let res = catch_unwind(|| task.run());
debug!("Worker {} got result {:?}", i, id);
}
}
}
});
}
ret
});
}
ret
}
/// Spawns a future on the executor.
pub fn spawn<F>(&self, future: F) -> JoinHandle
@ -157,46 +156,62 @@ ret.parkers.push(p.unparker().clone());
{
let job_id = JobId::new();
let _job_id = job_id.clone();
let __job_id = job_id.clone();
let finished_sender = self.sender.clone();
let injector = self.global_queue.clone();
// Create a task and schedule it for execution.
let (task, handle) = async_task::spawn(async {
let _ = future.await;
}, move |task| injector.push(MeliTask { task, id: _job_id }), ());
let (task, handle) = async_task::spawn(
async move {
let _ = future.await;
finished_sender
.send(ThreadEvent::JobFinished(__job_id))
.unwrap();
},
move |task| injector.push(MeliTask { task, id: _job_id }),
(),
);
task.schedule();
for unparker in self.parkers.iter() {
unparker.unpark();
unparker.unpark();
}
// Return a join handle that retrieves the output of the future.
JoinHandle(handle)
}
///// Spawns a future on the executor.
pub fn spawn_specialized<F, R>(&self, future: F) -> (oneshot::Receiver<R>, JobId)
where
F: Future<Output = R> + Send + 'static,
R: Send + 'static,
{
let (sender, receiver) = oneshot::channel();
///// Spawns a future on the executor.
pub fn spawn_specialized<F, R>(&self, future: F) -> (oneshot::Receiver<R>, JobId)
where
F: Future<Output = R> + Send + 'static,
R: Send + core::fmt::Debug + 'static,
{
let (sender, receiver) = oneshot::channel();
let finished_sender = self.sender.clone();
let job_id = JobId::new();
let _job_id = job_id.clone();
let __job_id = job_id.clone();
let injector = self.global_queue.clone();
// Create a task and schedule it for execution.
let (task, handle) = async_task::spawn(async {
let res = future.await;
sender.send(res);
}, move |task| injector.push(MeliTask { task, id: _job_id }), ());
let (task, handle) = async_task::spawn(
async move {
let res = future.await;
sender.send(res).unwrap();
finished_sender
.send(ThreadEvent::JobFinished(__job_id))
.unwrap();
},
move |task| injector.push(MeliTask { task, id: _job_id }),
(),
);
task.schedule();
for unparker in self.parkers.iter() {
unparker.unpark();
unparker.unpark();
}
(receiver, job_id)
}
}
}
///// Spawns a future on the executor.
//fn spawn<F, R>(future: F) -> JoinHandle<R>
//where
@ -212,7 +227,7 @@ where
//}
/// Awaits the output of a spawned future.
pub struct JoinHandle( async_task::JoinHandle<(), ()>);
pub struct JoinHandle(async_task::JoinHandle<(), ()>);
impl Future for JoinHandle {
type Output = ();

15
src/state.rs
View File

@ -39,6 +39,7 @@ use std::collections::HashMap;
use std::env;
use std::io::Write;
use std::os::unix::io::RawFd;
use std::sync::Arc;
use std::thread;
use termion::raw::IntoRawMode;
use termion::screen::AlternateScreen;
@ -96,7 +97,7 @@ pub struct Context {
receiver: Receiver<ThreadEvent>,
input: InputHandler,
work_controller: WorkController,
job_executor: JobExecutor,
job_executor: Arc<JobExecutor>,
pub children: Vec<std::process::Child>,
pub plugin_manager: PluginManager,
@ -246,7 +247,7 @@ impl State {
let mut account_hashes = HashMap::with_capacity_and_hasher(1, Default::default());
let work_controller = WorkController::new(sender.clone());
let job_executor = JobExecutor::new(sender.clone());
let job_executor = Arc::new(JobExecutor::new(sender.clone()));
let accounts: Vec<Account> = {
let mut file_accs = settings
.accounts
@ -274,7 +275,7 @@ impl State {
a_s.clone(),
&backends,
work_controller.get_context(),
&job_executor,
job_executor.clone(),
sender.clone(),
NotifyFn::new(Box::new(move |f: MailboxHash| {
sender
@ -360,10 +361,10 @@ impl State {
debug!("inserting mailbox hashes:");
for i in 0..s.context.accounts.len() {
if s.context.is_online(i).is_ok() && s.context.accounts[i].is_empty() {
return Err(MeliError::new(format!(
"Account {} has no mailboxes configured.",
s.context.accounts[i].name()
)));
//return Err(MeliError::new(format!(
// "Account {} has no mailboxes configured.",
// s.context.accounts[i].name()
//)));
}
}
s.context.restore_input();