From 522f6673501b5f5edd89bfea1dde64ef53927579 Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Thu, 30 Jul 2020 20:58:53 +0300 Subject: [PATCH] melib: add experimental NNTP backend Closes #54 --- melib/src/backends.rs | 10 + melib/src/backends/nntp.rs | 568 +++++++++++++++++++++ melib/src/backends/nntp/connection.rs | 376 ++++++++++++++ melib/src/backends/nntp/mailbox.rs | 98 ++++ melib/src/backends/nntp/operations.rs | 93 ++++ melib/src/backends/nntp/protocol_parser.rs | 163 ++++++ 6 files changed, 1308 insertions(+) create mode 100644 melib/src/backends/nntp.rs create mode 100644 melib/src/backends/nntp/connection.rs create mode 100644 melib/src/backends/nntp/mailbox.rs create mode 100644 melib/src/backends/nntp/operations.rs create mode 100644 melib/src/backends/nntp/protocol_parser.rs diff --git a/melib/src/backends.rs b/melib/src/backends.rs index a84b3586..1244f6d2 100644 --- a/melib/src/backends.rs +++ b/melib/src/backends.rs @@ -33,6 +33,8 @@ macro_rules! tag_hash { #[cfg(feature = "imap_backend")] pub mod imap; +#[cfg(feature = "imap_backend")] +pub mod nntp; #[cfg(feature = "notmuch_backend")] pub mod notmuch; #[cfg(feature = "notmuch_backend")] @@ -47,6 +49,7 @@ pub mod mbox; pub use self::imap::ImapType; #[cfg(feature = "jmap_backend")] pub use self::jmap::JmapType; +pub use self::nntp::NntpType; use crate::async_workers::*; use crate::conf::AccountSettings; use crate::error::{MeliError, Result}; @@ -144,6 +147,13 @@ impl Backends { validate_conf_fn: Box::new(imap::ImapType::validate_config), }, ); + b.register( + "nntp".to_string(), + Backend { + create_fn: Box::new(|| Box::new(|f, i| nntp::NntpType::new(f, i))), + validate_conf_fn: Box::new(nntp::NntpType::validate_config), + }, + ); } #[cfg(feature = "notmuch_backend")] { diff --git a/melib/src/backends/nntp.rs b/melib/src/backends/nntp.rs new file mode 100644 index 00000000..c4745932 --- /dev/null +++ b/melib/src/backends/nntp.rs @@ -0,0 +1,568 @@ +/* + * meli - nntp module. + * + * Copyright 2019 Manos Pitsidianakis + * + * This file is part of meli. + * + * meli is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * meli is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with meli. If not, see . + */ + +use crate::get_conf_val; +use crate::get_path_hash; +use smallvec::SmallVec; +#[macro_use] +mod protocol_parser; +pub use protocol_parser::*; +mod mailbox; +pub use mailbox::*; +mod operations; +pub use operations::*; +mod connection; +pub use connection::*; + +use crate::async_workers::{Async, WorkContext}; +use crate::backends::*; +use crate::conf::AccountSettings; +use crate::email::*; +use crate::error::{MeliError, Result}; +use futures::lock::Mutex as FutureMutex; +use futures::stream::Stream; +use std::collections::{hash_map::DefaultHasher, BTreeMap}; +use std::collections::{HashMap, HashSet}; +use std::future::Future; +use std::hash::Hasher; +use std::pin::Pin; +use std::str::FromStr; +use std::sync::{Arc, Mutex, RwLock}; +use std::time::Instant; +pub type UID = usize; + +pub static SUPPORTED_CAPABILITIES: &[&str] = &[ + #[cfg(feature = "deflate_compression")] + "COMPRESS DEFLATE", + "VERSION 2", +]; + +#[derive(Debug, Clone)] +pub struct NntpServerConf { + pub server_hostname: String, + pub server_username: String, + //pub server_password: String, + pub server_port: u16, + pub use_starttls: bool, + pub use_tls: bool, + pub danger_accept_invalid_certs: bool, + pub extension_use: NntpExtensionUse, +} + +pub struct IsSubscribedFn(Box bool + Send + Sync>); + +impl std::fmt::Debug for IsSubscribedFn { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "IsSubscribedFn Box") + } +} + +impl std::ops::Deref for IsSubscribedFn { + type Target = Box bool + Send + Sync>; + fn deref(&self) -> &Box bool + Send + Sync> { + &self.0 + } +} +type Capabilities = HashSet>; + +#[derive(Debug)] +pub struct UIDStore { + account_hash: AccountHash, + account_name: Arc, + offline_cache: bool, + capabilities: Arc>, + hash_index: Arc>>, + uid_index: Arc>>, + + mailboxes: Arc>>, + is_online: Arc)>>, + refresh_events: Arc>>, + sender: Arc>>, +} + +impl Default for UIDStore { + fn default() -> Self { + UIDStore { + account_hash: 0, + account_name: Arc::new(String::new()), + offline_cache: false, + capabilities: Default::default(), + hash_index: Default::default(), + uid_index: Default::default(), + mailboxes: Arc::new(FutureMutex::new(Default::default())), + is_online: Arc::new(Mutex::new(( + Instant::now(), + Err(MeliError::new("Account is uninitialised.")), + ))), + refresh_events: Default::default(), + sender: Arc::new(RwLock::new(None)), + } + } +} + +#[derive(Debug)] +pub struct NntpType { + is_subscribed: Arc, + connection: Arc>, + server_conf: NntpServerConf, + uid_store: Arc, + can_create_flags: Arc>, +} + +impl MailBackend for NntpType { + fn capabilities(&self) -> MailBackendCapabilities { + const CAPABILITIES: MailBackendCapabilities = MailBackendCapabilities { + is_async: true, + is_remote: true, + supports_search: false, + supports_tags: false, + }; + CAPABILITIES + } + + fn fetch_async( + &mut self, + mailbox_hash: MailboxHash, + ) -> Result>> + Send + 'static>>> { + let uid_store = self.uid_store.clone(); + let connection = self.connection.clone(); + Ok(Box::pin(async_stream::try_stream! { + { + let f = &uid_store.mailboxes.lock().await[&mailbox_hash]; + f.exists.lock().unwrap().clear(); + f.unseen.lock().unwrap().clear(); + }; + let ret = fetch_envs(mailbox_hash, connection, &uid_store).await?; + yield ret; + })) + } + + fn refresh_async( + &mut self, + _mailbox_hash: MailboxHash, + _sender: RefreshEventConsumer, + ) -> ResultFuture<()> { + Err(MeliError::new("Unimplemented.")) + } + + fn mailboxes_async(&self) -> ResultFuture> { + let uid_store = self.uid_store.clone(); + let connection = self.connection.clone(); + Ok(Box::pin(async move { + NntpType::nntp_mailboxes(&connection).await?; + let mailboxes_lck = uid_store.mailboxes.lock().await; + let ret = mailboxes_lck + .iter() + .map(|(h, f)| (*h, Box::new(Clone::clone(f)) as Mailbox)) + .collect(); + Ok(ret) + })) + } + + fn is_online_async(&self) -> ResultFuture<()> { + let connection = self.connection.clone(); + Ok(Box::pin(async move { + match timeout(std::time::Duration::from_secs(3), connection.lock()).await { + Ok(mut conn) => { + debug!("is_online_async"); + match debug!(timeout(std::time::Duration::from_secs(3), conn.connect()).await) { + Ok(Ok(())) => Ok(()), + Err(err) | Ok(Err(err)) => { + conn.stream = Err(err.clone()); + debug!(conn.connect().await) + } + } + } + Err(err) => Err(err), + } + })) + } + + fn fetch(&mut self, _mailbox_hash: MailboxHash) -> Result>>> { + Err(MeliError::new("Unimplemented.")) + } + + fn refresh( + &mut self, + _mailbox_hash: MailboxHash, + _sender: RefreshEventConsumer, + ) -> Result> { + Err(MeliError::new("Unimplemented.")) + } + + fn watch( + &self, + _sender: RefreshEventConsumer, + _work_context: WorkContext, + ) -> Result { + Err(MeliError::new("Unimplemented.")) + } + + fn watch_async(&self, _sender: RefreshEventConsumer) -> ResultFuture<()> { + Err(MeliError::new("Unimplemented.")) + } + + fn mailboxes(&self) -> Result> { + Err(MeliError::new("Unimplemented.")) + } + + fn operation(&self, env_hash: EnvelopeHash) -> Result> { + let (uid, mailbox_hash) = if let Some(v) = + self.uid_store.hash_index.lock().unwrap().get(&env_hash) + { + *v + } else { + return Err(MeliError::new( + "Message not found in local cache, it might have been deleted before you requested it." + )); + }; + Ok(Box::new(NntpOp::new( + uid, + mailbox_hash, + self.connection.clone(), + self.uid_store.clone(), + ))) + } + + fn save( + &self, + _bytes: Vec, + _mailbox_hash: MailboxHash, + _flags: Option, + ) -> ResultFuture<()> { + Err(MeliError::new("Unimplemented.")) + } + + fn copy_messages( + &mut self, + _env_hashes: EnvelopeHashBatch, + _source_mailbox_hash: MailboxHash, + _destination_mailbox_hash: MailboxHash, + _move_: bool, + _destination_flags: Option, + ) -> ResultFuture<()> { + Err(MeliError::new("Unimplemented.")) + } + + fn set_flags( + &mut self, + _env_hashes: EnvelopeHashBatch, + _mailbox_hash: MailboxHash, + _flags: SmallVec<[(std::result::Result, bool); 8]>, + ) -> ResultFuture<()> { + Err(MeliError::new("Unimplemented.")) + } + + fn as_any(&self) -> &dyn ::std::any::Any { + self + } + + fn as_any_mut(&mut self) -> &mut dyn ::std::any::Any { + self + } + + fn tags(&self) -> Option>>> { + None + } + + fn create_mailbox( + &mut self, + _path: String, + ) -> ResultFuture<(MailboxHash, HashMap)> { + Err(MeliError::new("Unimplemented.")) + } + + fn delete_mailbox( + &mut self, + _mailbox_hash: MailboxHash, + ) -> ResultFuture> { + Err(MeliError::new("Unimplemented.")) + } + + fn set_mailbox_subscription( + &mut self, + _mailbox_hash: MailboxHash, + _new_val: bool, + ) -> ResultFuture<()> { + Err(MeliError::new("Unimplemented.")) + } + + fn rename_mailbox( + &mut self, + _mailbox_hash: MailboxHash, + _new_path: String, + ) -> ResultFuture { + Err(MeliError::new("Unimplemented.")) + } + + fn set_mailbox_permissions( + &mut self, + _mailbox_hash: MailboxHash, + _val: crate::backends::MailboxPermissions, + ) -> ResultFuture<()> { + Err(MeliError::new("Unimplemented.")) + } + + fn search( + &self, + _query: crate::search::Query, + _mailbox_hash: Option, + ) -> ResultFuture> { + Err(MeliError::new("Unimplemented.")) + } +} + +impl NntpType { + pub fn new( + s: &AccountSettings, + is_subscribed: Box bool + Send + Sync>, + ) -> Result> { + let server_hostname = get_conf_val!(s["server_hostname"])?; + /*let server_username = get_conf_val!(s["server_username"], "")?; + let server_password = if !s.extra.contains_key("server_password_command") { + get_conf_val!(s["server_password"], "")?.to_string() + } else { + let invocation = get_conf_val!(s["server_password_command"])?; + let output = std::process::Command::new("sh") + .args(&["-c", invocation]) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .output()?; + if !output.status.success() { + return Err(MeliError::new(format!( + "({}) server_password_command `{}` returned {}: {}", + s.name, + get_conf_val!(s["server_password_command"])?, + output.status, + String::from_utf8_lossy(&output.stderr) + ))); + } + std::str::from_utf8(&output.stdout)?.trim_end().to_string() + }; + */ + let server_port = get_conf_val!(s["server_port"], 119)?; + let use_tls = get_conf_val!(s["use_tls"], true)?; + let use_starttls = use_tls && get_conf_val!(s["use_starttls"], !(server_port == 993))?; + let danger_accept_invalid_certs: bool = + get_conf_val!(s["danger_accept_invalid_certs"], false)?; + let server_conf = NntpServerConf { + server_hostname: server_hostname.to_string(), + server_username: String::new(), + //server_password, + server_port, + use_tls, + use_starttls, + danger_accept_invalid_certs, + extension_use: NntpExtensionUse::default(), + }; + let account_hash = { + let mut hasher = DefaultHasher::new(); + hasher.write(s.name.as_bytes()); + hasher.finish() + }; + let account_name = Arc::new(s.name().to_string()); + let mut mailboxes = HashMap::default(); + for (k, _f) in s.mailboxes.iter() { + let mailbox_hash = get_path_hash!(&k); + mailboxes.insert( + mailbox_hash, + NntpMailbox { + hash: mailbox_hash, + nntp_path: k.to_string(), + high_watermark: Arc::new(Mutex::new(0)), + low_watermark: Arc::new(Mutex::new(0)), + exists: Default::default(), + unseen: Default::default(), + }, + ); + } + if mailboxes.is_empty() { + return Err(MeliError::new(format!( + "{} has no newsgroups configured.", + account_name + ))); + } + let uid_store: Arc = Arc::new(UIDStore { + account_hash, + account_name, + offline_cache: get_conf_val!(s["X_header_caching"], false)?, + mailboxes: Arc::new(FutureMutex::new(mailboxes)), + ..UIDStore::default() + }); + let connection = NntpConnection::new_connection(&server_conf, uid_store.clone()); + + Ok(Box::new(NntpType { + server_conf, + is_subscribed: Arc::new(IsSubscribedFn(is_subscribed)), + can_create_flags: Arc::new(Mutex::new(false)), + connection: Arc::new(FutureMutex::new(connection)), + uid_store, + })) + } + + pub async fn nntp_mailboxes(connection: &Arc>) -> Result<()> { + let mut res = String::with_capacity(8 * 1024); + let mut conn = connection.lock().await; + let command = { + let mailboxes_lck = conn.uid_store.mailboxes.lock().await; + mailboxes_lck + .values() + .fold("LIST ACTIVE ".to_string(), |mut acc, x| { + if acc.len() != "LIST ACTIVE ".len() { + acc.push(','); + } + acc.push_str(x.name()); + acc + }) + }; + conn.send_command(command.as_bytes()).await?; + conn.read_response(&mut res, true).await?; + if !res.starts_with("215 ") { + return Err(MeliError::new(format!( + "Could not get newsgroups {}: expected LIST ACTIVE response but got: {}", + &conn.uid_store.account_name, res + ))); + } + debug!(&res); + let mut mailboxes_lck = conn.uid_store.mailboxes.lock().await; + for l in res.split_rn().skip(1) { + let s = l.split_whitespace().collect::>(); + if s.len() != 3 { + continue; + } + let mailbox_hash = get_path_hash!(&s[0]); + mailboxes_lck.entry(mailbox_hash).and_modify(|m| { + *m.high_watermark.lock().unwrap() = usize::from_str(s[1]).unwrap_or(0); + *m.low_watermark.lock().unwrap() = usize::from_str(s[2]).unwrap_or(0); + }); + } + Ok(()) + } + + pub fn validate_config(s: &AccountSettings) -> Result<()> { + get_conf_val!(s["server_hostname"])?; + get_conf_val!(s["server_username"], String::new())?; + if !s.extra.contains_key("server_password_command") { + get_conf_val!(s["server_password"], String::new())?; + } else if s.extra.contains_key("server_password") { + return Err(MeliError::new(format!( + "Configuration error ({}): both server_password and server_password_command are set, cannot choose", + s.name.as_str(), + ))); + } + let server_port = get_conf_val!(s["server_port"], 119)?; + let use_tls = get_conf_val!(s["use_tls"], true)?; + let use_starttls = get_conf_val!(s["use_starttls"], !(server_port == 993))?; + if !use_tls && use_starttls { + return Err(MeliError::new(format!( + "Configuration error ({}): incompatible use_tls and use_starttls values: use_tls = false, use_starttls = true", + s.name.as_str(), + ))); + } + get_conf_val!(s["danger_accept_invalid_certs"], false)?; + Ok(()) + } + + pub fn capabilities(&self) -> Vec { + self.uid_store + .capabilities + .lock() + .unwrap() + .iter() + .map(|c| String::from_utf8_lossy(c).into()) + .collect::>() + } +} + +async fn fetch_envs( + mailbox_hash: MailboxHash, + connection: Arc>, + uid_store: &UIDStore, +) -> Result> { + let mut res = String::with_capacity(8 * 1024); + let mut conn = connection.lock().await; + let path = uid_store.mailboxes.lock().await[&mailbox_hash] + .name() + .to_string(); + conn.send_command(format!("GROUP {}", path).as_bytes()) + .await?; + conn.read_response(&mut res, false).await?; + if !res.starts_with("211 ") { + return Err(MeliError::new(format!( + "{} Could not select newsgroup {}: expected GROUP response but got: {}", + &uid_store.account_name, path, res + ))); + } + /* + * Parameters + group Name of newsgroup + number Estimated number of articles in the group + low Reported low water mark + high Reported high water mark + */ + let s = res.split_whitespace().collect::>(); + if s.len() != 5 { + return Err(MeliError::new(format!( + "{} Could not select newsgroup {}: expected GROUP response but got: {}", + &uid_store.account_name, path, res + ))); + } + let total = usize::from_str(&s[1]).unwrap_or(0); + let _low = usize::from_str(&s[2]).unwrap_or(0); + let high = usize::from_str(&s[3]).unwrap_or(0); + drop(s); + + conn.send_command(format!("OVER {}-{}", high.saturating_sub(250), high).as_bytes()) + .await?; + conn.read_response(&mut res, true).await?; + if !res.starts_with("224 ") { + return Err(MeliError::new(format!( + "{} Could not select newsgroup {}: expected OVER response but got: {}", + &uid_store.account_name, path, res + ))); + } + let mut ret = Vec::with_capacity(total); + //hash_index: Arc>>, + //uid_index: Arc>>, + let mut hash_index_lck = uid_store.hash_index.lock().unwrap(); + let mut uid_index_lck = uid_store.uid_index.lock().unwrap(); + for l in res.split_rn().skip(1) { + debug!(&l); + let (_, (num, env)) = debug!(protocol_parser::over_article(&l))?; + hash_index_lck.insert(env.hash(), (num, mailbox_hash)); + uid_index_lck.insert((mailbox_hash, num), env.hash()); + ret.push(env); + } + Ok(ret) +} + +use futures::future::{self, Either}; + +async fn timeout(dur: std::time::Duration, f: impl Future) -> Result { + futures::pin_mut!(f); + match future::select(f, smol::Timer::after(dur)).await { + Either::Left((out, _)) => Ok(out), + Either::Right(_) => { + Err(MeliError::new("Timedout").set_kind(crate::error::ErrorKind::Network)) + } + } +} diff --git a/melib/src/backends/nntp/connection.rs b/melib/src/backends/nntp/connection.rs new file mode 100644 index 00000000..26cfc3ff --- /dev/null +++ b/melib/src/backends/nntp/connection.rs @@ -0,0 +1,376 @@ +/* + * meli - nntp module. + * + * Copyright 2017 - 2019 Manos Pitsidianakis + * + * This file is part of meli. + * + * meli is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * meli is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with meli. If not, see . + */ + +use crate::backends::MailboxHash; +use crate::connections::{lookup_ipv4, Connection}; +use crate::email::parser::BytesExt; +use crate::error::*; +extern crate native_tls; +use futures::io::{AsyncReadExt, AsyncWriteExt}; +use native_tls::TlsConnector; +pub use smol::Async as AsyncWrapper; +use std::collections::HashSet; +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Instant; + +use super::{Capabilities, NntpServerConf, UIDStore}; + +#[derive(Debug, Clone, Copy)] +pub struct NntpExtensionUse { + #[cfg(feature = "deflate_compression")] + pub deflate: bool, +} + +impl Default for NntpExtensionUse { + fn default() -> Self { + Self { + #[cfg(feature = "deflate_compression")] + deflate: true, + } + } +} + +#[derive(Debug)] +pub struct NntpStream { + pub stream: AsyncWrapper, + pub extension_use: NntpExtensionUse, + pub current_mailbox: MailboxSelection, +} + +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub enum MailboxSelection { + None, + Select(MailboxHash), + Examine(MailboxHash), +} + +impl MailboxSelection { + pub fn take(&mut self) -> Self { + std::mem::replace(self, MailboxSelection::None) + } +} + +async fn try_await(cl: impl Future> + Send) -> Result<()> { + cl.await +} + +#[derive(Debug)] +pub struct NntpConnection { + pub stream: Result, + pub server_conf: NntpServerConf, + pub uid_store: Arc, +} + +impl NntpStream { + pub async fn new_connection( + server_conf: &NntpServerConf, + ) -> Result<(Capabilities, NntpStream)> { + use std::net::TcpStream; + let path = &server_conf.server_hostname; + + let stream = { + let addr = lookup_ipv4(path, server_conf.server_port)?; + AsyncWrapper::new(Connection::Tcp( + TcpStream::connect_timeout(&addr, std::time::Duration::new(4, 0)) + .chain_err_kind(crate::error::ErrorKind::Network)?, + )) + .chain_err_kind(crate::error::ErrorKind::Network)? + }; + let mut res = String::with_capacity(8 * 1024); + let mut ret = NntpStream { + stream, + extension_use: NntpExtensionUse::default(), + current_mailbox: MailboxSelection::None, + }; + + ret.read_response(&mut res, false).await?; + ret.send_command(b"CAPABILITIES").await?; + ret.read_response(&mut res, true).await?; + if !res.starts_with("101 ") { + return Err(MeliError::new(format!( + "Could not connect to {}: expected CAPABILITIES response but got:{}", + &server_conf.server_hostname, res + ))); + } + let capabilities: Vec<&str> = res.lines().skip(1).collect(); + + if !capabilities + .iter() + .any(|cap| cap.eq_ignore_ascii_case("VERSION 2")) + { + return Err(MeliError::new(format!( + "Could not connect to {}: server is not NNTP compliant", + &server_conf.server_hostname + ))); + } + + if server_conf.use_tls { + let mut connector = TlsConnector::builder(); + if server_conf.danger_accept_invalid_certs { + connector.danger_accept_invalid_certs(true); + } + let connector = connector + .build() + .chain_err_kind(crate::error::ErrorKind::Network)?; + + if server_conf.use_starttls { + ret.stream + .write_all(b"STARTTLS\r\n") + .await + .chain_err_kind(crate::error::ErrorKind::Network)?; + ret.read_response(&mut res, false).await?; + if !res.starts_with("382 ") { + return Err(MeliError::new(format!( + "Could not connect to {}: could not begin TLS negotiation, got: {}", + &server_conf.server_hostname, res + ))); + } + } + + { + // FIXME: This is blocking + let socket = ret + .stream + .into_inner() + .chain_err_kind(crate::error::ErrorKind::Network)?; + let mut conn_result = connector.connect(path, socket); + if let Err(native_tls::HandshakeError::WouldBlock(midhandshake_stream)) = + conn_result + { + let mut midhandshake_stream = Some(midhandshake_stream); + loop { + match midhandshake_stream.take().unwrap().handshake() { + Ok(r) => { + conn_result = Ok(r); + break; + } + Err(native_tls::HandshakeError::WouldBlock(stream)) => { + midhandshake_stream = Some(stream); + } + p => { + p.chain_err_kind(crate::error::ErrorKind::Network)?; + } + } + } + } + ret.stream = AsyncWrapper::new(Connection::Tls( + conn_result.chain_err_kind(crate::error::ErrorKind::Network)?, + )) + .chain_err_kind(crate::error::ErrorKind::Network)?; + } + } + //ret.send_command( + // format!( + // "LOGIN \"{}\" \"{}\"", + // &server_conf.server_username, &server_conf.server_password + // ) + // .as_bytes(), + //) + //.await?; + + ret.send_command(b"CAPABILITIES").await?; + ret.read_response(&mut res, true).await?; + if !res.starts_with("101 ") { + return Err(MeliError::new(format!( + "Could not connect to {}: expected CAPABILITIES response but got:{}", + &server_conf.server_hostname, res + ))); + } + let capabilities: HashSet> = + res.lines().skip(1).map(|l| l.as_bytes().to_vec()).collect(); + #[cfg(feature = "deflate_compression")] + #[cfg(feature = "deflate_compression")] + if capabilities.contains(&b"COMPRESS DEFLATE"[..]) && ret.extension_use.deflate { + ret.send_command(b"COMPRESS DEFLATE").await?; + ret.read_response(&mut res, false).await?; + if !res.starts_with("206 ") { + crate::log( + format!( + "Could not use COMPRESS=DEFLATE in account `{}`: server replied with `{}`", + server_conf.server_hostname, res + ), + crate::LoggingLevel::WARN, + ); + } else { + let NntpStream { + stream, + extension_use, + current_mailbox, + } = ret; + let stream = stream.into_inner()?; + return Ok(( + capabilities, + NntpStream { + stream: AsyncWrapper::new(stream.deflate())?, + extension_use, + current_mailbox, + }, + )); + } + } + + Ok((capabilities, ret)) + } + + pub async fn read_response(&mut self, ret: &mut String, is_multiline: bool) -> Result<()> { + self.read_lines(ret, is_multiline).await?; + Ok(()) + } + + pub async fn read_lines(&mut self, ret: &mut String, is_multiline: bool) -> Result<()> { + let mut buf: Vec = vec![0; Connection::IO_BUF_SIZE]; + ret.clear(); + let mut last_line_idx: usize = 0; + loop { + match self.stream.read(&mut buf).await { + Ok(0) => break, + Ok(b) => { + ret.push_str(unsafe { std::str::from_utf8_unchecked(&buf[0..b]) }); + if let Some(mut pos) = ret[last_line_idx..].rfind("\r\n") { + if !is_multiline { + break; + } else if let Some(pos) = ret.find("\r\n.\r\n") { + ret.replace_range(pos + "\r\n".len()..pos + "\r\n.\r\n".len(), ""); + break; + } + if ret[last_line_idx..].starts_with("205 ") { + return Err(MeliError::new(format!("Disconnected: {}", ret))); + } + if let Some(prev_line) = + ret[last_line_idx..pos + last_line_idx].rfind("\r\n") + { + last_line_idx += prev_line + "\r\n".len(); + pos -= prev_line + "\r\n".len(); + } + last_line_idx += pos + "\r\n".len(); + } + } + Err(e) => { + return Err(MeliError::from(e).set_err_kind(crate::error::ErrorKind::Network)); + } + } + } + //debug!("returning nntp response:\n{:?}", &ret); + Ok(()) + } + + pub async fn send_command(&mut self, command: &[u8]) -> Result<()> { + if let Err(err) = try_await(async move { + let command = command.trim(); + self.stream.write_all(command).await?; + self.stream.write_all(b"\r\n").await?; + self.stream.flush().await?; + debug!("sent: {}", unsafe { + std::str::from_utf8_unchecked(command) + }); + Ok(()) + }) + .await + { + debug!("stream send_command err {:?}", err); + Err(err.set_err_kind(crate::error::ErrorKind::Network)) + } else { + Ok(()) + } + } +} + +impl NntpConnection { + pub fn new_connection( + server_conf: &NntpServerConf, + uid_store: Arc, + ) -> NntpConnection { + NntpConnection { + stream: Err(MeliError::new("Offline".to_string())), + server_conf: server_conf.clone(), + uid_store, + } + } + + pub fn connect<'a>(&'a mut self) -> Pin> + Send + 'a>> { + Box::pin(async move { + if let (instant, ref mut status @ Ok(())) = *self.uid_store.is_online.lock().unwrap() { + if Instant::now().duration_since(instant) >= std::time::Duration::new(60 * 30, 0) { + *status = Err(MeliError::new("Connection timed out")); + self.stream = Err(MeliError::new("Connection timed out")); + } + } + if self.stream.is_ok() { + self.uid_store.is_online.lock().unwrap().0 = Instant::now(); + return Ok(()); + } + let new_stream = NntpStream::new_connection(&self.server_conf).await; + if let Err(err) = new_stream.as_ref() { + *self.uid_store.is_online.lock().unwrap() = (Instant::now(), Err(err.clone())); + } else { + *self.uid_store.is_online.lock().unwrap() = (Instant::now(), Ok(())); + } + let (capabilities, stream) = new_stream?; + self.stream = Ok(stream); + *self.uid_store.capabilities.lock().unwrap() = capabilities; + Ok(()) + }) + } + + pub fn read_response<'a>( + &'a mut self, + ret: &'a mut String, + is_multiline: bool, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + ret.clear(); + self.stream.as_mut()?.read_response(ret, is_multiline).await + }) + } + + pub async fn read_lines(&mut self, ret: &mut String, is_multiline: bool) -> Result<()> { + self.stream.as_mut()?.read_lines(ret, is_multiline).await?; + Ok(()) + } + + pub async fn send_command(&mut self, command: &[u8]) -> Result<()> { + if let Err(err) = + try_await(async { self.stream.as_mut()?.send_command(command).await }).await + { + self.stream = Err(err.clone()); + debug!(err.kind); + if err.kind.is_network() { + debug!(self.connect().await)?; + } + Err(err) + } else { + Ok(()) + } + } + + pub fn add_refresh_event(&mut self, ev: crate::backends::RefreshEvent) { + if let Some(ref sender) = self.uid_store.sender.read().unwrap().as_ref() { + sender.send(ev); + for ev in self.uid_store.refresh_events.lock().unwrap().drain(..) { + sender.send(ev); + } + } else { + self.uid_store.refresh_events.lock().unwrap().push(ev); + } + } +} diff --git a/melib/src/backends/nntp/mailbox.rs b/melib/src/backends/nntp/mailbox.rs new file mode 100644 index 00000000..62c17017 --- /dev/null +++ b/melib/src/backends/nntp/mailbox.rs @@ -0,0 +1,98 @@ +/* + * meli - nntp module. + * + * Copyright 2019 Manos Pitsidianakis + * + * This file is part of meli. + * + * meli is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * meli is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with meli. If not, see . + */ +use crate::backends::imap::LazyCountSet; +use crate::backends::{ + BackendMailbox, Mailbox, MailboxHash, MailboxPermissions, SpecialUsageMailbox, +}; +use crate::error::*; +use std::sync::{Arc, Mutex}; + +#[derive(Debug, Default, Clone)] +pub struct NntpMailbox { + pub(super) hash: MailboxHash, + pub(super) nntp_path: String, + + pub high_watermark: Arc>, + pub low_watermark: Arc>, + + pub exists: Arc>, + pub unseen: Arc>, +} + +impl NntpMailbox { + pub fn nntp_path(&self) -> &str { + &self.nntp_path + } +} + +impl BackendMailbox for NntpMailbox { + fn hash(&self) -> MailboxHash { + self.hash + } + + fn name(&self) -> &str { + &self.nntp_path + } + + fn path(&self) -> &str { + &self.nntp_path + } + + fn change_name(&mut self, s: &str) { + self.nntp_path = s.to_string(); + } + + fn children(&self) -> &[MailboxHash] { + &[] + } + + fn clone(&self) -> Mailbox { + Box::new(std::clone::Clone::clone(self)) + } + + fn special_usage(&self) -> SpecialUsageMailbox { + SpecialUsageMailbox::default() + } + + fn parent(&self) -> Option { + None + } + + fn permissions(&self) -> MailboxPermissions { + MailboxPermissions::default() + } + + fn is_subscribed(&self) -> bool { + true + } + + fn set_is_subscribed(&mut self, _new_val: bool) -> Result<()> { + Err(MeliError::new("Cannot set subscription in NNTP.")) + } + + fn set_special_usage(&mut self, _new_val: SpecialUsageMailbox) -> Result<()> { + Err(MeliError::new("Cannot set special usage in NNTP.")) + } + + fn count(&self) -> Result<(usize, usize)> { + Ok((self.unseen.lock()?.len(), self.exists.lock()?.len())) + } +} diff --git a/melib/src/backends/nntp/operations.rs b/melib/src/backends/nntp/operations.rs new file mode 100644 index 00000000..0dc60d22 --- /dev/null +++ b/melib/src/backends/nntp/operations.rs @@ -0,0 +1,93 @@ +/* + * meli - nntp module. + * + * Copyright 2017 - 2019 Manos Pitsidianakis + * + * This file is part of meli. + * + * meli is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * meli is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with meli. If not, see . + */ + +use super::*; + +use crate::backends::*; +use crate::email::*; +use crate::error::MeliError; +use std::sync::Arc; + +/// `BackendOp` implementor for Nntp +#[derive(Debug, Clone)] +pub struct NntpOp { + uid: usize, + mailbox_hash: MailboxHash, + connection: Arc>, + uid_store: Arc, +} + +impl NntpOp { + pub fn new( + uid: usize, + mailbox_hash: MailboxHash, + connection: Arc>, + uid_store: Arc, + ) -> Self { + NntpOp { + uid, + connection, + mailbox_hash, + uid_store, + } + } +} + +impl BackendOp for NntpOp { + fn as_bytes(&mut self) -> ResultFuture> { + let mailbox_hash = self.mailbox_hash; + let uid = self.uid; + let uid_store = self.uid_store.clone(); + let connection = self.connection.clone(); + Ok(Box::pin(async move { + let mut res = String::with_capacity(8 * 1024); + let mut conn = connection.lock().await; + let path = uid_store.mailboxes.lock().await[&mailbox_hash] + .name() + .to_string(); + conn.send_command(format!("GROUP {}", path).as_bytes()) + .await?; + conn.read_response(&mut res, false).await?; + if !res.starts_with("211 ") { + return Err(MeliError::new(format!( + "{} Could not select newsgroup {}: expected GROUP response but got: {}", + &uid_store.account_name, path, res + ))); + } + conn.send_command(format!("ARTICLE {}", uid).as_bytes()) + .await?; + conn.read_response(&mut res, true).await?; + if !res.starts_with("220 ") { + return Err(MeliError::new(format!( + "{} Could not select article {}: expected ARTICLE response but got: {}", + &uid_store.account_name, path, res + ))); + } + let pos = res.find("\r\n").unwrap_or(0) + 2; + + Ok(res.as_bytes()[pos..].to_vec()) + })) + } + + fn fetch_flags(&self) -> ResultFuture { + Ok(Box::pin(async move { Ok(Flag::default()) })) + } +} diff --git a/melib/src/backends/nntp/protocol_parser.rs b/melib/src/backends/nntp/protocol_parser.rs new file mode 100644 index 00000000..63df5f95 --- /dev/null +++ b/melib/src/backends/nntp/protocol_parser.rs @@ -0,0 +1,163 @@ +/* + * meli - melib crate. + * + * Copyright 2017-2020 Manos Pitsidianakis + * + * This file is part of meli. + * + * meli is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * meli is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with meli. If not, see . + */ + +use super::*; +use crate::email::parser::IResult; +use nom::{ + bytes::complete::{is_not, tag}, + combinator::opt, +}; +use std::str::FromStr; + +pub struct NntpLineIterator<'a> { + slice: &'a str, +} + +impl<'a> std::iter::DoubleEndedIterator for NntpLineIterator<'a> { + fn next_back(&mut self) -> Option { + if self.slice.is_empty() { + None + } else if let Some(pos) = self.slice.rfind("\r\n") { + if self.slice[..pos].is_empty() { + self.slice = &self.slice[..pos]; + None + } else if let Some(prev_pos) = self.slice[..pos].rfind("\r\n") { + let ret = &self.slice[prev_pos + 2..pos + 2]; + self.slice = &self.slice[..prev_pos + 2]; + Some(ret) + } else { + let ret = self.slice; + self.slice = &self.slice[ret.len()..]; + Some(ret) + } + } else { + let ret = self.slice; + self.slice = &self.slice[ret.len()..]; + Some(ret) + } + } +} + +impl<'a> Iterator for NntpLineIterator<'a> { + type Item = &'a str; + + fn next(&mut self) -> Option<&'a str> { + if self.slice.is_empty() { + None + } else if let Some(pos) = self.slice.find("\r\n") { + let ret = &self.slice[..pos + 2]; + self.slice = &self.slice[pos + 2..]; + Some(ret) + } else { + let ret = self.slice; + self.slice = &self.slice[ret.len()..]; + Some(ret) + } + } +} + +pub trait NntpLineSplit { + fn split_rn(&self) -> NntpLineIterator; +} + +impl NntpLineSplit for str { + fn split_rn(&self) -> NntpLineIterator { + NntpLineIterator { slice: self } + } +} + +pub fn over_article(input: &str) -> IResult<&str, (UID, Envelope)> { + /* + "0" or article number (see below) + Subject header content + From header content + Date header content + Message-ID header content + References header content + :bytes metadata item + :lines metadata item + */ + let (input, num) = is_not("\t")(input)?; + let (input, _) = tag("\t")(input)?; + let (input, subject) = opt(is_not("\t"))(input)?; + let (input, _) = tag("\t")(input)?; + let (input, from) = opt(is_not("\t"))(input)?; + let (input, _) = tag("\t")(input)?; + let (input, date) = opt(is_not("\t"))(input)?; + let (input, _) = tag("\t")(input)?; + let (input, message_id) = opt(is_not("\t"))(input)?; + let (input, _) = tag("\t")(input)?; + let (input, references) = opt(is_not("\t"))(input)?; + let (input, _) = tag("\t")(input)?; + let (input, _bytes) = opt(is_not("\t"))(input)?; + let (input, _) = tag("\t")(input)?; + let (input, _lines) = opt(is_not("\t\r\n"))(input)?; + let (input, _other_headers) = opt(is_not("\r\n"))(input)?; + let (input, _) = tag("\r\n")(input)?; + Ok(( + input, + ({ + let env_hash = { + let mut hasher = DefaultHasher::new(); + hasher.write(num.as_bytes()); + hasher.write(message_id.unwrap_or_default().as_bytes()); + hasher.finish() + }; + let mut env = Envelope::new(env_hash); + if let Some(date) = date { + env.set_date(date.as_bytes()); + if let Ok(d) = crate::email::parser::generic::date(env.date_as_str().as_bytes()) { + env.set_datetime(d); + } + } + + if let Some(subject) = subject { + env.set_subject(subject.into()); + } + + if let Some(from) = from { + if let Ok((_, from)) = + crate::email::parser::address::rfc2822address_list(from.as_bytes()) + { + env.set_from(from); + } + } + + if let Some(references) = references { + { + if let Ok((_, r)) = + crate::email::parser::address::references(references.as_bytes()) + { + for v in r { + env.push_references(v); + } + } + } + env.set_references(references.as_bytes()); + } + + if let Some(message_id) = message_id { + env.set_message_id(message_id.as_bytes()); + } + (usize::from_str(num).unwrap(), env) + }), + )) +}