melib: add experimental NNTP backend

Closes #54

melib/src/backends.rs

@@ -33,6 +33,8 @@ macro_rules! tag_hash {
 
 #[cfg(feature = "imap_backend")]
 pub mod imap;
+#[cfg(feature = "imap_backend")]
+pub mod nntp;
 #[cfg(feature = "notmuch_backend")]
 pub mod notmuch;
 #[cfg(feature = "notmuch_backend")]
@@ -47,6 +49,7 @@ pub mod mbox;
 pub use self::imap::ImapType;
 #[cfg(feature = "jmap_backend")]
 pub use self::jmap::JmapType;
+pub use self::nntp::NntpType;
 use crate::async_workers::*;
 use crate::conf::AccountSettings;
 use crate::error::{MeliError, Result};
@@ -144,6 +147,13 @@ impl Backends {
                     validate_conf_fn: Box::new(imap::ImapType::validate_config),
                 },
             );
+            b.register(
+                "nntp".to_string(),
+                Backend {
+                    create_fn: Box::new(|| Box::new(|f, i| nntp::NntpType::new(f, i))),
+                    validate_conf_fn: Box::new(nntp::NntpType::validate_config),
+                },
+            );
         }
         #[cfg(feature = "notmuch_backend")]
         {

melib/src/backends/nntp.rs

@@ -0,0 +1,568 @@
+/*
+ * meli - nntp module.
+ *
+ * Copyright 2019 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use crate::get_conf_val;
+use crate::get_path_hash;
+use smallvec::SmallVec;
+#[macro_use]
+mod protocol_parser;
+pub use protocol_parser::*;
+mod mailbox;
+pub use mailbox::*;
+mod operations;
+pub use operations::*;
+mod connection;
+pub use connection::*;
+
+use crate::async_workers::{Async, WorkContext};
+use crate::backends::*;
+use crate::conf::AccountSettings;
+use crate::email::*;
+use crate::error::{MeliError, Result};
+use futures::lock::Mutex as FutureMutex;
+use futures::stream::Stream;
+use std::collections::{hash_map::DefaultHasher, BTreeMap};
+use std::collections::{HashMap, HashSet};
+use std::future::Future;
+use std::hash::Hasher;
+use std::pin::Pin;
+use std::str::FromStr;
+use std::sync::{Arc, Mutex, RwLock};
+use std::time::Instant;
+pub type UID = usize;
+
+pub static SUPPORTED_CAPABILITIES: &[&str] = &[
+    #[cfg(feature = "deflate_compression")]
+    "COMPRESS DEFLATE",
+    "VERSION 2",
+];
+
+#[derive(Debug, Clone)]
+pub struct NntpServerConf {
+    pub server_hostname: String,
+    pub server_username: String,
+    //pub server_password: String,
+    pub server_port: u16,
+    pub use_starttls: bool,
+    pub use_tls: bool,
+    pub danger_accept_invalid_certs: bool,
+    pub extension_use: NntpExtensionUse,
+}
+
+pub struct IsSubscribedFn(Box<dyn Fn(&str) -> bool + Send + Sync>);
+
+impl std::fmt::Debug for IsSubscribedFn {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "IsSubscribedFn Box")
+    }
+}
+
+impl std::ops::Deref for IsSubscribedFn {
+    type Target = Box<dyn Fn(&str) -> bool + Send + Sync>;
+    fn deref(&self) -> &Box<dyn Fn(&str) -> bool + Send + Sync> {
+        &self.0
+    }
+}
+type Capabilities = HashSet<Vec<u8>>;
+
+#[derive(Debug)]
+pub struct UIDStore {
+    account_hash: AccountHash,
+    account_name: Arc<String>,
+    offline_cache: bool,
+    capabilities: Arc<Mutex<Capabilities>>,
+    hash_index: Arc<Mutex<HashMap<EnvelopeHash, (UID, MailboxHash)>>>,
+    uid_index: Arc<Mutex<HashMap<(MailboxHash, UID), EnvelopeHash>>>,
+
+    mailboxes: Arc<FutureMutex<HashMap<MailboxHash, NntpMailbox>>>,
+    is_online: Arc<Mutex<(Instant, Result<()>)>>,
+    refresh_events: Arc<Mutex<Vec<RefreshEvent>>>,
+    sender: Arc<RwLock<Option<RefreshEventConsumer>>>,
+}
+
+impl Default for UIDStore {
+    fn default() -> Self {
+        UIDStore {
+            account_hash: 0,
+            account_name: Arc::new(String::new()),
+            offline_cache: false,
+            capabilities: Default::default(),
+            hash_index: Default::default(),
+            uid_index: Default::default(),
+            mailboxes: Arc::new(FutureMutex::new(Default::default())),
+            is_online: Arc::new(Mutex::new((
+                Instant::now(),
+                Err(MeliError::new("Account is uninitialised.")),
+            ))),
+            refresh_events: Default::default(),
+            sender: Arc::new(RwLock::new(None)),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct NntpType {
+    is_subscribed: Arc<IsSubscribedFn>,
+    connection: Arc<FutureMutex<NntpConnection>>,
+    server_conf: NntpServerConf,
+    uid_store: Arc<UIDStore>,
+    can_create_flags: Arc<Mutex<bool>>,
+}
+
+impl MailBackend for NntpType {
+    fn capabilities(&self) -> MailBackendCapabilities {
+        const CAPABILITIES: MailBackendCapabilities = MailBackendCapabilities {
+            is_async: true,
+            is_remote: true,
+            supports_search: false,
+            supports_tags: false,
+        };
+        CAPABILITIES
+    }
+
+    fn fetch_async(
+        &mut self,
+        mailbox_hash: MailboxHash,
+    ) -> Result<Pin<Box<dyn Stream<Item = Result<Vec<Envelope>>> + Send + 'static>>> {
+        let uid_store = self.uid_store.clone();
+        let connection = self.connection.clone();
+        Ok(Box::pin(async_stream::try_stream! {
+            {
+                let f = &uid_store.mailboxes.lock().await[&mailbox_hash];
+                f.exists.lock().unwrap().clear();
+                f.unseen.lock().unwrap().clear();
+            };
+            let ret = fetch_envs(mailbox_hash, connection, &uid_store).await?;
+            yield ret;
+        }))
+    }
+
+    fn refresh_async(
+        &mut self,
+        _mailbox_hash: MailboxHash,
+        _sender: RefreshEventConsumer,
+    ) -> ResultFuture<()> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn mailboxes_async(&self) -> ResultFuture<HashMap<MailboxHash, Mailbox>> {
+        let uid_store = self.uid_store.clone();
+        let connection = self.connection.clone();
+        Ok(Box::pin(async move {
+            NntpType::nntp_mailboxes(&connection).await?;
+            let mailboxes_lck = uid_store.mailboxes.lock().await;
+            let ret = mailboxes_lck
+                .iter()
+                .map(|(h, f)| (*h, Box::new(Clone::clone(f)) as Mailbox))
+                .collect();
+            Ok(ret)
+        }))
+    }
+
+    fn is_online_async(&self) -> ResultFuture<()> {
+        let connection = self.connection.clone();
+        Ok(Box::pin(async move {
+            match timeout(std::time::Duration::from_secs(3), connection.lock()).await {
+                Ok(mut conn) => {
+                    debug!("is_online_async");
+                    match debug!(timeout(std::time::Duration::from_secs(3), conn.connect()).await) {
+                        Ok(Ok(())) => Ok(()),
+                        Err(err) | Ok(Err(err)) => {
+                            conn.stream = Err(err.clone());
+                            debug!(conn.connect().await)
+                        }
+                    }
+                }
+                Err(err) => Err(err),
+            }
+        }))
+    }
+
+    fn fetch(&mut self, _mailbox_hash: MailboxHash) -> Result<Async<Result<Vec<Envelope>>>> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn refresh(
+        &mut self,
+        _mailbox_hash: MailboxHash,
+        _sender: RefreshEventConsumer,
+    ) -> Result<Async<()>> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn watch(
+        &self,
+        _sender: RefreshEventConsumer,
+        _work_context: WorkContext,
+    ) -> Result<std::thread::ThreadId> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn watch_async(&self, _sender: RefreshEventConsumer) -> ResultFuture<()> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn mailboxes(&self) -> Result<HashMap<MailboxHash, Mailbox>> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn operation(&self, env_hash: EnvelopeHash) -> Result<Box<dyn BackendOp>> {
+        let (uid, mailbox_hash) = if let Some(v) =
+            self.uid_store.hash_index.lock().unwrap().get(&env_hash)
+        {
+            *v
+        } else {
+            return Err(MeliError::new(
+                    "Message not found in local cache, it might have been deleted before you requested it."
+                ));
+        };
+        Ok(Box::new(NntpOp::new(
+            uid,
+            mailbox_hash,
+            self.connection.clone(),
+            self.uid_store.clone(),
+        )))
+    }
+
+    fn save(
+        &self,
+        _bytes: Vec<u8>,
+        _mailbox_hash: MailboxHash,
+        _flags: Option<Flag>,
+    ) -> ResultFuture<()> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn copy_messages(
+        &mut self,
+        _env_hashes: EnvelopeHashBatch,
+        _source_mailbox_hash: MailboxHash,
+        _destination_mailbox_hash: MailboxHash,
+        _move_: bool,
+        _destination_flags: Option<Flag>,
+    ) -> ResultFuture<()> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn set_flags(
+        &mut self,
+        _env_hashes: EnvelopeHashBatch,
+        _mailbox_hash: MailboxHash,
+        _flags: SmallVec<[(std::result::Result<Flag, String>, bool); 8]>,
+    ) -> ResultFuture<()> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn as_any(&self) -> &dyn ::std::any::Any {
+        self
+    }
+
+    fn as_any_mut(&mut self) -> &mut dyn ::std::any::Any {
+        self
+    }
+
+    fn tags(&self) -> Option<Arc<RwLock<BTreeMap<u64, String>>>> {
+        None
+    }
+
+    fn create_mailbox(
+        &mut self,
+        _path: String,
+    ) -> ResultFuture<(MailboxHash, HashMap<MailboxHash, Mailbox>)> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn delete_mailbox(
+        &mut self,
+        _mailbox_hash: MailboxHash,
+    ) -> ResultFuture<HashMap<MailboxHash, Mailbox>> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn set_mailbox_subscription(
+        &mut self,
+        _mailbox_hash: MailboxHash,
+        _new_val: bool,
+    ) -> ResultFuture<()> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn rename_mailbox(
+        &mut self,
+        _mailbox_hash: MailboxHash,
+        _new_path: String,
+    ) -> ResultFuture<Mailbox> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn set_mailbox_permissions(
+        &mut self,
+        _mailbox_hash: MailboxHash,
+        _val: crate::backends::MailboxPermissions,
+    ) -> ResultFuture<()> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn search(
+        &self,
+        _query: crate::search::Query,
+        _mailbox_hash: Option<MailboxHash>,
+    ) -> ResultFuture<SmallVec<[EnvelopeHash; 512]>> {
+        Err(MeliError::new("Unimplemented."))
+    }
+}
+
+impl NntpType {
+    pub fn new(
+        s: &AccountSettings,
+        is_subscribed: Box<dyn Fn(&str) -> bool + Send + Sync>,
+    ) -> Result<Box<dyn MailBackend>> {
+        let server_hostname = get_conf_val!(s["server_hostname"])?;
+        /*let server_username = get_conf_val!(s["server_username"], "")?;
+        let server_password = if !s.extra.contains_key("server_password_command") {
+            get_conf_val!(s["server_password"], "")?.to_string()
+        } else {
+            let invocation = get_conf_val!(s["server_password_command"])?;
+            let output = std::process::Command::new("sh")
+                .args(&["-c", invocation])
+                .stdin(std::process::Stdio::piped())
+                .stdout(std::process::Stdio::piped())
+                .stderr(std::process::Stdio::piped())
+                .output()?;
+            if !output.status.success() {
+                return Err(MeliError::new(format!(
+                    "({}) server_password_command `{}` returned {}: {}",
+                    s.name,
+                    get_conf_val!(s["server_password_command"])?,
+                    output.status,
+                    String::from_utf8_lossy(&output.stderr)
+                )));
+            }
+            std::str::from_utf8(&output.stdout)?.trim_end().to_string()
+        };
+        */
+        let server_port = get_conf_val!(s["server_port"], 119)?;
+        let use_tls = get_conf_val!(s["use_tls"], true)?;
+        let use_starttls = use_tls && get_conf_val!(s["use_starttls"], !(server_port == 993))?;
+        let danger_accept_invalid_certs: bool =
+            get_conf_val!(s["danger_accept_invalid_certs"], false)?;
+        let server_conf = NntpServerConf {
+            server_hostname: server_hostname.to_string(),
+            server_username: String::new(),
+            //server_password,
+            server_port,
+            use_tls,
+            use_starttls,
+            danger_accept_invalid_certs,
+            extension_use: NntpExtensionUse::default(),
+        };
+        let account_hash = {
+            let mut hasher = DefaultHasher::new();
+            hasher.write(s.name.as_bytes());
+            hasher.finish()
+        };
+        let account_name = Arc::new(s.name().to_string());
+        let mut mailboxes = HashMap::default();
+        for (k, _f) in s.mailboxes.iter() {
+            let mailbox_hash = get_path_hash!(&k);
+            mailboxes.insert(
+                mailbox_hash,
+                NntpMailbox {
+                    hash: mailbox_hash,
+                    nntp_path: k.to_string(),
+                    high_watermark: Arc::new(Mutex::new(0)),
+                    low_watermark: Arc::new(Mutex::new(0)),
+                    exists: Default::default(),
+                    unseen: Default::default(),
+                },
+            );
+        }
+        if mailboxes.is_empty() {
+            return Err(MeliError::new(format!(
+                "{} has no newsgroups configured.",
+                account_name
+            )));
+        }
+        let uid_store: Arc<UIDStore> = Arc::new(UIDStore {
+            account_hash,
+            account_name,
+            offline_cache: get_conf_val!(s["X_header_caching"], false)?,
+            mailboxes: Arc::new(FutureMutex::new(mailboxes)),
+            ..UIDStore::default()
+        });
+        let connection = NntpConnection::new_connection(&server_conf, uid_store.clone());
+
+        Ok(Box::new(NntpType {
+            server_conf,
+            is_subscribed: Arc::new(IsSubscribedFn(is_subscribed)),
+            can_create_flags: Arc::new(Mutex::new(false)),
+            connection: Arc::new(FutureMutex::new(connection)),
+            uid_store,
+        }))
+    }
+
+    pub async fn nntp_mailboxes(connection: &Arc<FutureMutex<NntpConnection>>) -> Result<()> {
+        let mut res = String::with_capacity(8 * 1024);
+        let mut conn = connection.lock().await;
+        let command = {
+            let mailboxes_lck = conn.uid_store.mailboxes.lock().await;
+            mailboxes_lck
+                .values()
+                .fold("LIST ACTIVE ".to_string(), |mut acc, x| {
+                    if acc.len() != "LIST ACTIVE ".len() {
+                        acc.push(',');
+                    }
+                    acc.push_str(x.name());
+                    acc
+                })
+        };
+        conn.send_command(command.as_bytes()).await?;
+        conn.read_response(&mut res, true).await?;
+        if !res.starts_with("215 ") {
+            return Err(MeliError::new(format!(
+                "Could not get newsgroups {}: expected LIST ACTIVE response but got: {}",
+                &conn.uid_store.account_name, res
+            )));
+        }
+        debug!(&res);
+        let mut mailboxes_lck = conn.uid_store.mailboxes.lock().await;
+        for l in res.split_rn().skip(1) {
+            let s = l.split_whitespace().collect::<SmallVec<[&str; 4]>>();
+            if s.len() != 3 {
+                continue;
+            }
+            let mailbox_hash = get_path_hash!(&s[0]);
+            mailboxes_lck.entry(mailbox_hash).and_modify(|m| {
+                *m.high_watermark.lock().unwrap() = usize::from_str(s[1]).unwrap_or(0);
+                *m.low_watermark.lock().unwrap() = usize::from_str(s[2]).unwrap_or(0);
+            });
+        }
+        Ok(())
+    }
+
+    pub fn validate_config(s: &AccountSettings) -> Result<()> {
+        get_conf_val!(s["server_hostname"])?;
+        get_conf_val!(s["server_username"], String::new())?;
+        if !s.extra.contains_key("server_password_command") {
+            get_conf_val!(s["server_password"], String::new())?;
+        } else if s.extra.contains_key("server_password") {
+            return Err(MeliError::new(format!(
+                "Configuration error ({}): both server_password and server_password_command are set, cannot choose",
+                s.name.as_str(),
+            )));
+        }
+        let server_port = get_conf_val!(s["server_port"], 119)?;
+        let use_tls = get_conf_val!(s["use_tls"], true)?;
+        let use_starttls = get_conf_val!(s["use_starttls"], !(server_port == 993))?;
+        if !use_tls && use_starttls {
+            return Err(MeliError::new(format!(
+                "Configuration error ({}): incompatible use_tls and use_starttls values: use_tls = false, use_starttls = true",
+                s.name.as_str(),
+            )));
+        }
+        get_conf_val!(s["danger_accept_invalid_certs"], false)?;
+        Ok(())
+    }
+
+    pub fn capabilities(&self) -> Vec<String> {
+        self.uid_store
+            .capabilities
+            .lock()
+            .unwrap()
+            .iter()
+            .map(|c| String::from_utf8_lossy(c).into())
+            .collect::<Vec<String>>()
+    }
+}
+
+async fn fetch_envs(
+    mailbox_hash: MailboxHash,
+    connection: Arc<FutureMutex<NntpConnection>>,
+    uid_store: &UIDStore,
+) -> Result<Vec<Envelope>> {
+    let mut res = String::with_capacity(8 * 1024);
+    let mut conn = connection.lock().await;
+    let path = uid_store.mailboxes.lock().await[&mailbox_hash]
+        .name()
+        .to_string();
+    conn.send_command(format!("GROUP {}", path).as_bytes())
+        .await?;
+    conn.read_response(&mut res, false).await?;
+    if !res.starts_with("211 ") {
+        return Err(MeliError::new(format!(
+            "{} Could not select newsgroup {}: expected GROUP response but got: {}",
+            &uid_store.account_name, path, res
+        )));
+    }
+    /*
+     *   Parameters
+             group     Name of newsgroup
+             number    Estimated number of articles in the group
+             low       Reported low water mark
+             high      Reported high water mark
+    */
+    let s = res.split_whitespace().collect::<SmallVec<[&str; 6]>>();
+    if s.len() != 5 {
+        return Err(MeliError::new(format!(
+            "{} Could not select newsgroup {}: expected GROUP response but got: {}",
+            &uid_store.account_name, path, res
+        )));
+    }
+    let total = usize::from_str(&s[1]).unwrap_or(0);
+    let _low = usize::from_str(&s[2]).unwrap_or(0);
+    let high = usize::from_str(&s[3]).unwrap_or(0);
+    drop(s);
+
+    conn.send_command(format!("OVER {}-{}", high.saturating_sub(250), high).as_bytes())
+        .await?;
+    conn.read_response(&mut res, true).await?;
+    if !res.starts_with("224 ") {
+        return Err(MeliError::new(format!(
+            "{} Could not select newsgroup {}: expected OVER response but got: {}",
+            &uid_store.account_name, path, res
+        )));
+    }
+    let mut ret = Vec::with_capacity(total);
+    //hash_index: Arc<Mutex<HashMap<EnvelopeHash, (UID, MailboxHash)>>>,
+    //uid_index: Arc<Mutex<HashMap<(MailboxHash, UID), EnvelopeHash>>>,
+    let mut hash_index_lck = uid_store.hash_index.lock().unwrap();
+    let mut uid_index_lck = uid_store.uid_index.lock().unwrap();
+    for l in res.split_rn().skip(1) {
+        debug!(&l);
+        let (_, (num, env)) = debug!(protocol_parser::over_article(&l))?;
+        hash_index_lck.insert(env.hash(), (num, mailbox_hash));
+        uid_index_lck.insert((mailbox_hash, num), env.hash());
+        ret.push(env);
+    }
+    Ok(ret)
+}
+
+use futures::future::{self, Either};
+
+async fn timeout<O>(dur: std::time::Duration, f: impl Future<Output = O>) -> Result<O> {
+    futures::pin_mut!(f);
+    match future::select(f, smol::Timer::after(dur)).await {
+        Either::Left((out, _)) => Ok(out),
+        Either::Right(_) => {
+            Err(MeliError::new("Timedout").set_kind(crate::error::ErrorKind::Network))
+        }
+    }
+}

melib/src/backends/nntp/connection.rs

@@ -0,0 +1,376 @@
+/*
+ * meli - nntp module.
+ *
+ * Copyright 2017 - 2019 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use crate::backends::MailboxHash;
+use crate::connections::{lookup_ipv4, Connection};
+use crate::email::parser::BytesExt;
+use crate::error::*;
+extern crate native_tls;
+use futures::io::{AsyncReadExt, AsyncWriteExt};
+use native_tls::TlsConnector;
+pub use smol::Async as AsyncWrapper;
+use std::collections::HashSet;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::time::Instant;
+
+use super::{Capabilities, NntpServerConf, UIDStore};
+
+#[derive(Debug, Clone, Copy)]
+pub struct NntpExtensionUse {
+    #[cfg(feature = "deflate_compression")]
+    pub deflate: bool,
+}
+
+impl Default for NntpExtensionUse {
+    fn default() -> Self {
+        Self {
+            #[cfg(feature = "deflate_compression")]
+            deflate: true,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct NntpStream {
+    pub stream: AsyncWrapper<Connection>,
+    pub extension_use: NntpExtensionUse,
+    pub current_mailbox: MailboxSelection,
+}
+
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+pub enum MailboxSelection {
+    None,
+    Select(MailboxHash),
+    Examine(MailboxHash),
+}
+
+impl MailboxSelection {
+    pub fn take(&mut self) -> Self {
+        std::mem::replace(self, MailboxSelection::None)
+    }
+}
+
+async fn try_await(cl: impl Future<Output = Result<()>> + Send) -> Result<()> {
+    cl.await
+}
+
+#[derive(Debug)]
+pub struct NntpConnection {
+    pub stream: Result<NntpStream>,
+    pub server_conf: NntpServerConf,
+    pub uid_store: Arc<UIDStore>,
+}
+
+impl NntpStream {
+    pub async fn new_connection(
+        server_conf: &NntpServerConf,
+    ) -> Result<(Capabilities, NntpStream)> {
+        use std::net::TcpStream;
+        let path = &server_conf.server_hostname;
+
+        let stream = {
+            let addr = lookup_ipv4(path, server_conf.server_port)?;
+            AsyncWrapper::new(Connection::Tcp(
+                TcpStream::connect_timeout(&addr, std::time::Duration::new(4, 0))
+                    .chain_err_kind(crate::error::ErrorKind::Network)?,
+            ))
+            .chain_err_kind(crate::error::ErrorKind::Network)?
+        };
+        let mut res = String::with_capacity(8 * 1024);
+        let mut ret = NntpStream {
+            stream,
+            extension_use: NntpExtensionUse::default(),
+            current_mailbox: MailboxSelection::None,
+        };
+
+        ret.read_response(&mut res, false).await?;
+        ret.send_command(b"CAPABILITIES").await?;
+        ret.read_response(&mut res, true).await?;
+        if !res.starts_with("101 ") {
+            return Err(MeliError::new(format!(
+                "Could not connect to {}: expected CAPABILITIES response but got:{}",
+                &server_conf.server_hostname, res
+            )));
+        }
+        let capabilities: Vec<&str> = res.lines().skip(1).collect();
+
+        if !capabilities
+            .iter()
+            .any(|cap| cap.eq_ignore_ascii_case("VERSION 2"))
+        {
+            return Err(MeliError::new(format!(
+                "Could not connect to {}: server is not NNTP compliant",
+                &server_conf.server_hostname
+            )));
+        }
+
+        if server_conf.use_tls {
+            let mut connector = TlsConnector::builder();
+            if server_conf.danger_accept_invalid_certs {
+                connector.danger_accept_invalid_certs(true);
+            }
+            let connector = connector
+                .build()
+                .chain_err_kind(crate::error::ErrorKind::Network)?;
+
+            if server_conf.use_starttls {
+                ret.stream
+                    .write_all(b"STARTTLS\r\n")
+                    .await
+                    .chain_err_kind(crate::error::ErrorKind::Network)?;
+                ret.read_response(&mut res, false).await?;
+                if !res.starts_with("382 ") {
+                    return Err(MeliError::new(format!(
+                        "Could not connect to {}: could not begin TLS negotiation, got: {}",
+                        &server_conf.server_hostname, res
+                    )));
+                }
+            }
+
+            {
+                // FIXME: This is blocking
+                let socket = ret
+                    .stream
+                    .into_inner()
+                    .chain_err_kind(crate::error::ErrorKind::Network)?;
+                let mut conn_result = connector.connect(path, socket);
+                if let Err(native_tls::HandshakeError::WouldBlock(midhandshake_stream)) =
+                    conn_result
+                {
+                    let mut midhandshake_stream = Some(midhandshake_stream);
+                    loop {
+                        match midhandshake_stream.take().unwrap().handshake() {
+                            Ok(r) => {
+                                conn_result = Ok(r);
+                                break;
+                            }
+                            Err(native_tls::HandshakeError::WouldBlock(stream)) => {
+                                midhandshake_stream = Some(stream);
+                            }
+                            p => {
+                                p.chain_err_kind(crate::error::ErrorKind::Network)?;
+                            }
+                        }
+                    }
+                }
+                ret.stream = AsyncWrapper::new(Connection::Tls(
+                    conn_result.chain_err_kind(crate::error::ErrorKind::Network)?,
+                ))
+                .chain_err_kind(crate::error::ErrorKind::Network)?;
+            }
+        }
+        //ret.send_command(
+        //    format!(
+        //        "LOGIN \"{}\" \"{}\"",
+        //        &server_conf.server_username, &server_conf.server_password
+        //    )
+        //    .as_bytes(),
+        //)
+        //.await?;
+
+        ret.send_command(b"CAPABILITIES").await?;
+        ret.read_response(&mut res, true).await?;
+        if !res.starts_with("101 ") {
+            return Err(MeliError::new(format!(
+                "Could not connect to {}: expected CAPABILITIES response but got:{}",
+                &server_conf.server_hostname, res
+            )));
+        }
+        let capabilities: HashSet<Vec<u8>> =
+            res.lines().skip(1).map(|l| l.as_bytes().to_vec()).collect();
+        #[cfg(feature = "deflate_compression")]
+        #[cfg(feature = "deflate_compression")]
+        if capabilities.contains(&b"COMPRESS DEFLATE"[..]) && ret.extension_use.deflate {
+            ret.send_command(b"COMPRESS DEFLATE").await?;
+            ret.read_response(&mut res, false).await?;
+            if !res.starts_with("206 ") {
+                crate::log(
+                    format!(
+                        "Could not use COMPRESS=DEFLATE in account `{}`: server replied with `{}`",
+                        server_conf.server_hostname, res
+                    ),
+                    crate::LoggingLevel::WARN,
+                );
+            } else {
+                let NntpStream {
+                    stream,
+                    extension_use,
+                    current_mailbox,
+                } = ret;
+                let stream = stream.into_inner()?;
+                return Ok((
+                    capabilities,
+                    NntpStream {
+                        stream: AsyncWrapper::new(stream.deflate())?,
+                        extension_use,
+                        current_mailbox,
+                    },
+                ));
+            }
+        }
+
+        Ok((capabilities, ret))
+    }
+
+    pub async fn read_response(&mut self, ret: &mut String, is_multiline: bool) -> Result<()> {
+        self.read_lines(ret, is_multiline).await?;
+        Ok(())
+    }
+
+    pub async fn read_lines(&mut self, ret: &mut String, is_multiline: bool) -> Result<()> {
+        let mut buf: Vec<u8> = vec![0; Connection::IO_BUF_SIZE];
+        ret.clear();
+        let mut last_line_idx: usize = 0;
+        loop {
+            match self.stream.read(&mut buf).await {
+                Ok(0) => break,
+                Ok(b) => {
+                    ret.push_str(unsafe { std::str::from_utf8_unchecked(&buf[0..b]) });
+                    if let Some(mut pos) = ret[last_line_idx..].rfind("\r\n") {
+                        if !is_multiline {
+                            break;
+                        } else if let Some(pos) = ret.find("\r\n.\r\n") {
+                            ret.replace_range(pos + "\r\n".len()..pos + "\r\n.\r\n".len(), "");
+                            break;
+                        }
+                        if ret[last_line_idx..].starts_with("205 ") {
+                            return Err(MeliError::new(format!("Disconnected: {}", ret)));
+                        }
+                        if let Some(prev_line) =
+                            ret[last_line_idx..pos + last_line_idx].rfind("\r\n")
+                        {
+                            last_line_idx += prev_line + "\r\n".len();
+                            pos -= prev_line + "\r\n".len();
+                        }
+                        last_line_idx += pos + "\r\n".len();
+                    }
+                }
+                Err(e) => {
+                    return Err(MeliError::from(e).set_err_kind(crate::error::ErrorKind::Network));
+                }
+            }
+        }
+        //debug!("returning nntp response:\n{:?}", &ret);
+        Ok(())
+    }
+
+    pub async fn send_command(&mut self, command: &[u8]) -> Result<()> {
+        if let Err(err) = try_await(async move {
+            let command = command.trim();
+            self.stream.write_all(command).await?;
+            self.stream.write_all(b"\r\n").await?;
+            self.stream.flush().await?;
+            debug!("sent: {}", unsafe {
+                std::str::from_utf8_unchecked(command)
+            });
+            Ok(())
+        })
+        .await
+        {
+            debug!("stream send_command err {:?}", err);
+            Err(err.set_err_kind(crate::error::ErrorKind::Network))
+        } else {
+            Ok(())
+        }
+    }
+}
+
+impl NntpConnection {
+    pub fn new_connection(
+        server_conf: &NntpServerConf,
+        uid_store: Arc<UIDStore>,
+    ) -> NntpConnection {
+        NntpConnection {
+            stream: Err(MeliError::new("Offline".to_string())),
+            server_conf: server_conf.clone(),
+            uid_store,
+        }
+    }
+
+    pub fn connect<'a>(&'a mut self) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'a>> {
+        Box::pin(async move {
+            if let (instant, ref mut status @ Ok(())) = *self.uid_store.is_online.lock().unwrap() {
+                if Instant::now().duration_since(instant) >= std::time::Duration::new(60 * 30, 0) {
+                    *status = Err(MeliError::new("Connection timed out"));
+                    self.stream = Err(MeliError::new("Connection timed out"));
+                }
+            }
+            if self.stream.is_ok() {
+                self.uid_store.is_online.lock().unwrap().0 = Instant::now();
+                return Ok(());
+            }
+            let new_stream = NntpStream::new_connection(&self.server_conf).await;
+            if let Err(err) = new_stream.as_ref() {
+                *self.uid_store.is_online.lock().unwrap() = (Instant::now(), Err(err.clone()));
+            } else {
+                *self.uid_store.is_online.lock().unwrap() = (Instant::now(), Ok(()));
+            }
+            let (capabilities, stream) = new_stream?;
+            self.stream = Ok(stream);
+            *self.uid_store.capabilities.lock().unwrap() = capabilities;
+            Ok(())
+        })
+    }
+
+    pub fn read_response<'a>(
+        &'a mut self,
+        ret: &'a mut String,
+        is_multiline: bool,
+    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'a>> {
+        Box::pin(async move {
+            ret.clear();
+            self.stream.as_mut()?.read_response(ret, is_multiline).await
+        })
+    }
+
+    pub async fn read_lines(&mut self, ret: &mut String, is_multiline: bool) -> Result<()> {
+        self.stream.as_mut()?.read_lines(ret, is_multiline).await?;
+        Ok(())
+    }
+
+    pub async fn send_command(&mut self, command: &[u8]) -> Result<()> {
+        if let Err(err) =
+            try_await(async { self.stream.as_mut()?.send_command(command).await }).await
+        {
+            self.stream = Err(err.clone());
+            debug!(err.kind);
+            if err.kind.is_network() {
+                debug!(self.connect().await)?;
+            }
+            Err(err)
+        } else {
+            Ok(())
+        }
+    }
+
+    pub fn add_refresh_event(&mut self, ev: crate::backends::RefreshEvent) {
+        if let Some(ref sender) = self.uid_store.sender.read().unwrap().as_ref() {
+            sender.send(ev);
+            for ev in self.uid_store.refresh_events.lock().unwrap().drain(..) {
+                sender.send(ev);
+            }
+        } else {
+            self.uid_store.refresh_events.lock().unwrap().push(ev);
+        }
+    }
+}

melib/src/backends/nntp/mailbox.rs

@@ -0,0 +1,98 @@
+/*
+ * meli - nntp module.
+ *
+ * Copyright 2019 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+use crate::backends::imap::LazyCountSet;
+use crate::backends::{
+    BackendMailbox, Mailbox, MailboxHash, MailboxPermissions, SpecialUsageMailbox,
+};
+use crate::error::*;
+use std::sync::{Arc, Mutex};
+
+#[derive(Debug, Default, Clone)]
+pub struct NntpMailbox {
+    pub(super) hash: MailboxHash,
+    pub(super) nntp_path: String,
+
+    pub high_watermark: Arc<Mutex<usize>>,
+    pub low_watermark: Arc<Mutex<usize>>,
+
+    pub exists: Arc<Mutex<LazyCountSet>>,
+    pub unseen: Arc<Mutex<LazyCountSet>>,
+}
+
+impl NntpMailbox {
+    pub fn nntp_path(&self) -> &str {
+        &self.nntp_path
+    }
+}
+
+impl BackendMailbox for NntpMailbox {
+    fn hash(&self) -> MailboxHash {
+        self.hash
+    }
+
+    fn name(&self) -> &str {
+        &self.nntp_path
+    }
+
+    fn path(&self) -> &str {
+        &self.nntp_path
+    }
+
+    fn change_name(&mut self, s: &str) {
+        self.nntp_path = s.to_string();
+    }
+
+    fn children(&self) -> &[MailboxHash] {
+        &[]
+    }
+
+    fn clone(&self) -> Mailbox {
+        Box::new(std::clone::Clone::clone(self))
+    }
+
+    fn special_usage(&self) -> SpecialUsageMailbox {
+        SpecialUsageMailbox::default()
+    }
+
+    fn parent(&self) -> Option<MailboxHash> {
+        None
+    }
+
+    fn permissions(&self) -> MailboxPermissions {
+        MailboxPermissions::default()
+    }
+
+    fn is_subscribed(&self) -> bool {
+        true
+    }
+
+    fn set_is_subscribed(&mut self, _new_val: bool) -> Result<()> {
+        Err(MeliError::new("Cannot set subscription in NNTP."))
+    }
+
+    fn set_special_usage(&mut self, _new_val: SpecialUsageMailbox) -> Result<()> {
+        Err(MeliError::new("Cannot set special usage in NNTP."))
+    }
+
+    fn count(&self) -> Result<(usize, usize)> {
+        Ok((self.unseen.lock()?.len(), self.exists.lock()?.len()))
+    }
+}

melib/src/backends/nntp/operations.rs

@@ -0,0 +1,93 @@
+/*
+ * meli - nntp module.
+ *
+ * Copyright 2017 - 2019 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+
+use crate::backends::*;
+use crate::email::*;
+use crate::error::MeliError;
+use std::sync::Arc;
+
+/// `BackendOp` implementor for Nntp
+#[derive(Debug, Clone)]
+pub struct NntpOp {
+    uid: usize,
+    mailbox_hash: MailboxHash,
+    connection: Arc<FutureMutex<NntpConnection>>,
+    uid_store: Arc<UIDStore>,
+}
+
+impl NntpOp {
+    pub fn new(
+        uid: usize,
+        mailbox_hash: MailboxHash,
+        connection: Arc<FutureMutex<NntpConnection>>,
+        uid_store: Arc<UIDStore>,
+    ) -> Self {
+        NntpOp {
+            uid,
+            connection,
+            mailbox_hash,
+            uid_store,
+        }
+    }
+}
+
+impl BackendOp for NntpOp {
+    fn as_bytes(&mut self) -> ResultFuture<Vec<u8>> {
+        let mailbox_hash = self.mailbox_hash;
+        let uid = self.uid;
+        let uid_store = self.uid_store.clone();
+        let connection = self.connection.clone();
+        Ok(Box::pin(async move {
+            let mut res = String::with_capacity(8 * 1024);
+            let mut conn = connection.lock().await;
+            let path = uid_store.mailboxes.lock().await[&mailbox_hash]
+                .name()
+                .to_string();
+            conn.send_command(format!("GROUP {}", path).as_bytes())
+                .await?;
+            conn.read_response(&mut res, false).await?;
+            if !res.starts_with("211 ") {
+                return Err(MeliError::new(format!(
+                    "{} Could not select newsgroup {}: expected GROUP response but got: {}",
+                    &uid_store.account_name, path, res
+                )));
+            }
+            conn.send_command(format!("ARTICLE {}", uid).as_bytes())
+                .await?;
+            conn.read_response(&mut res, true).await?;
+            if !res.starts_with("220 ") {
+                return Err(MeliError::new(format!(
+                    "{} Could not select article {}: expected ARTICLE response but got: {}",
+                    &uid_store.account_name, path, res
+                )));
+            }
+            let pos = res.find("\r\n").unwrap_or(0) + 2;
+
+            Ok(res.as_bytes()[pos..].to_vec())
+        }))
+    }
+
+    fn fetch_flags(&self) -> ResultFuture<Flag> {
+        Ok(Box::pin(async move { Ok(Flag::default()) }))
+    }
+}

melib/src/backends/nntp/protocol_parser.rs

@@ -0,0 +1,163 @@
+/*
+ * meli - melib crate.
+ *
+ * Copyright 2017-2020 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+use crate::email::parser::IResult;
+use nom::{
+    bytes::complete::{is_not, tag},
+    combinator::opt,
+};
+use std::str::FromStr;
+
+pub struct NntpLineIterator<'a> {
+    slice: &'a str,
+}
+
+impl<'a> std::iter::DoubleEndedIterator for NntpLineIterator<'a> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        if self.slice.is_empty() {
+            None
+        } else if let Some(pos) = self.slice.rfind("\r\n") {
+            if self.slice[..pos].is_empty() {
+                self.slice = &self.slice[..pos];
+                None
+            } else if let Some(prev_pos) = self.slice[..pos].rfind("\r\n") {
+                let ret = &self.slice[prev_pos + 2..pos + 2];
+                self.slice = &self.slice[..prev_pos + 2];
+                Some(ret)
+            } else {
+                let ret = self.slice;
+                self.slice = &self.slice[ret.len()..];
+                Some(ret)
+            }
+        } else {
+            let ret = self.slice;
+            self.slice = &self.slice[ret.len()..];
+            Some(ret)
+        }
+    }
+}
+
+impl<'a> Iterator for NntpLineIterator<'a> {
+    type Item = &'a str;
+
+    fn next(&mut self) -> Option<&'a str> {
+        if self.slice.is_empty() {
+            None
+        } else if let Some(pos) = self.slice.find("\r\n") {
+            let ret = &self.slice[..pos + 2];
+            self.slice = &self.slice[pos + 2..];
+            Some(ret)
+        } else {
+            let ret = self.slice;
+            self.slice = &self.slice[ret.len()..];
+            Some(ret)
+        }
+    }
+}
+
+pub trait NntpLineSplit {
+    fn split_rn(&self) -> NntpLineIterator;
+}
+
+impl NntpLineSplit for str {
+    fn split_rn(&self) -> NntpLineIterator {
+        NntpLineIterator { slice: self }
+    }
+}
+
+pub fn over_article(input: &str) -> IResult<&str, (UID, Envelope)> {
+    /*
+    "0" or article number (see below)
+    Subject header content
+    From header content
+    Date header content
+    Message-ID header content
+    References header content
+    :bytes metadata item
+    :lines metadata item
+    */
+    let (input, num) = is_not("\t")(input)?;
+    let (input, _) = tag("\t")(input)?;
+    let (input, subject) = opt(is_not("\t"))(input)?;
+    let (input, _) = tag("\t")(input)?;
+    let (input, from) = opt(is_not("\t"))(input)?;
+    let (input, _) = tag("\t")(input)?;
+    let (input, date) = opt(is_not("\t"))(input)?;
+    let (input, _) = tag("\t")(input)?;
+    let (input, message_id) = opt(is_not("\t"))(input)?;
+    let (input, _) = tag("\t")(input)?;
+    let (input, references) = opt(is_not("\t"))(input)?;
+    let (input, _) = tag("\t")(input)?;
+    let (input, _bytes) = opt(is_not("\t"))(input)?;
+    let (input, _) = tag("\t")(input)?;
+    let (input, _lines) = opt(is_not("\t\r\n"))(input)?;
+    let (input, _other_headers) = opt(is_not("\r\n"))(input)?;
+    let (input, _) = tag("\r\n")(input)?;
+    Ok((
+        input,
+        ({
+            let env_hash = {
+                let mut hasher = DefaultHasher::new();
+                hasher.write(num.as_bytes());
+                hasher.write(message_id.unwrap_or_default().as_bytes());
+                hasher.finish()
+            };
+            let mut env = Envelope::new(env_hash);
+            if let Some(date) = date {
+                env.set_date(date.as_bytes());
+                if let Ok(d) = crate::email::parser::generic::date(env.date_as_str().as_bytes()) {
+                    env.set_datetime(d);
+                }
+            }
+
+            if let Some(subject) = subject {
+                env.set_subject(subject.into());
+            }
+
+            if let Some(from) = from {
+                if let Ok((_, from)) =
+                    crate::email::parser::address::rfc2822address_list(from.as_bytes())
+                {
+                    env.set_from(from);
+                }
+            }
+
+            if let Some(references) = references {
+                {
+                    if let Ok((_, r)) =
+                        crate::email::parser::address::references(references.as_bytes())
+                    {
+                        for v in r {
+                            env.push_references(v);
+                        }
+                    }
+                }
+                env.set_references(references.as_bytes());
+            }
+
+            if let Some(message_id) = message_id {
+                env.set_message_id(message_id.as_bytes());
+            }
+            (usize::from_str(num).unwrap(), env)
+        }),
+    ))
+}