/* * meli - nntp module. * * Copyright 2017 - 2019 Manos Pitsidianakis * * This file is part of meli. * * meli is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * meli is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with meli. If not, see . */ use crate::backends::{BackendMailbox, MailboxHash}; use crate::connections::{lookup_ipv4, Connection}; use crate::email::parser::BytesExt; use crate::error::*; extern crate native_tls; use futures::io::{AsyncReadExt, AsyncWriteExt}; use native_tls::TlsConnector; pub use smol::Async as AsyncWrapper; use std::collections::HashSet; use std::future::Future; use std::pin::Pin; use std::sync::Arc; use std::time::Instant; use super::{Capabilities, NntpServerConf, UIDStore}; #[derive(Debug, Clone, Copy)] pub struct NntpExtensionUse { #[cfg(feature = "deflate_compression")] pub deflate: bool, } impl Default for NntpExtensionUse { fn default() -> Self { Self { #[cfg(feature = "deflate_compression")] deflate: false, } } } #[derive(Debug)] pub struct NntpStream { pub stream: AsyncWrapper, pub extension_use: NntpExtensionUse, pub current_mailbox: MailboxSelection, pub supports_submission: bool, } #[derive(Debug, Copy, Clone, Eq, PartialEq)] pub enum MailboxSelection { None, Select(MailboxHash), } impl MailboxSelection { pub fn take(&mut self) -> Self { std::mem::replace(self, MailboxSelection::None) } } async fn try_await(cl: impl Future> + Send) -> Result<()> { cl.await } #[derive(Debug)] pub struct NntpConnection { pub stream: Result, pub server_conf: NntpServerConf, pub uid_store: Arc, } impl NntpStream { pub async fn new_connection( server_conf: &NntpServerConf, ) -> Result<(Capabilities, NntpStream)> { use std::net::TcpStream; let path = &server_conf.server_hostname; let stream = { let addr = lookup_ipv4(path, server_conf.server_port)?; AsyncWrapper::new(Connection::Tcp( TcpStream::connect_timeout(&addr, std::time::Duration::new(16, 0)) .chain_err_kind(crate::error::ErrorKind::Network)?, )) .chain_err_kind(crate::error::ErrorKind::Network)? }; let mut res = String::with_capacity(8 * 1024); let mut ret = NntpStream { stream, extension_use: server_conf.extension_use, current_mailbox: MailboxSelection::None, supports_submission: false, }; if server_conf.use_tls { let mut connector = TlsConnector::builder(); if server_conf.danger_accept_invalid_certs { connector.danger_accept_invalid_certs(true); } let connector = connector .build() .chain_err_kind(crate::error::ErrorKind::Network)?; if server_conf.use_starttls { ret.read_response(&mut res, false, &["200 ", "201 "]) .await?; ret.supports_submission = res.starts_with("200"); ret.send_command(b"CAPABILITIES").await?; ret.read_response(&mut res, true, command_to_replycodes("CAPABILITIES")) .await?; if !res.starts_with("101 ") { return Err(MeliError::new(format!( "Could not connect to {}: expected CAPABILITIES response but got:{}", &server_conf.server_hostname, res ))); } let capabilities: Vec<&str> = res.lines().skip(1).collect(); if !capabilities .iter() .any(|cap| cap.eq_ignore_ascii_case("VERSION 2")) { return Err(MeliError::new(format!( "Could not connect to {}: server is not NNTP VERSION 2 compliant", &server_conf.server_hostname ))); } if !capabilities .iter() .any(|cap| cap.eq_ignore_ascii_case("STARTTLS")) { return Err(MeliError::new(format!( "Could not connect to {}: server does not support STARTTLS", &server_conf.server_hostname ))); } ret.stream .write_all(b"STARTTLS\r\n") .await .chain_err_kind(crate::error::ErrorKind::Network)?; ret.stream .flush() .await .chain_err_kind(crate::error::ErrorKind::Network)?; ret.read_response(&mut res, false, command_to_replycodes("STARTTLS")) .await?; if !res.starts_with("382 ") { return Err(MeliError::new(format!( "Could not connect to {}: could not begin TLS negotiation, got: {}", &server_conf.server_hostname, res ))); } } { // FIXME: This is blocking let socket = ret .stream .into_inner() .chain_err_kind(crate::error::ErrorKind::Network)?; let mut conn_result = connector.connect(path, socket); if let Err(native_tls::HandshakeError::WouldBlock(midhandshake_stream)) = conn_result { let mut midhandshake_stream = Some(midhandshake_stream); loop { match midhandshake_stream.take().unwrap().handshake() { Ok(r) => { conn_result = Ok(r); break; } Err(native_tls::HandshakeError::WouldBlock(stream)) => { midhandshake_stream = Some(stream); } p => { p.chain_err_kind(crate::error::ErrorKind::Network)?; } } } } ret.stream = AsyncWrapper::new(Connection::Tls( conn_result.chain_err_kind(crate::error::ErrorKind::Network)?, )) .chain_err_summary(|| format!("Could not initiate TLS negotiation to {}.", path)) .chain_err_kind(crate::error::ErrorKind::Network)?; } } //ret.send_command( // format!( // "LOGIN \"{}\" \"{}\"", // &server_conf.server_username, &server_conf.server_password // ) // .as_bytes(), //) //.await?; if let Err(err) = ret .stream .get_ref() .set_keepalive(Some(std::time::Duration::new(60 * 9, 0))) { crate::log( format!("Could not set TCP keepalive in NNTP connection: {}", err), crate::LoggingLevel::WARN, ); } ret.read_response(&mut res, false, &["200 ", "201 "]) .await?; ret.supports_submission = res.starts_with("200"); ret.send_command(b"CAPABILITIES").await?; ret.read_response(&mut res, true, command_to_replycodes("CAPABILITIES")) .await?; if !res.starts_with("101 ") { return Err(MeliError::new(format!( "Could not connect to {}: expected CAPABILITIES response but got:{}", &server_conf.server_hostname, res ))); } let capabilities: HashSet = res.lines().skip(1).map(|l| l.trim().to_string()).collect(); if !capabilities .iter() .any(|cap| cap.eq_ignore_ascii_case("VERSION 2")) { return Err(MeliError::new(format!( "Could not connect to {}: server is not NNTP compliant", &server_conf.server_hostname ))); } if !capabilities .iter() .any(|cap| cap.eq_ignore_ascii_case("POST")) { ret.supports_submission = false; } if server_conf.require_auth { if capabilities.iter().any(|c| c.starts_with("AUTHINFO USER")) { ret.send_command( format!("AUTHINFO USER {}", server_conf.server_username).as_bytes(), ) .await?; ret.read_response(&mut res, false, command_to_replycodes("AUTHINFO USER")) .await .chain_err_summary(|| format!("Authentication state error: {}", res)) .chain_err_kind(ErrorKind::Authentication)?; if res.starts_with("381 ") { ret.send_command( format!("AUTHINFO PASS {}", server_conf.server_password).as_bytes(), ) .await?; ret.read_response(&mut res, false, command_to_replycodes("AUTHINFO PASS")) .await .chain_err_summary(|| format!("Authentication state error: {}", res)) .chain_err_kind(ErrorKind::Authentication)?; } } else { return Err(MeliError::new(format!( "Could not connect: no supported auth mechanisms in server capabilities: {:?}", capabilities )) .set_err_kind(ErrorKind::Authentication)); } } #[cfg(feature = "deflate_compression")] if capabilities.contains("COMPRESS DEFLATE") && ret.extension_use.deflate { ret.send_command(b"COMPRESS DEFLATE").await?; ret.read_response(&mut res, false, command_to_replycodes("COMPRESS DEFLATE")) .await .chain_err_summary(|| { format!( "Could not use COMPRESS DEFLATE in account `{}`: server replied with `{}`", server_conf.server_hostname, res ) })?; let NntpStream { stream, extension_use, current_mailbox, supports_submission, } = ret; let stream = stream.into_inner()?; return Ok(( capabilities, NntpStream { stream: AsyncWrapper::new(stream.deflate())?, extension_use, current_mailbox, supports_submission, }, )); } Ok((capabilities, ret)) } pub async fn read_response( &mut self, ret: &mut String, is_multiline: bool, expected_reply_code: &[&str], ) -> Result<()> { self.read_lines(ret, is_multiline, expected_reply_code) .await?; Ok(()) } pub async fn read_lines( &mut self, ret: &mut String, is_multiline: bool, expected_reply_code: &[&str], ) -> Result<()> { let mut buf: Vec = vec![0; Connection::IO_BUF_SIZE]; ret.clear(); let mut last_line_idx: usize = 0; loop { match self.stream.read(&mut buf).await { Ok(0) => break, Ok(b) => { ret.push_str(unsafe { std::str::from_utf8_unchecked(&buf[0..b]) }); if ret.len() > 4 { if ret.starts_with("205 ") { return Err(MeliError::new(format!("Disconnected: {}", ret))); } else if ret.starts_with("501 ") || ret.starts_with("500 ") { return Err(MeliError::new(format!("Syntax error: {}", ret))); } else if ret.starts_with("403 ") { return Err(MeliError::new(format!("Internal error: {}", ret))); } else if ret.starts_with("502 ") || ret.starts_with("480 ") || ret.starts_with("483 ") || ret.starts_with("401 ") { return Err(MeliError::new(format!("Connection state error: {}", ret)) .set_err_kind(ErrorKind::Authentication)); } else if !expected_reply_code.iter().any(|r| ret.starts_with(r)) { return Err(MeliError::new(format!("Unexpected reply code: {}", ret))); } } if let Some(mut pos) = ret[last_line_idx..].rfind("\r\n") { if !is_multiline { break; } else if let Some(pos) = ret.find("\r\n.\r\n") { ret.replace_range(pos + "\r\n".len()..pos + "\r\n.\r\n".len(), ""); break; } if let Some(prev_line) = ret[last_line_idx..pos + last_line_idx].rfind("\r\n") { last_line_idx += prev_line + "\r\n".len(); pos -= prev_line + "\r\n".len(); } last_line_idx += pos + "\r\n".len(); } } Err(e) => { return Err(MeliError::from(e).set_err_kind(crate::error::ErrorKind::Network)); } } } //debug!("returning nntp response:\n{:?}", &ret); Ok(()) } pub async fn send_command(&mut self, command: &[u8]) -> Result<()> { debug!("sending: {}", unsafe { std::str::from_utf8_unchecked(command) }); if let Err(err) = try_await(async move { let command = command.trim(); self.stream.write_all(command).await?; self.stream.write_all(b"\r\n").await?; self.stream.flush().await?; debug!("sent: {}", unsafe { std::str::from_utf8_unchecked(command) }); Ok(()) }) .await { debug!("stream send_command err {:?}", err); Err(err.set_err_kind(crate::error::ErrorKind::Network)) } else { Ok(()) } } pub async fn send_multiline_data_block(&mut self, data: &[u8]) -> Result<()> { if let Err(err) = try_await(async move { let mut ptr = 0; while let Some(pos) = data[ptr..].find("\n") { let l = &data[ptr..ptr + pos].trim_end(); if l.starts_with(b".") { self.stream.write_all(b".").await?; } self.stream.write_all(l).await?; self.stream.write_all(b"\r\n").await?; ptr += pos + 1; } let l = &data[ptr..].trim_end(); if !l.is_empty() { if l.starts_with(b".") { self.stream.write_all(b".").await?; } self.stream.write_all(l).await?; self.stream.write_all(b"\r\n").await?; } self.stream.write_all(b".\r\n").await?; self.stream.flush().await?; debug!("sent data block {} bytes", data.len()); Ok(()) }) .await { debug!("stream send_multiline_data_block err {:?}", err); Err(err.set_err_kind(crate::error::ErrorKind::Network)) } else { Ok(()) } } } impl NntpConnection { pub fn new_connection( server_conf: &NntpServerConf, uid_store: Arc, ) -> NntpConnection { NntpConnection { stream: Err(MeliError::new("Offline".to_string())), server_conf: server_conf.clone(), uid_store, } } pub fn connect<'a>(&'a mut self) -> Pin> + Send + 'a>> { Box::pin(async move { if let (instant, ref mut status @ Ok(())) = *self.uid_store.is_online.lock().unwrap() { if Instant::now().duration_since(instant) >= std::time::Duration::new(60 * 30, 0) { *status = Err(MeliError::new("Connection timed out")); self.stream = Err(MeliError::new("Connection timed out")); } } if self.stream.is_ok() { self.uid_store.is_online.lock().unwrap().0 = Instant::now(); return Ok(()); } let new_stream = NntpStream::new_connection(&self.server_conf).await; if let Err(err) = new_stream.as_ref() { *self.uid_store.is_online.lock().unwrap() = (Instant::now(), Err(err.clone())); } else { *self.uid_store.is_online.lock().unwrap() = (Instant::now(), Ok(())); } let (capabilities, stream) = new_stream?; self.stream = Ok(stream); *self.uid_store.capabilities.lock().unwrap() = capabilities; Ok(()) }) } pub fn read_response<'a>( &'a mut self, ret: &'a mut String, is_multiline: bool, expected_reply_code: &'static [&str], ) -> Pin> + Send + 'a>> { Box::pin(async move { ret.clear(); self.stream .as_mut()? .read_response(ret, is_multiline, expected_reply_code) .await }) } pub async fn read_lines( &mut self, ret: &mut String, is_multiline: bool, expected_reply_code: &[&str], ) -> Result<()> { self.stream .as_mut()? .read_lines(ret, is_multiline, expected_reply_code) .await?; Ok(()) } pub async fn send_command(&mut self, command: &[u8]) -> Result<()> { if let Err(err) = try_await(async { self.stream.as_mut()?.send_command(command).await }).await { self.stream = Err(err.clone()); debug!(err.kind); if err.kind.is_network() { debug!(self.connect().await)?; } Err(err) } else { Ok(()) } } pub fn add_refresh_event(&mut self, ev: crate::backends::RefreshEvent) { (self.uid_store.event_consumer)( self.uid_store.account_hash, crate::backends::BackendEvent::Refresh(ev), ); } pub async fn select_group( &mut self, mailbox_hash: MailboxHash, force: bool, res: &mut String, ) -> Result<()> { if !force { match self.stream.as_ref()?.current_mailbox { MailboxSelection::Select(m) if m == mailbox_hash => return Ok(()), _ => {} } } let path = self.uid_store.mailboxes.lock().await[&mailbox_hash] .name() .to_string(); self.send_command(format!("GROUP {}", path).as_bytes()) .await?; self.read_response(res, false, command_to_replycodes("GROUP")) .await .chain_err_summary(|| { format!( "{} Could not select newsgroup {}: expected GROUP response but got: {}", &self.uid_store.account_name, path, res ) })?; self.stream.as_mut()?.current_mailbox = MailboxSelection::Select(mailbox_hash); Ok(()) } pub async fn send_multiline_data_block(&mut self, message: &[u8]) -> Result<()> { self.stream .as_mut()? .send_multiline_data_block(message) .await } } pub fn command_to_replycodes(c: &str) -> &'static [&'static str] { if c.starts_with("OVER") { &["224 "] } else if c.starts_with("LIST") { &["215 "] } else if c.starts_with("POST") { &["340 "] } else if c.starts_with("STARTTLS") { &["382 "] } else if c.starts_with("GROUP") { &["211 "] } else if c.starts_with("CAPABILITIES") { &["101 "] } else if c.starts_with("ARTICLE") { &["220 "] } else if c.starts_with("DATE") { &["111 "] } else if c.starts_with("NEWNEWS") { &["230 "] } else if c.starts_with("AUTHINFO USER") { &["281 ", "381 "] } else if c.starts_with("AUTHINFO PASS") { &["281 "] } else if c.starts_with("COMPRESS DEFLATE") { &["206 "] } else { &[] } }