melib/imap: add COMPRESS=DEFLATE support

Closes #53
memfd
Manos Pitsidianakis 2020-07-28 16:16:08 +03:00
parent d8f2a08e7b
commit 9a29f4245f
Signed by: Manos Pitsidianakis
GPG Key ID: 73627C2F690DF710
6 changed files with 167 additions and 34 deletions

37
Cargo.lock generated
View File

@ -1,5 +1,11 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "adler"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
[[package]]
name = "arc-swap"
version = "0.4.7"
@ -207,6 +213,15 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac"
[[package]]
name = "crc32fast"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1"
dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam"
version = "0.7.3"
@ -414,6 +429,18 @@ dependencies = [
"winapi 0.3.8",
]
[[package]]
name = "flate2"
version = "1.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68c90b0fc46cf89d227cc78b40e494ff81287a92dd07631e5af0d06fe3cf885e"
dependencies = [
"cfg-if",
"crc32fast",
"libc",
"miniz_oxide",
]
[[package]]
name = "fnv"
version = "1.0.7"
@ -936,6 +963,7 @@ dependencies = [
"crossbeam",
"data-encoding",
"encoding",
"flate2",
"futures",
"libc",
"libloading",
@ -999,6 +1027,15 @@ dependencies = [
"unicase",
]
[[package]]
name = "miniz_oxide"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be0f75932c1f6cfae3c04000e40114adf955636e19040f9c0a2c380702aa1c7f"
dependencies = [
"adler",
]
[[package]]
name = "mio"
version = "0.6.22"

View File

@ -235,6 +235,11 @@ Do not validate TLS certificates.
Use IDLE extension.
.\" default value
.Pq Em true
.It Ic use_deflate Ar boolean
.Pq Em optional
Use COMPRESS=DEFLATE extension (if built with DEFLATE support).
.\" default value
.Pq Em true
.El
.Ss JMAP only
JMAP specific options

View File

@ -46,9 +46,10 @@ futures = "0.3.5"
smol = "0.1.18"
async-stream = "0.2.1"
base64 = { version = "0.12.3", optional = true }
flate2 = { version = "1.0.16", optional = true }
[features]
default = ["unicode_algorithms", "imap_backend", "maildir_backend", "mbox_backend", "vcard", "sqlite3", "smtp"]
default = ["unicode_algorithms", "imap_backend", "maildir_backend", "mbox_backend", "vcard", "sqlite3", "smtp", "deflate_compression"]
debug-tracing = []
unicode_algorithms = ["unicode-segmentation"]
@ -60,3 +61,4 @@ jmap_backend = ["reqwest", "serde_json" ]
vcard = []
sqlite3 = ["rusqlite", ]
smtp = ["native-tls", "base64"]
deflate_compression = ["flate2", ]

View File

@ -61,6 +61,8 @@ pub static SUPPORTED_CAPABILITIES: &[&str] = &[
"LOGIN",
"LOGINDISABLED",
"LIST-STATUS",
#[cfg(feature = "deflate_compression")]
"COMPRESS=DEFLATE",
"ENABLE",
"IMAP4REV1",
"SPECIAL-USE",
@ -137,6 +139,7 @@ macro_rules! get_conf_val {
pub struct UIDStore {
account_hash: AccountHash,
cache_headers: bool,
account_name: Arc<String>,
capabilities: Arc<Mutex<Capabilities>>,
uidvalidity: Arc<Mutex<HashMap<MailboxHash, UID>>>,
hash_index: Arc<Mutex<HashMap<EnvelopeHash, (UID, MailboxHash)>>>,
@ -157,6 +160,7 @@ impl Default for UIDStore {
UIDStore {
account_hash: 0,
cache_headers: false,
account_name: Arc::new(String::new()),
capabilities: Default::default(),
uidvalidity: Default::default(),
hash_index: Default::default(),
@ -177,7 +181,6 @@ impl Default for UIDStore {
#[derive(Debug)]
pub struct ImapType {
account_name: String,
is_subscribed: Arc<IsSubscribedFn>,
connection: Arc<FutureMutex<ImapConnection>>,
server_conf: ImapServerConf,
@ -1081,6 +1084,8 @@ impl ImapType {
protocol: ImapProtocol::IMAP {
extension_use: ImapExtensionUse {
idle: get_conf_val!(s["use_idle"], true)?,
#[cfg(feature = "deflate_compression")]
deflate: get_conf_val!(s["use_deflate"], true)?,
},
},
};
@ -1089,18 +1094,18 @@ impl ImapType {
hasher.write(s.name.as_bytes());
hasher.finish()
};
let account_name = Arc::new(s.name().to_string());
let uid_store: Arc<UIDStore> = Arc::new(UIDStore {
account_hash,
cache_headers: get_conf_val!(s["X_header_caching"], false)?,
account_name,
..UIDStore::default()
});
let connection = ImapConnection::new_connection(&server_conf, uid_store.clone());
Ok(Box::new(ImapType {
account_name: s.name().to_string(),
server_conf,
is_subscribed: Arc::new(IsSubscribedFn(is_subscribed)),
can_create_flags: Arc::new(Mutex::new(false)),
connection: Arc::new(FutureMutex::new(connection)),
uid_store,
@ -1269,6 +1274,15 @@ impl ImapType {
get_conf_val!(s["danger_accept_invalid_certs"], false)?;
get_conf_val!(s["X_header_caching"], false)?;
get_conf_val!(s["use_idle"], true)?;
#[cfg(feature = "deflate_compression")]
get_conf_val!(s["use_deflate"], true)?;
#[cfg(not(feature = "deflate_compression"))]
if s.extra.contains_key("use_deflate") {
return Err(MeliError::new(format!(
"Configuration error ({}): setting `use_deflate` is set but this version of meli isn't compiled with DEFLATE support.",
s.name.as_str(),
)));
}
Ok(())
}

View File

@ -47,12 +47,16 @@ pub enum ImapProtocol {
#[derive(Debug, Clone, Copy)]
pub struct ImapExtensionUse {
pub idle: bool,
#[cfg(feature = "deflate_compression")]
pub deflate: bool,
}
impl Default for ImapExtensionUse {
fn default() -> Self {
Self {
idle: true,
#[cfg(feature = "deflate_compression")]
deflate: false,
}
}
}
@ -89,12 +93,6 @@ pub struct ImapConnection {
pub uid_store: Arc<UIDStore>,
}
impl Drop for ImapStream {
fn drop(&mut self) {
//self.send_command(b"LOGOUT").ok().take();
}
}
impl ImapStream {
pub async fn new_connection(
server_conf: &ImapServerConf,
@ -127,7 +125,7 @@ impl ImapStream {
))
.chain_err_kind(crate::error::ErrorKind::Network)?;
if server_conf.use_starttls {
let mut buf = vec![0; 1024];
let mut buf = vec![0; Connection::IO_BUF_SIZE];
match server_conf.protocol {
ImapProtocol::IMAP { .. } => socket
.write_all(format!("M{} STARTTLS\r\n", cmd_id).as_bytes())
@ -364,7 +362,7 @@ impl ImapStream {
termination_string: &str,
keep_termination_string: bool,
) -> Result<()> {
let mut buf: [u8; 1024] = [0; 1024];
let mut buf: Vec<u8> = vec![0; Connection::IO_BUF_SIZE];
ret.clear();
let mut last_line_idx: usize = 0;
loop {
@ -434,6 +432,7 @@ impl ImapStream {
self.stream.write_all(command).await?;
self.stream.write_all(b"\r\n").await?;
self.stream.flush().await?;
match self.protocol {
ImapProtocol::IMAP { .. } => {
debug!("sent: M{} {}", self.cmd_id - 1, unsafe {
@ -493,27 +492,72 @@ impl ImapConnection {
}
}
pub async fn connect(&mut self) -> Result<()> {
if let (instant, ref mut status @ Ok(())) = *self.uid_store.is_online.lock().unwrap() {
if Instant::now().duration_since(instant) >= std::time::Duration::new(60 * 30, 0) {
*status = Err(MeliError::new("Connection timed out"));
self.stream = Err(MeliError::new("Connection timed out"));
pub fn connect<'a>(&'a mut self) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'a>> {
Box::pin(async move {
if let (instant, ref mut status @ Ok(())) = *self.uid_store.is_online.lock().unwrap() {
if Instant::now().duration_since(instant) >= std::time::Duration::new(60 * 30, 0) {
*status = Err(MeliError::new("Connection timed out"));
self.stream = Err(MeliError::new("Connection timed out"));
}
}
}
if self.stream.is_ok() {
self.uid_store.is_online.lock().unwrap().0 = Instant::now();
return Ok(());
}
let new_stream = debug!(ImapStream::new_connection(&self.server_conf).await);
if let Err(err) = new_stream.as_ref() {
*self.uid_store.is_online.lock().unwrap() = (Instant::now(), Err(err.clone()));
} else {
*self.uid_store.is_online.lock().unwrap() = (Instant::now(), Ok(()));
}
let (capabilities, stream) = new_stream?;
self.stream = Ok(stream);
*self.uid_store.capabilities.lock().unwrap() = capabilities;
Ok(())
if self.stream.is_ok() {
self.uid_store.is_online.lock().unwrap().0 = Instant::now();
return Ok(());
}
let new_stream = debug!(ImapStream::new_connection(&self.server_conf).await);
if let Err(err) = new_stream.as_ref() {
*self.uid_store.is_online.lock().unwrap() = (Instant::now(), Err(err.clone()));
} else {
*self.uid_store.is_online.lock().unwrap() = (Instant::now(), Ok(()));
}
let (capabilities, stream) = new_stream?;
self.stream = Ok(stream);
match self.stream.as_ref()?.protocol {
ImapProtocol::IMAP {
extension_use:
ImapExtensionUse {
#[cfg(feature = "deflate_compression")]
deflate,
idle: _idle,
},
} =>
{
#[cfg(feature = "deflate_compression")]
if capabilities.contains(&b"COMPRESS=DEFLATE"[..]) && deflate {
let mut ret = String::new();
self.send_command(b"COMPRESS DEFLATE").await?;
self.read_response(&mut ret, RequiredResponses::empty())
.await?;
match ImapResponse::from(&ret) {
ImapResponse::No(code)
| ImapResponse::Bad(code)
| ImapResponse::Preauth(code)
| ImapResponse::Bye(code) => {
crate::log(format!("Could not use COMPRESS=DEFLATE in account `{}`: server replied with `{}`", self.uid_store.account_name, code), crate::LoggingLevel::WARN);
}
ImapResponse::Ok(_) => {
let ImapStream {
cmd_id,
stream,
protocol,
current_mailbox,
} = std::mem::replace(&mut self.stream, Err(MeliError::new("")))?;
let stream = stream.into_inner()?;
self.stream = Ok(ImapStream {
cmd_id,
stream: AsyncWrapper::new(stream.deflate())?,
protocol,
current_mailbox,
});
}
}
}
}
ImapProtocol::ManageSieve => {}
}
*self.uid_store.capabilities.lock().unwrap() = capabilities;
Ok(())
})
}
pub fn read_response<'a>(
@ -742,7 +786,7 @@ impl ImapConnection {
}
pub struct ImapBlockingConnection {
buf: [u8; 1024],
buf: Vec<u8>,
result: Vec<u8>,
prev_res_length: usize,
pub conn: ImapConnection,
@ -752,7 +796,7 @@ pub struct ImapBlockingConnection {
impl From<ImapConnection> for ImapBlockingConnection {
fn from(conn: ImapConnection) -> Self {
ImapBlockingConnection {
buf: [0; 1024],
buf: vec![0; Connection::IO_BUF_SIZE],
conn,
prev_res_length: 0,
result: Vec::with_capacity(8 * 1024),

View File

@ -18,6 +18,8 @@
* You should have received a copy of the GNU General Public License
* along with meli. If not, see <http://www.gnu.org/licenses/>.
*/
#[cfg(feature = "deflate_compression")]
use flate2::{read::DeflateDecoder, write::DeflateEncoder, Compression};
#[derive(Debug)]
pub enum Connection {
@ -25,11 +27,26 @@ pub enum Connection {
Fd(std::os::unix::io::RawFd),
#[cfg(feature = "imap_backend")]
Tls(native_tls::TlsStream<Self>),
#[cfg(feature = "deflate_compression")]
Deflate {
inner: DeflateEncoder<DeflateDecoder<Box<Self>>>,
},
}
use Connection::*;
impl Connection {
pub const IO_BUF_SIZE: usize = 64 * 1024;
#[cfg(feature = "deflate_compression")]
pub fn deflate(self) -> Self {
Connection::Deflate {
inner: DeflateEncoder::new(
DeflateDecoder::new_with_buf(Box::new(self), vec![0; Self::IO_BUF_SIZE]),
Compression::default(),
),
}
}
pub fn set_nonblocking(&self, nonblocking: bool) -> std::io::Result<()> {
match self {
Tcp(ref t) => t.set_nonblocking(nonblocking),
@ -50,6 +67,8 @@ impl Connection {
})?;
Ok(())
}
#[cfg(feature = "deflate_compression")]
Deflate { ref inner, .. } => inner.get_ref().get_ref().set_nonblocking(nonblocking),
}
}
@ -59,6 +78,8 @@ impl Connection {
#[cfg(feature = "imap_backend")]
Tls(ref t) => t.get_ref().set_read_timeout(dur),
Fd(_) => Ok(()),
#[cfg(feature = "deflate_compression")]
Deflate { ref inner, .. } => inner.get_ref().get_ref().set_read_timeout(dur),
}
}
@ -68,6 +89,8 @@ impl Connection {
#[cfg(feature = "imap_backend")]
Tls(ref t) => t.get_ref().set_write_timeout(dur),
Fd(_) => Ok(()),
#[cfg(feature = "deflate_compression")]
Deflate { ref inner, .. } => inner.get_ref().get_ref().set_write_timeout(dur),
}
}
}
@ -93,6 +116,8 @@ impl std::io::Read for Connection {
let _ = f.into_raw_fd();
ret
}
#[cfg(feature = "deflate_compression")]
Deflate { ref mut inner, .. } => inner.read(buf),
}
}
}
@ -110,6 +135,8 @@ impl std::io::Write for Connection {
let _ = f.into_raw_fd();
ret
}
#[cfg(feature = "deflate_compression")]
Deflate { ref mut inner, .. } => inner.write(buf),
}
}
@ -125,6 +152,8 @@ impl std::io::Write for Connection {
let _ = f.into_raw_fd();
ret
}
#[cfg(feature = "deflate_compression")]
Deflate { ref mut inner, .. } => inner.flush(),
}
}
}
@ -136,6 +165,8 @@ impl std::os::unix::io::AsRawFd for Connection {
#[cfg(feature = "imap_backend")]
Tls(ref t) => t.get_ref().as_raw_fd(),
Fd(f) => *f,
#[cfg(feature = "deflate_compression")]
Deflate { ref inner, .. } => inner.get_ref().get_ref().as_raw_fd(),
}
}
}