Cargo.lock: Update lexical-core version

Fixes compilation on macos 10.15.3, rustc 1.53.0

CHANGELOG.md

@@ -7,6 +7,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- Add import command to import email from files into accounts
+- Add add-attachment-file-picker command and `file_picker_command` setting to
+  use external commands to choose files when composing new mail
+
+## [alpha-0.6.2] - 2020-09-24
+
+### Added
+- Add customizable mailbox tree in sidebar
+- Make `dbus` dependency opt-out (feature is `dbus-notifications`)
+- Implemented JMAP async, search, tagging, syncing
+- Preserve account order from configuration file
+- Implemented IMAP `CONDSTORE` support for IMAP cache
+- Add `timeout` setting for IMAP
+- Implement TCP keepalive for IMAP
+- Rewrote email address parsers.
+- Implement `copy_messages` for maildir
+- Implement selection with motions
+
+### Fixed
+- Fixed various problems with IMAP cache
+- Fixed various problems with IMAP message counts
+- Fixed various problems with IMAP connection hanging
+- Fixed IMAP not reconnecting on dropped IDLE connections
+- Fixed various problems with notmuch backend
+
 ## [alpha-0.6.1] - 2020-08-02
 
 ### Added
@@ -82,3 +108,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [alpha-0.5.1]: https://github.com/meli/meli/releases/tag/alpha-0.5.1
 [alpha-0.6.0]: https://github.com/meli/meli/releases/tag/alpha-0.6.0
 [alpha-0.6.1]: https://github.com/meli/meli/releases/tag/alpha-0.6.1
+[alpha-0.6.2]: https://github.com/meli/meli/releases/tag/alpha-0.6.2

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "meli"
-version = "0.6.1"
+version = "0.6.2"
 authors = ["Manos Pitsidianakis <el13635@mail.ntua.gr>"]
 edition = "2018"
 
@@ -31,25 +31,21 @@ crossbeam = "0.7.2"
 signal-hook = "0.1.12"
 signal-hook-registry = "1.2.0"
 nix = "0.17.0"
-melib = { path = "melib", version = "0.6.1" }
+melib = { path = "melib", version = "0.6.2" }
 
 serde = "1.0.71"
 serde_derive = "1.0.71"
 serde_json = "1.0"
 toml = { version = "0.5.6", features = ["preserve_order", ] }
-indexmap = { version = "^1.5", features = ["serde-1", ] }
+indexmap = { version = "^1.6", features = ["serde-1", ] }
 linkify = "0.4.0"
 notify = "4.0.1" # >:c
-notify-rust = { version = "^4", optional = true }
 termion = "1.5.1"
-bincode = "1.2.0"
+bincode = "^1.3.0"
 uuid = { version = "0.8.1", features = ["serde", "v4"] }
 unicode-segmentation = "1.2.1" # >:c
 libc = {version = "0.2.59", features = ["extra_traits",]}
-rmp = "^0.8"
-rmpv = { version = "^0.4.2", features=["with-serde",] }
-rmp-serde = "^0.14.0"
-smallvec = { version = "^1.4.0", features = ["serde", ] }
+smallvec = { version = "^1.5.0", features = ["serde", ] }
 bitflags = "1.0"
 pcre2 = { version = "0.2.3", optional = true }
 structopt = { version = "0.3.14", default-features = false }
@@ -57,30 +53,37 @@ svg_crate = { version = "0.8.0", optional = true, package = "svg" }
 futures = "0.3.5"
 async-task = "3.0.0"
 num_cpus = "1.12.0"
+flate2 = { version = "1.0.16", optional = true }
+ 
+[target.'cfg(target_os="linux")'.dependencies]
+notify-rust = { version = "^4", optional = true }
 
 [build-dependencies]
 syn = { version = "1.0.31", features = [] }
 quote = "^1.0"
 proc-macro2 = "1.0.18"
+flate2 = { version = "1.0.16", optional = true }
 
 [profile.release]
 lto = "fat"
-opt-level = "z"
+codegen-units = 1
+opt-level = "s"
 debug = false
 
 [workspace]
 members = ["melib", "tools", ]
 
 [features]
-default = ["sqlite3", "notmuch", "regexp", "smtp", "dbus-notifications"]
+default = ["sqlite3", "notmuch", "regexp", "smtp", "dbus-notifications", "gpgme"]
 notmuch = ["melib/notmuch_backend", ]
 jmap = ["melib/jmap_backend",]
 sqlite3 = ["melib/sqlite3"]
 smtp = ["melib/smtp"]
 regexp = ["pcre2"]
 dbus-notifications = ["notify-rust",]
-cli-docs = []
+cli-docs = ["flate2"]
 svgscreenshot = ["svg_crate"]
+gpgme = ["melib/gpgme"]
 
 # Print tracing logs as meli runs in stderr
 # enable for debug tracing logs: build with --features=debug-tracing

Makefile

@@ -26,8 +26,10 @@ MANDIR ?= ${EXPANDED_PREFIX}/share/man
 CARGO_TARGET_DIR ?= target
 MIN_RUSTC ?= 1.39.0
 CARGO_BIN ?= cargo
+CARGO_ARGS ?=
 
 # Installation parameters
+DOCS_SUBDIR ?= docs/
 MANPAGES ?= meli.1 meli.conf.5 meli-themes.5
 FEATURES ?= --features "${MELI_FEATURES}"
 
@@ -47,7 +49,7 @@ GREEN ?= `[ -z $${NO_COLOR+x} ] && ([ -z $${TERM} ] && echo "" || tput setaf 2)
 .POSIX:
 .SUFFIXES:
 meli: check-deps
-	@${CARGO_BIN} build ${CARGO_COLOR}--target-dir="${CARGO_TARGET_DIR}" ${FEATURES} --release
+	@${CARGO_BIN} build ${CARGO_ARGS} ${CARGO_COLOR}--target-dir="${CARGO_TARGET_DIR}" ${FEATURES} --release
 
 help:
 	@echo "For a quick start, build and install locally:\n ${BOLD}${GREEN}PREFIX=~/.local make install${ANSI_RESET}\n"
@@ -83,15 +85,18 @@ help:
 	@echo -n "* NO_COLOR ${UNDERLINE}"
 	@[ $${NO_COLOR+x} ] && echo -n "set" || echo -n "unset"
 	@echo ${ANSI_RESET}
+	@echo "* CARGO_BIN = ${UNDERLINE}${CARGO_BIN}${ANSI_RESET}"
+	@echo "* CARGO_ARGS = ${UNDERLINE}${CARGO_ARGS}${ANSI_RESET}"
+	@echo "* MIN_RUSTC = ${UNDERLINE}${MIN_RUSTC}${ANSI_RESET}"
 	@#@echo "* CARGO_COLOR = ${CARGO_COLOR}"
 
 .PHONY: check
 check:
-	@${CARGO_BIN} test ${CARGO_COLOR}--target-dir="${CARGO_TARGET_DIR}" --workspace
+	@${CARGO_BIN} test ${CARGO_ARGS} ${CARGO_COLOR}--target-dir="${CARGO_TARGET_DIR}" --workspace
 
 .PHONY: check-deps
 check-deps:
-	@(if ! echo ${MIN_RUSTC}\\n`${CARGO_BIN} --version | cut -d ' ' -f 2` | sort -CV; then echo "rust version >= ${RED}${MIN_RUSTC}${ANSI_RESET} required, found: `which ${CARGO_BIN}` `${CARGO_BIN} --version | cut -d ' ' -f 2`" \
+	@(if ! echo ${MIN_RUSTC}\\n`${CARGO_BIN} --version | grep ^cargo | cut -d ' ' -f 2` | sort -CV; then echo "rust version >= ${RED}${MIN_RUSTC}${ANSI_RESET} required, found: `which ${CARGO_BIN}` `${CARGO_BIN} --version | cut -d ' ' -f 2`" \
 		"\nYour options:\n - Set CARGO_BIN to a supported version\n - Install a supported version from your distribution's package manager\n - Install a supported version from ${UNDERLINE}https://rustup.rs/${ANSI_RESET}" ; exit 1; fi)
 
 
@@ -120,7 +125,7 @@ install-doc:
 			SECTION=`echo $${MANPAGE} | rev | cut -d "." -f 1`; \
 			MANPAGEPATH=${DESTDIR}${MANDIR}/man$${SECTION}/$${MANPAGE}.gz; \
 			echo "  * installing $${MANPAGE} → ${GREEN}$${MANPAGEPATH}${ANSI_RESET}"; \
-			gzip -n < $${MANPAGE} > $${MANPAGEPATH} \
+			gzip -n < ${DOCS_SUBDIR}$${MANPAGE} > $${MANPAGEPATH} \
     ; done ; \
 	(case ":${MANPATHS}:" in \
   *:${DESTDIR}${MANDIR}:*) echo -n "";; \
@@ -136,7 +141,10 @@ install-bin: meli
   *:${DESTDIR}${BINDIR}:*) echo -n "";; \
 	*) echo "\n${RED}${BOLD}WARNING${ANSI_RESET}: ${UNDERLINE}Path ${DESTDIR}${BINDIR} is not contained in your PATH variable.${ANSI_RESET} Consider adding it if necessary.\nPATH variable: ${PATH}";; \
 	esac
-	@install -D ./${CARGO_TARGET_DIR}/release/meli $(DESTDIR)${BINDIR}/meli
+	@mkdir -p $(DESTDIR)${BINDIR}
+	@rm -f  $(DESTDIR)${BINDIR}/meli
+	@cp ./${CARGO_TARGET_DIR}/release/meli $(DESTDIR)${BINDIR}/meli
+	@chmod 755 $(DESTDIR)${BINDIR}/meli
 
 
 .PHONY: install
@@ -160,4 +168,4 @@ deb-dist:
 
 .PHONY: build-rustdoc
 build-rustdoc:
-	@RUSTDOCFLAGS="--crate-version ${VERSION}_${GIT_COMMIT}_${DATE}" ${CARGO_BIN} doc ${CARGO_COLOR}--target-dir="${CARGO_TARGET_DIR}" --all-features --no-deps --workspace --document-private-items --open
+	@RUSTDOCFLAGS="--crate-version ${VERSION}_${GIT_COMMIT}_${DATE}" ${CARGO_BIN} doc ${CARGO_ARGS} ${CARGO_COLOR}--target-dir="${CARGO_TARGET_DIR}" --all-features --no-deps --workspace --document-private-items --open

README.md

@@ -1,92 +1,87 @@
-# meli
+# meli [![GitHub license](https://img.shields.io/github/license/meli/meli)](https://github.com/meli/meli/blob/master/COPYING) [![Crates.io](https://img.shields.io/crates/v/meli)](https://crates.io/crates/meli)
+
+**BSD/Linux terminal email client with support for multiple accounts and Maildir / mbox / notmuch / IMAP / JMAP.**
+
+Community links:
+[mailing lists](https://lists.meli.delivery/) | `#meli` on OFTC IRC | Report bugs and/or feature requests in [meli's issue tracker](https://git.meli.delivery/meli/meli/issues "meli gitea issue tracker")
+
+| | | |
+:---:|:---:|:---:
+![Main view screenshot](./docs/screenshots/main.webp "mail meli view screenshot")  |  ![Compact main view screenshot](./docs/screenshots/compact.webp "compact main view screenshot") | ![Compose with embed terminal editor screenshot](./docs/screenshots/compose.webp "composing view screenshot")
+Main view             |  Compact main view | Compose with embed terminal editor
+
+Main repository:
+* https://git.meli.delivery/meli/meli
+
+Official mirrors:
+* https://github.com/meli/meli
+
+## Install
+- Try an [online interactive web demo](https://meli.delivery/wasm2.html "online interactive web demo") powered by WebAssembly
+- [`cargo install meli`](https://crates.io/crates/meli "crates.io meli package")
+- [Download and install pre-built debian package, static linux binary](https://github.com/meli/meli/releases/ "github releases for meli"), or
+- Install with [Nix](https://search.nixos.org/packages?show=meli&query=meli&from=0&size=30&sort=relevance&channel=unstable#disabled "nixos package search results for 'meli'")
+
+## Documentation
+
+See also [Quickstart tutorial](https://meli.delivery/documentation.html#quick-start).
+
+After installing meli, see `meli(1)`, `meli.conf(5)` and `meli-themes(5)` for documentation. Sample configuration and theme files can be found in the `docs/samples/` subdirectory. Manual pages are also [hosted online](https://meli.delivery/documentation.html "meli documentation").
+
+meli by default looks for a configuration file in this location: `$XDG_CONFIG_HOME/meli/config.toml`
+
+You can run meli with arbitrary configuration files by setting the `$MELI_CONFIG`
+environment variable to their locations, i.e.:
+
+```sh
+MELI_CONFIG=./test_config cargo run
+```
+
+## Build
 For a quick start, build and install locally:
 
 ```sh
  PREFIX=~/.local make install
 ```
 
-Available subcommands:
- - meli (builds meli with optimizations in `$CARGO_TARGET_DIR`)
- - install (installs binary in `$BINDIR` and documentation to `$MANDIR`)
- - uninstall
-Secondary subcommands:
- - clean (cleans build artifacts)
- - check-deps (checks dependencies)
- - install-bin (installs binary to `$BINDIR`)
- - install-doc (installs manpages to `$MANDIR`)
- - help (prints this information)
- - dist (creates release tarball named `meli-VERSION.tar.gz` in this directory)
- - deb-dist (builds debian package in the parent directory)
- - distclean (cleans distribution build artifacts)
- - build-rustdoc (builds rustdoc documentation for all packages in `$CARGO_TARGET_DIR`)
-
-The Makefile *should* be portable and not require a specific `make` version.
-
-# Documentation
-
-After installing meli, see `meli(1)` and `meli.conf(5)` for documentation. Sample configuration and theme files can be found in the `samples/` subdirectory.
-
-# Building
+Available subcommands for `make` are listed with `make help`. The Makefile *should* be POSIX portable and not require a specific `make` version.
 
 meli requires rust 1.39 and rust's package manager, Cargo. Information on how
 to get it on your system can be found here: <https://doc.rust-lang.org/cargo/getting-started/installation.html>
 
-With Cargo available, the project can be built with
+With Cargo available, the project can be built with `make` and the resulting binary will then be found under `target/release/meli`. Run `make install` to install the binary and man pages. This requires root, so I suggest you override the default paths and install it in your `$HOME`: `make PREFIX=$HOME/.local install`.
 
-```sh
-make
-```
+You can build and run meli with one command: `cargo run --release`.
 
-The resulting binary will then be found under `target/release/meli`
-
-Run:
-
-```sh
-make install
-```
-
-to install the binary and man pages. This requires root, so I suggest you override the default paths and install it in your `$HOME`:
-
-```sh
-make PREFIX=$HOME/.local install
-```
-
-See `meli(1)` and `meli.conf(5)` for documentation.
-
-You can build and run meli with one command:
-
-```sh
-cargo run --release
-```
-
-While the project is in early development, meli will only be developed for the
-linux kernel and respected linux distributions. Support for more UNIX-like OSes
-is on the roadmap.
-
-## Features
+### Build features
 
 Some functionality is held behind "feature gates", or compile-time flags. The following list explains each feature's purpose:
 
-- `dbus-notifications` enables showing notifications using `dbus`.
-- `notmuch` provides support for using a notmuch database as a mail backend
-- `jmap` provides support for connecting to a jmap server and use it as a mail backend
-- `sqlite3` provides support for builting fast search indexes in local sqlite3 databases
+- `gpgme` enables GPG support via `libgpgme` (on by default)
+- `dbus-notifications` enables showing notifications using `dbus` (on by default)
+- `notmuch` provides support for using a notmuch database as a mail backend (on by default)
+- `jmap` provides support for connecting to a jmap server and use it as a mail backend (off by default)
+- `sqlite3` provides support for builting fast search indexes in local sqlite3 databases (on by default)
 - `cli-docs` includes the manpage documentation compiled by either `mandoc` or `man` binary to plain text in `meli`'s command line. Embedded documentation can be viewed with the subcommand `meli man [PAGE]`
-- `svgscreenshot` provides support for taking screenshots of the current view of meli and saving it as SVG files. Its only purpose is taking screenshots for the official meli webpage.
-- `debug-tracing` enables various trace debug logs from various places around the meli code base. The trace log is printed in `stderr`.
+- `svgscreenshot` provides support for taking screenshots of the current view of meli and saving it as SVG files. Its only purpose is taking screenshots for the official meli webpage. (off by default)
+- `debug-tracing` enables various trace debug logs from various places around the meli code base. The trace log is printed in `stderr`. (off by default)
 
-## Building in Debian
+### Build Debian package (*deb*)
 
 Building with Debian's packaged cargo might require the installation of these
 two packages: `librust-openssl-sys-dev librust-libdbus-sys-dev`
 
 A `*.deb` package can be built with `make deb-dist`
 
-# Using notmuch
+### Using notmuch
 
 To use the optional notmuch backend feature, you must have `libnotmuch5` installed in your system. In Debian-like systems, install the `libnotmuch5` packages. meli detects the library's presence on runtime.
 
-# Building with JMAP
+### Using GPG
+
+To use the optional gpg feature, you must have `libgpgme` installed in your system. In Debian-like systems, install the `libgpgme11` package. meli detects the library's presence on runtime.
+
+### Building with JMAP
 
 To build with JMAP support, prepend the environment variable `MELI_FEATURES='jmap'` to your make invocation:
 
@@ -111,18 +106,7 @@ are printed in stderr, thus you can run meli with a redirection (i.e `2> log`)
 
 Code style follows the default rustfmt profile.
 
-# Configuration
-
-meli by default looks for a configuration file in this location: `$XDG_CONFIG_HOME/meli/config.toml`
-
-You can run meli with arbitrary configuration files by setting the `$MELI_CONFIG`
-environment variable to their locations, ie:
-
-```sh
-MELI_CONFIG=./test_config cargo run
-```
-
-# Testing
+## Testing
 
 How to run specific tests:
 
@@ -130,14 +114,14 @@ How to run specific tests:
 cargo test -p {melib, meli} (-- --nocapture) (--test test_name)
 ```
 
-# Profiling
+## Profiling
 
 ```sh
 perf record -g target/debug/bin
 perf script | stackcollapse-perf | rust-unmangle | flamegraph > perf.svg
 ```
 
-# Running fuzz targets
+## Running fuzz targets
 
 Note: `cargo-fuzz` requires the nightly toolchain.
 

build.rs

@@ -37,6 +37,8 @@ fn main() {
     ]);
     #[cfg(feature = "cli-docs")]
     {
+        use flate2::Compression;
+        use flate2::GzBuilder;
         const MANDOC_OPTS: &[&'static str] = &["-T", "utf8", "-I", "os=Generated by mandoc(1)"];
         use std::env;
         use std::fs::File;
@@ -45,38 +47,60 @@ fn main() {
         use std::process::Command;
         let out_dir = env::var("OUT_DIR").unwrap();
         let mut out_dir_path = Path::new(&out_dir).to_path_buf();
-        out_dir_path.push("meli.txt");
+        out_dir_path.push("meli.txt.gz");
 
         let output = Command::new("mandoc")
             .args(MANDOC_OPTS)
-            .arg("meli.1")
+            .arg("docs/meli.1")
             .output()
-            .or_else(|_| Command::new("man").arg("-l").arg("meli.1").output())
+            .or_else(|_| Command::new("man").arg("-l").arg("docs/meli.1").output())
             .unwrap();
 
-        let mut file = File::create(&out_dir_path).unwrap();
-        file.write_all(&output.stdout).unwrap();
+        let file = File::create(&out_dir_path).unwrap();
+        let mut gz = GzBuilder::new()
+            .comment(output.stdout.len().to_string().into_bytes())
+            .write(file, Compression::default());
+        gz.write_all(&output.stdout).unwrap();
+        gz.finish().unwrap();
         out_dir_path.pop();
 
-        out_dir_path.push("meli.conf.txt");
+        out_dir_path.push("meli.conf.txt.gz");
         let output = Command::new("mandoc")
             .args(MANDOC_OPTS)
-            .arg("meli.conf.5")
+            .arg("docs/meli.conf.5")
             .output()
-            .or_else(|_| Command::new("man").arg("-l").arg("meli.conf.5").output())
+            .or_else(|_| {
+                Command::new("man")
+                    .arg("-l")
+                    .arg("docs/meli.conf.5")
+                    .output()
+            })
             .unwrap();
-        let mut file = File::create(&out_dir_path).unwrap();
-        file.write_all(&output.stdout).unwrap();
+        let file = File::create(&out_dir_path).unwrap();
+        let mut gz = GzBuilder::new()
+            .comment(output.stdout.len().to_string().into_bytes())
+            .write(file, Compression::default());
+        gz.write_all(&output.stdout).unwrap();
+        gz.finish().unwrap();
         out_dir_path.pop();
 
-        out_dir_path.push("meli-themes.txt");
+        out_dir_path.push("meli-themes.txt.gz");
         let output = Command::new("mandoc")
             .args(MANDOC_OPTS)
-            .arg("meli-themes.5")
+            .arg("docs/meli-themes.5")
             .output()
-            .or_else(|_| Command::new("man").arg("-l").arg("meli-themes.5").output())
+            .or_else(|_| {
+                Command::new("man")
+                    .arg("-l")
+                    .arg("docs/meli-themes.5")
+                    .output()
+            })
             .unwrap();
-        let mut file = File::create(&out_dir_path).unwrap();
-        file.write_all(&output.stdout).unwrap();
+        let file = File::create(&out_dir_path).unwrap();
+        let mut gz = GzBuilder::new()
+            .comment(output.stdout.len().to_string().into_bytes())
+            .write(file, Compression::default());
+        gz.write_all(&output.stdout).unwrap();
+        gz.finish().unwrap();
     }
 }

config_macros.rs

@@ -87,6 +87,14 @@ use super::*;
                 }
                 let override_ident: syn::Ident = format_ident!("{}Override", s.ident);
                 let mut field_tokentrees = vec![];
+                let mut attrs_tokens = vec![];
+                for attr in &s.attrs {
+                    if let Ok(syn::Meta::List(ml)) = attr.parse_meta() {
+                        if ml.path.get_ident().is_some() && ml.path.get_ident().unwrap() == "cfg" {
+                            attrs_tokens.push(attr);
+                        }
+                    }
+                }
                 let mut field_idents = vec![];
                 for f in &s.fields {
                     let ident = &f.ident;
@@ -146,6 +154,7 @@ use super::*;
                 //let fields = &s.fields;
 
                 let literal_struct = quote! {
+                    #(#attrs_tokens)*
                     #[derive(Debug, Serialize, Deserialize, Clone)]
                     #[serde(deny_unknown_fields)]
                     pub struct #override_ident {
@@ -153,6 +162,7 @@ use super::*;
                     }
 
 
+                    #(#attrs_tokens)*
                     impl Default for #override_ident {
                         fn default() -> Self {
                             #override_ident {

contrib/oauth2.py

@@ -0,0 +1,348 @@
+#!/usr/bin/env python3
+#
+# Copyright 2012 Google Inc.
+# Copyright 2020 Manos Pitsidianakis
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+     # http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Performs client tasks for testing IMAP OAuth2 authentication.
+
+To use this script, you'll need to have registered with Google as an OAuth
+application and obtained an OAuth client ID and client secret.
+See https://developers.google.com/identity/protocols/OAuth2 for instructions on
+registering and for documentation of the APIs invoked by this code.
+
+This script has 3 modes of operation.
+
+1. The first mode is used to generate and authorize an OAuth2 token, the
+first step in logging in via OAuth2.
+
+  oauth2 --user=xxx@gmail.com \
+    --client_id=1038[...].apps.googleusercontent.com \
+    --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ \
+    --generate_oauth2_token
+
+The script will converse with Google and generate an oauth request
+token, then present you with a URL you should visit in your browser to
+authorize the token. Once you get the verification code from the Google
+website, enter it into the script to get your OAuth access token. The output
+from this command will contain the access token, a refresh token, and some
+metadata about the tokens. The access token can be used until it expires, and
+the refresh token lasts indefinitely, so you should record these values for
+reuse.
+
+2. The script will generate new access tokens using a refresh token.
+
+  oauth2 --user=xxx@gmail.com \
+    --client_id=1038[...].apps.googleusercontent.com \
+    --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ \
+    --refresh_token=1/Yzm6MRy4q1xi7Dx2DuWXNgT6s37OrP_DW_IoyTum4YA
+
+3. The script will generate an OAuth2 string that can be fed
+directly to IMAP or SMTP. This is triggered with the --generate_oauth2_string
+option.
+
+  oauth2 --generate_oauth2_string --user=xxx@gmail.com \
+    --access_token=ya29.AGy[...]ezLg
+
+The output of this mode will be a base64-encoded string. To use it, connect to a
+IMAPFE and pass it as the second argument to the AUTHENTICATE command.
+
+  a AUTHENTICATE XOAUTH2 a9sha9sfs[...]9dfja929dk==
+"""
+
+import base64
+import imaplib
+import json
+from optparse import OptionParser
+import smtplib
+import sys
+import urllib.request, urllib.parse, urllib.error
+
+
+def SetupOptionParser():
+  # Usage message is the module's docstring.
+  parser = OptionParser(usage=__doc__)
+  parser.add_option('--generate_oauth2_token',
+                    action='store_true',
+                    dest='generate_oauth2_token',
+                    help='generates an OAuth2 token for testing')
+  parser.add_option('--generate_oauth2_string',
+                    action='store_true',
+                    dest='generate_oauth2_string',
+                    help='generates an initial client response string for '
+                         'OAuth2')
+  parser.add_option('--client_id',
+                    default=None,
+                    help='Client ID of the application that is authenticating. '
+                         'See OAuth2 documentation for details.')
+  parser.add_option('--client_secret',
+                    default=None,
+                    help='Client secret of the application that is '
+                         'authenticating. See OAuth2 documentation for '
+                         'details.')
+  parser.add_option('--access_token',
+                    default=None,
+                    help='OAuth2 access token')
+  parser.add_option('--refresh_token',
+                    default=None,
+                    help='OAuth2 refresh token')
+  parser.add_option('--scope',
+                    default='https://mail.google.com/',
+                    help='scope for the access token. Multiple scopes can be '
+                         'listed separated by spaces with the whole argument '
+                         'quoted.')
+  parser.add_option('--test_imap_authentication',
+                    action='store_true',
+                    dest='test_imap_authentication',
+                    help='attempts to authenticate to IMAP')
+  parser.add_option('--test_smtp_authentication',
+                    action='store_true',
+                    dest='test_smtp_authentication',
+                    help='attempts to authenticate to SMTP')
+  parser.add_option('--user',
+                    default=None,
+                    help='email address of user whose account is being '
+                         'accessed')
+  parser.add_option('--quiet',
+                    action='store_true',
+                    default=False,
+                    dest='quiet',
+                    help='Omit verbose descriptions and only print '
+                         'machine-readable outputs.')
+  return parser
+
+
+# The URL root for accessing Google Accounts.
+GOOGLE_ACCOUNTS_BASE_URL = 'https://accounts.google.com'
+
+
+# Hardcoded dummy redirect URI for non-web apps.
+REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob'
+
+
+def AccountsUrl(command):
+  """Generates the Google Accounts URL.
+
+  Args:
+    command: The command to execute.
+
+  Returns:
+    A URL for the given command.
+  """
+  return '%s/%s' % (GOOGLE_ACCOUNTS_BASE_URL, command)
+
+
+def UrlEscape(text):
+  # See OAUTH 5.1 for a definition of which characters need to be escaped.
+  return urllib.parse.quote(text, safe='~-._')
+
+
+def UrlUnescape(text):
+  # See OAUTH 5.1 for a definition of which characters need to be escaped.
+  return urllib.parse.unquote(text)
+
+
+def FormatUrlParams(params):
+  """Formats parameters into a URL query string.
+
+  Args:
+    params: A key-value map.
+
+  Returns:
+    A URL query string version of the given parameters.
+  """
+  param_fragments = []
+  for param in sorted(iter(params.items()), key=lambda x: x[0]):
+    param_fragments.append('%s=%s' % (param[0], UrlEscape(param[1])))
+  return '&'.join(param_fragments)
+
+
+def GeneratePermissionUrl(client_id, scope='https://mail.google.com/'):
+  """Generates the URL for authorizing access.
+
+  This uses the "OAuth2 for Installed Applications" flow described at
+  https://developers.google.com/accounts/docs/OAuth2InstalledApp
+
+  Args:
+    client_id: Client ID obtained by registering your app.
+    scope: scope for access token, e.g. 'https://mail.google.com'
+  Returns:
+    A URL that the user should visit in their browser.
+  """
+  params = {}
+  params['client_id'] = client_id
+  params['redirect_uri'] = REDIRECT_URI
+  params['scope'] = scope
+  params['response_type'] = 'code'
+  return '%s?%s' % (AccountsUrl('o/oauth2/auth'),
+                    FormatUrlParams(params))
+
+
+def AuthorizeTokens(client_id, client_secret, authorization_code):
+  """Obtains OAuth access token and refresh token.
+
+  This uses the application portion of the "OAuth2 for Installed Applications"
+  flow at https://developers.google.com/accounts/docs/OAuth2InstalledApp#handlingtheresponse
+
+  Args:
+    client_id: Client ID obtained by registering your app.
+    client_secret: Client secret obtained by registering your app.
+    authorization_code: code generated by Google Accounts after user grants
+        permission.
+  Returns:
+    The decoded response from the Google Accounts server, as a dict. Expected
+    fields include 'access_token', 'expires_in', and 'refresh_token'.
+  """
+  params = {}
+  params['client_id'] = client_id
+  params['client_secret'] = client_secret
+  params['code'] = authorization_code
+  params['redirect_uri'] = REDIRECT_URI
+  params['grant_type'] = 'authorization_code'
+  request_url = AccountsUrl('o/oauth2/token')
+
+  response = urllib.request.urlopen(request_url, urllib.parse.urlencode(params).encode()).read()
+  return json.loads(response)
+
+
+def RefreshToken(client_id, client_secret, refresh_token):
+  """Obtains a new token given a refresh token.
+
+  See https://developers.google.com/accounts/docs/OAuth2InstalledApp#refresh
+
+  Args:
+    client_id: Client ID obtained by registering your app.
+    client_secret: Client secret obtained by registering your app.
+    refresh_token: A previously-obtained refresh token.
+  Returns:
+    The decoded response from the Google Accounts server, as a dict. Expected
+    fields include 'access_token', 'expires_in', and 'refresh_token'.
+  """
+  params = {}
+  params['client_id'] = client_id
+  params['client_secret'] = client_secret
+  params['refresh_token'] = refresh_token
+  params['grant_type'] = 'refresh_token'
+  request_url = AccountsUrl('o/oauth2/token')
+
+  response = urllib.request.urlopen(request_url, urllib.parse.urlencode(params).encode()).read()
+  return json.loads(response)
+
+
+def GenerateOAuth2String(username, access_token, base64_encode=True):
+  """Generates an IMAP OAuth2 authentication string.
+
+  See https://developers.google.com/google-apps/gmail/oauth2_overview
+
+  Args:
+    username: the username (email address) of the account to authenticate
+    access_token: An OAuth2 access token.
+    base64_encode: Whether to base64-encode the output.
+
+  Returns:
+    The SASL argument for the OAuth2 mechanism.
+  """
+  auth_string = 'user=%s\1auth=Bearer %s\1\1' % (username, access_token)
+  if base64_encode:
+    auth_string = base64.b64encode(bytes(auth_string, 'utf-8'))
+  return auth_string
+
+
+def TestImapAuthentication(user, auth_string):
+  """Authenticates to IMAP with the given auth_string.
+
+  Prints a debug trace of the attempted IMAP connection.
+
+  Args:
+    user: The Gmail username (full email address)
+    auth_string: A valid OAuth2 string, as returned by GenerateOAuth2String.
+        Must not be base64-encoded, since imaplib does its own base64-encoding.
+  """
+  print()
+  imap_conn = imaplib.IMAP4_SSL('imap.gmail.com')
+  imap_conn.debug = 4
+  imap_conn.authenticate('XOAUTH2', lambda x: auth_string)
+  imap_conn.select('INBOX')
+
+
+def TestSmtpAuthentication(user, auth_string):
+  """Authenticates to SMTP with the given auth_string.
+
+  Args:
+    user: The Gmail username (full email address)
+    auth_string: A valid OAuth2 string, not base64-encoded, as returned by
+        GenerateOAuth2String.
+  """
+  print()
+  smtp_conn = smtplib.SMTP('smtp.gmail.com', 587)
+  smtp_conn.set_debuglevel(True)
+  smtp_conn.ehlo('test')
+  smtp_conn.starttls()
+  smtp_conn.docmd('AUTH', 'XOAUTH2 ' + base64.b64encode(auth_string))
+
+
+def RequireOptions(options, *args):
+  missing = [arg for arg in args if getattr(options, arg) is None]
+  if missing:
+    print('Missing options: %s' % ' '.join(missing), file=sys.stderr)
+    sys.exit(-1)
+
+
+def main(argv):
+  options_parser = SetupOptionParser()
+  (options, args) = options_parser.parse_args()
+  if options.refresh_token:
+    RequireOptions(options, 'client_id', 'client_secret')
+    response = RefreshToken(options.client_id, options.client_secret,
+                            options.refresh_token)
+    if options.quiet:
+      print(response['access_token'])
+    else:
+      print('Access Token: %s' % response['access_token'])
+      print('Access Token Expiration Seconds: %s' % response['expires_in'])
+  elif options.generate_oauth2_string:
+    RequireOptions(options, 'user', 'access_token')
+    oauth2_string = GenerateOAuth2String(options.user, options.access_token)
+    if options.quiet:
+      print(oauth2_string.decode('utf-8'))
+    else:
+      print('OAuth2 argument:\n' + oauth2_string.decode('utf-8'))
+  elif options.generate_oauth2_token:
+    RequireOptions(options, 'client_id', 'client_secret')
+    print('To authorize token, visit this url and follow the directions:')
+    print('  %s' % GeneratePermissionUrl(options.client_id, options.scope))
+    authorization_code = input('Enter verification code: ')
+    response = AuthorizeTokens(options.client_id, options.client_secret,
+                                authorization_code)
+    print('Refresh Token: %s' % response['refresh_token'])
+    print('Access Token: %s' % response['access_token'])
+    print('Access Token Expiration Seconds: %s' % response['expires_in'])
+  elif options.test_imap_authentication:
+    RequireOptions(options, 'user', 'access_token')
+    TestImapAuthentication(options.user,
+        GenerateOAuth2String(options.user, options.access_token,
+                             base64_encode=False))
+  elif options.test_smtp_authentication:
+    RequireOptions(options, 'user', 'access_token')
+    TestSmtpAuthentication(options.user,
+        GenerateOAuth2String(options.user, options.access_token,
+                             base64_encode=False))
+  else:
+    options_parser.print_help()
+    print('Nothing to do, exiting.')
+    return
+
+
+if __name__ == '__main__':
+  main(sys.argv)

debian/changelog

@@ -1,3 +1,25 @@
+meli (0.6.2-1) buster; urgency=low
+
+  Added
+  - Add customizable mailbox tree in sidebar
+  - Make `dbus` dependency opt-out (feature is `dbus-notifications`)
+  - Implemented JMAP async, search, tagging, syncing
+  - Preserve account order from configuration file
+  - Implemented IMAP `CONDSTORE` support for IMAP cache
+  - Add `timeout` setting for IMAP
+  - Implement TCP keepalive for IMAP
+  - Rewrote email address parsers.
+  - Implement `copy_messages` for maildir
+  - Implement selection with motions
+
+  Fixed
+  - Fixed various problems with IMAP cache
+  - Fixed various problems with IMAP message counts
+  - Fixed various problems with IMAP connection hanging
+  - Fixed IMAP not reconnecting on dropped IDLE connections
+  - Fixed various problems with notmuch backend
+
+ -- Manos Pitsidianakis <epilys@nessuent.xyz>  Thu, 24 Sep 2020 18:14:00 +0200
 meli (0.6.1-1) buster; urgency=low
 
   * added experimental NNTP backend

debian/meli.docs

@@ -1,3 +1,3 @@
-meli.1
-meli.conf.5
-meli-themes.5
+docs/meli.1
+docs/meli.conf.5
+docs/meli-themes.5

docs/meli-themes.5

@@ -193,6 +193,8 @@ widgets.options.highlighted
 .It
 mail.sidebar
 .It
+mail.sidebar_divider
+.It
 mail.sidebar_unread_count
 .It
 mail.sidebar_index
@@ -261,6 +263,10 @@ mail.listing.conversations.selected
 .It
 mail.view.headers
 .It
+mail.view.headers_names
+.It
+mail.view.headers_area
+.It
 mail.view.body
 .It
 mail.view.thread.indentation.a
@@ -600,7 +606,7 @@ Yellow6:148:_:Grey93:255
 .Xr meli 1 ,
 .Xr meli.conf 5
 .Sh CONFORMING TO
-TOML Standard v.0.5.0 https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.5.0.md
+TOML Standard v.0.5.0 https://toml.io/en/v0.5.0
 .sp
 https://no-color.org/
 .Sh AUTHORS

docs/meli.1

@@ -171,7 +171,7 @@ To search in specific fields, prepend your search keyword with "field:" like so:
 .Pp
 .D1 not ((from:unrealistic and (to:complex or not "query")) or flags:seen,draft)
 .Pp
-.D1 alladdresses:mailing@list.tld and cc:me@domain.tld
+.D1 alladdresses:mailing@example.com and cc:me@example.com
 .Pp
 Boolean operators are
 .Em or Ns
@@ -360,7 +360,9 @@ is the default mode
 .It COMMAND
 commands are issued in
 .Em COMMAND
-mode, by default started with Space and exited with
+mode, by default started with
+.Cm \&:
+and exited with
 .Cm Esc
 key.
 .It EMBED
@@ -395,15 +397,29 @@ where
 is a mailbox prefixed with the
 .Ar n
 number in the side menu for the current account
-.It Cm toggle_thread_snooze
+.It Cm toggle thread_snooze
 don't issue notifications for thread under cursor in thread listing
 .It Cm search Ar STRING
 search mailbox with
 .Ar STRING
-key.
-Escape exits search results
-.It Cm set read, set unread
-Set read status of message.
+query.
+Escape exits search results.
+.It Cm select Ar STRING
+select threads matching
+.Ar STRING
+query.
+.It Cm set seen, set unseen
+Set seen status of message.
+.It Cm import Ar FILEPATH Ar MAILBOX_PATH
+Import mail from file into given mailbox.
+.It Cm copyto, moveto Ar MAILBOX_PATH
+Copy or move to other mailbox.
+.It Cm copyto, moveto Ar ACCOUNT Ar MAILBOX_PATH
+Copy or move to another account's  mailbox.
+.It Cm delete
+Delete selected threads.
+.It Cm export-mbox Ar FILEPATH
+Export selected threads to mboxcl2 file.
 .It Cm create-mailbox Ar ACCOUNT Ar MAILBOX_PATH
 create mailbox with given path.
 Be careful with backends and separator sensitivity (eg IMAP)
@@ -439,6 +455,16 @@ as an attachment
 in composer, pipe
 .Ar CMD Ar ARGS
 output into an attachment
+.It Cm add-attachment-file-picker
+Launch command defined in the configuration value
+.Ic file_picker_command
+in
+.Xr meli.conf 5 TERMINAL
+.It Cm add-attachment-file-picker < Ar CMD Ar ARGS
+Launch command
+.Ar CMD Ar ARGS Ns
+\&.
+The command should print file paths in stderr, separated by NULL bytes.
 .It Cm remove-attachment Ar INDEX
 remove attachment with given index
 .It Cm toggle sign
@@ -464,6 +490,15 @@ to
 .It Cm printenv Ar KEY
 print environment variable
 .Ar KEY
+.It Cm quit
+Quits
+.Nm Ns
+\&.
+.It Cm reload-config
+Reloads configuration but only if account configuration is unchanged.
+Useful if you want to reload some settings without restarting
+.Nm Ns
+\&.
 .El
 .Sh SHORTCUTS
 See

docs/meli.conf.5

@@ -78,7 +78,7 @@ example configuration
 root_mailbox = "/path/to/root/folder"
 format = "Maildir"
 index_style = "Compact"
-identity="email@address.tld"
+identity="email@example.com"
 subscribed_mailboxes = ["folder", "folder/Sent"] # or [ "*", ] for all mailboxes
 display_name = "Name"
 
@@ -107,7 +107,7 @@ script = "notify-send"
 [composing]
 # required for sending e-mail
 send_mail = 'msmtp --read-recipients --read-envelope-from'
-#send_mail = { hostname = "smtp.mail.tld", port = 587, auth = { type = "auto", username = "user", password = { type = "command_eval", value = "gpg2 --no-tty -q -d ~/.passwords/user.gpg" } }, security = { type = "STARTTLS" } }
+#send_mail = { hostname = "smtp.example.com", port = 587, auth = { type = "auto", username = "user", password = { type = "command_eval", value = "gpg2 --no-tty -q -d ~/.passwords/user.gpg" } }, security = { type = "STARTTLS" } }
 editor_command = 'vim +/^$'
 
 [shortcuts]
@@ -152,7 +152,7 @@ plain:shows one row per mail, regardless of threading
 .Bl -tag -width 36n
 .It Ic display_name Ar String
 .Pq Em optional
-A name which can be combined with your address: "Name <email@address.tld>".
+A name which can be combined with your address: "Name <email@example.com>".
 .It Ic read_only Ar boolean
 Attempt to not make any changes to this account.
 .Pq Em false
@@ -199,14 +199,14 @@ format = "notmuch"
   [accounts.notmuch.mailboxes]
   "INBOX" = {  query="tag:inbox", subscribe = true }
   "Drafts" = {  query="tag:draft", subscribe = true }
-  "Sent" = {  query="from:username@server.tld from:username2@server.tld", subscribe = true }
+  "Sent" = {  query="from:username@example.com from:username2@example.com", subscribe = true }
 .Ed
 .Ss IMAP only
 IMAP specific options are:
 .Bl -tag -width 36n
 .It Ic server_hostname Ar String
 example:
-.Qq mail.example.tld
+.Qq mail.example.com
 .It Ic server_username Ar String
 Server username
 .It Ic server_password Ar String
@@ -235,11 +235,25 @@ Do not validate TLS certificates.
 Use IDLE extension.
 .\" default value
 .Pq Em true
+.It Ic use_condstore Ar boolean
+.Pq Em optional
+Use CONDSTORE extension.
+.\" default value
+.Pq Em true
 .It Ic use_deflate Ar boolean
 .Pq Em optional
 Use COMPRESS=DEFLATE extension (if built with DEFLATE support).
 .\" default value
 .Pq Em true
+.It Ic use_oauth2 Ar boolean
+.Pq Em optional
+Use OAUTH2 authentication.
+Can only be used with
+.Ic server_password_command
+which should return a base64-encoded OAUTH2 token ready to be passed to IMAP.
+For help on setup with Gmail, see Gmail section below.
+.\" default value
+.Pq Em false
 .It Ic timeout Ar integer
 .Pq Em optional
 Timeout to use for server connections in seconds.
@@ -247,12 +261,41 @@ A timeout of 0 seconds means there's no timeout.
 .\" default value
 .Pq Em 16
 .El
+.Ss Gmail
+Gmail has non-standard IMAP behaviors that need to be worked around.
+.Ss Gmail - sending mail
+Option
+.Ic store_sent_mail
+should be disabled since Gmail auto-saves sent mail by its own.
+.Ss Gmail OAUTH2
+To use OAUTH2, you must go through a process to register your own private "application" with Google that can use OAUTH2 tokens.
+For convenience in the meli repository under the
+.Pa contrib/
+directory you can find a python3 file named oauth2.py to generate and request the appropriate data to perform OAUTH2 authentication.
+Steps:
+.Bl -bullet -compact
+.It
+In Google APIs, create a custom OAuth client ID and note down the Client ID and Client Secret.
+You may need to create a consent screen; follow the steps described in the website.
+.It
+Run the oauth2.py script as follows (after adjusting binary paths and credentials):
+.Cm python3 oauth2.py --user=xxx@gmail.com --client_id=1038[...].apps.googleusercontent.com --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ --generate_oauth2_token
+and follow the instructions.
+Note down the refresh token.
+.It
+In
+.Ic server_password_command
+enter a command like this (after adjusting binary paths and credentials):
+.Cm TOKEN=$(python3 oauth2.py --user=xxx@gmail.com --quiet --client_id=1038[...].apps.googleusercontent.com --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ --refresh_token=1/Yzm6MRy4q1xi7Dx2DuWXNgT6s37OrP_DW_IoyTum4YA) && python3 oauth2.py --user=xxx@gmail.com --generate_oauth2_string --quiet --access_token=$TOKEN
+.It
+On startup, meli should evaluate this command which if successful must only return a base64-encoded token ready to be passed to IMAP.
+.El
 .Ss JMAP only
 JMAP specific options
 .Bl -tag -width 36n
 .It Ic server_hostname Ar String
 example:
-.Qq mail.example.tld
+.Qq mail.example.com
 .It Ic server_username Ar String
 Server username
 .It Ic server_password Ar String
@@ -360,9 +403,9 @@ and
 \&.
 Example:
 .Bd -literal
-[accounts."imap.domain.tld".mailboxes."INBOX"]
+[accounts."imap.example.com".mailboxes."INBOX"]
   index_style = "plain"
-  [accounts."imap.domain.tld".mailboxes."INBOX".pager]
+  [accounts."imap.example.com".mailboxes."INBOX".pager]
     filter = ""
 .Ed
 .El
@@ -397,6 +440,37 @@ Add meli User-Agent header in new drafts
 .Pq Em true
 .It Ic default_header_values Ar hash table String[String]
 Default header values used when creating a new draft.
+.It Ic store_sent_mail Ar boolean
+.Pq Em optional
+Store sent mail after successful submission.
+This setting is meant to be disabled for non-standard behaviour in gmail, which auto-saves sent mail on its own.
+.\" default value
+.Pq Em true
+.It Ic attribution_format_string Ar String
+.Pq Em optional
+The attribution line appears above the quoted reply text.
+The format specifiers for the replied address are:
+.Bl -bullet -compact
+.It
+.Li %+f
+— the sender's name and email address.
+.It
+.Li %+n
+— the sender's name (or email address, if no name is included).
+.It
+.Li %+a
+— the sender's email address.
+.El
+The format string is passed to
+.Xr strftime 3
+with the replied envelope's date.
+.\" default value
+.Pq Em "On %a, %0e %b %Y %H:%M, %+f wrote:%n"
+.It Ic attribution_use_posix_locale Ar boolean
+.Pq Em optional
+Whether the strftime call for the attribution string uses the POSIX locale instead of the user's active locale.
+.\" default value
+.Pq Em true
 .El
 .Sh SHORTCUTS
 Shortcuts can take the following values:
@@ -466,12 +540,16 @@ exit_thread = 'i'
 Toggle help and shortcuts view.
 .\" default value
 .Pq Em \&?
+.It Ic quit
+Quit application.
+.\" default value
+.Pq Ql Em q
 .It Ic enter_command_mode
 Enter
 .Em COMMAND
 mode.
 .\" default value
-.Pq Ql Em \ 
+.Pq Ql Em \&:
 .It Ic next_tab
 Go to next tab.
 .\" default value
@@ -787,19 +865,27 @@ Sets the string to print in the mailbox tree for a leaf level where its root has
 .It Ic sidebar_mailbox_tree_no_sibling_leaf Ar String
 .Pq Em optional
 Sets the string to print in the mailbox tree for a leaf level where its root has no sibling.
+.It Ic sidebar_divider Ar char
+.Pq Em optional
+Sets the character to print as the divider between the accounts list and the message list.
+.It Ic show_menu_scrollbar Ar boolean
+.Pq Em optional
+ Show auto-hiding scrollbar in accounts sidebar menu.
+.\" default value
+.Pq Em true
 .El
 .Ss Examples of sidebar mailbox tree customization
 The default values
-
-.Bd
+.sp
+.Bd -literal
  has_sibling = " "
  no_sibling = " ";
  has_sibling_leaf = " "
  no_sibling_leaf = " "
 .Ed
-
+.sp
 render a mailbox tree like the following:
-
+.sp
 .Bd -literal
 0  Inbox 3
 1   Archive
@@ -807,20 +893,20 @@ render a mailbox tree like the following:
 3   Lists
 4    example-list-a
 5    example-list-b
-6   Sent 
+6   Sent
 7   Spam
 8   Trash
 .Ed
-
+.sp
 Other possible trees:
-
+.sp
 .Bd -literal
 has_sibling = " ┃"
 no_sibling = "  "
 has_sibling_leaf = " ┣━"
 no_sibling_leaf = " ┗━"
 .Ed
-
+.sp
 .Bd -literal
 0  Inbox 3
 1   ┣━Archive
@@ -828,20 +914,20 @@ no_sibling_leaf = " ┗━"
 3   ┣━Lists
 4   ┃ ┣━example-list-a
 5   ┃ ┗━example-list-b
-6   ┣━Sent 
+6   ┣━Sent
 7   ┣━Spam
 8   ┗━Trash
 .Ed
-
+.sp
 A completely ASCII one:
-
+.sp
 .Bd -literal
 has_sibling = " |"
 no_sibling = "  "
 has_sibling_leaf = " |\\_"
 no_sibling_leaf = " \\_"
 .Ed
-
+.sp
 .Bd -literal
 0  Inbox 3
 1   |\\_Archive
@@ -849,11 +935,11 @@ no_sibling_leaf = " \\_"
 3   |\\_Lists
 4   | |\\_example-list-a
 5   |  \\_example-list-b
-6   |\\_Sent 
+6   |\\_Sent
 7   |\\_Spam
 8    \\_Trash
 .Ed
-
+.sp
 .Sh TAGS
 .Bl -tag -width 36n
 .It Ic colours Ar hash table String[Color]
@@ -892,11 +978,6 @@ Always sign sent messages
 Key to be used when signing/encrypting (not functional yet)
 .\" default value
 .Pq Em none
-.It Ic gpg_binary Ar String
-.Pq Em optional
-The gpg binary name or file location to use
-.\" default value
-.Pq Em "gpg2"
 .El
 .Sh TERMINAL
 .Bl -tag -width 36n
@@ -920,6 +1001,14 @@ If false, no ANSI colors are used.
 Set window title in xterm compatible terminals An empty string means no window title is set.
 .\" default value
 .Pq Em "meli"
+.It Ic file_picker_command Ar String
+.Pq Em optional
+Set command that prints file paths in stderr, separated by NULL bytes.
+Used with
+.Ic add-attachment-file-picker
+when composing new mail.
+.\" default value
+.Pq Em None
 .It Ic themes Ar hash table String[String[Attribute]]
 Define UI themes.
 See
@@ -938,6 +1027,80 @@ theme = "themeB"
 [terminal.themes.themeC]
 \&...
 .Ed
+.It Ic use_mouse Ar bool
+Use mouse events.
+This will disable text selection, but you will be able to resize some widgets.
+This setting can be toggled with
+.Cm toggle mouse Ns
+\&.
+.\" default value
+.Pq Em false
+.It Ic mouse_flag Ar String
+String to show in status bar if mouse is active.
+.\" default value
+.Pq Em 🖱️
+.It Ic progress_spinner_sequence Ar Either \&< Integer, ProgressSpinner \&>
+Choose between 37 built in sequences (integers between 0-36) or define your own list of strings for the progress spinner animation.
+Set to an empty array to disable the progress spinner.
+.\" default value
+.Pq Em 20
+.Pp
+Builtin sequences are:
+.Bd -literal
+0   ["-", "\\", "|", "/"]
+1   ["▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"]
+2   ["⣀", "⣄", "⣤", "⣦", "⣶", "⣷", "⣿"]
+3   ["⣀", "⣄", "⣆", "⣇", "⣧", "⣷", "⣿"]
+4   ["○", "◔", "◐", "◕", "⬤"]
+5   ["□", "◱", "◧", "▣", "■"]
+6   ["□", "◱", "▨", "▩", "■"]
+7   ["□", "◱", "▥", "▦", "■"]
+8   ["░", "▒", "▓", "█"]
+9   ["░", "█"]
+10  ["⬜", "⬛"]
+11  ["▱", "▰"]
+12  ["▭", "◼"]
+13  ["▯", "▮"]
+14  ["◯", "⬤"]
+15  ["⚪", "⚫"]
+16  ["▖", "▗", "▘", "▝", "▞", "▚", "▙", "▟", "▜", "▛"]
+17  ["|", "/", "-", "\\"]
+18  [".", "o", "O", "@", "*"]
+19  ["◡◡", "⊙⊙", "◠◠", "⊙⊙"]
+20  ["◜ ", " ◝", " ◞", "◟ "]
+21  ["←", "↖", "↑", "↗", "→", "↘", "↓", "↙"]
+22  ["▁", "▃", "▄", "▅", "▆", "▇", "█", "▇", "▆", "▅", "▄", "▃"]
+23  [ "▉", "▊", "▋", "▌", "▍", "▎", "▏", "▎", "▍", "▌", "▋", "▊", "▉" ]
+24  ["▖", "▘", "▝", "▗"]
+25  ["▌", "▀", "▐", "▄"]
+26  ["┤", "┘", "┴", "└", "├", "┌", "┬", "┐"]
+27  ["◢", "◣", "◤", "◥"]
+28  ["⠁", "⠂", "⠄", "⡀", "⢀", "⠠", "⠐", "⠈"]
+29  ["⢎⡰", "⢎⡡", "⢎⡑", "⢎⠱", "⠎⡱", "⢊⡱", "⢌⡱", "⢆⡱"]
+30  [".", "o", "O", "°", "O", "o", "."]
+31  ["㊂", "㊀", "㊁"]
+32  ["💛 ", "💙 ", "💜 ", "💚 ", "❤️ "]
+33  [ "🕛 ", "🕐 ", "🕑 ", "🕒 ", "🕓 ", "🕔 ", "🕕 ", "🕖 ", "🕗 ", "🕘 ", "🕙 ", "🕚 " ]
+34  ["🌍 ", "🌎 ", "🌏 "]
+35  [ "[    ]", "[=   ]", "[==  ]", "[=== ]", "[ ===]", "[  ==]", "[   =]", "[    ]", "[   =]", "[  ==]", "[ ===]", "[====]", "[=== ]", "[==  ]", "[=   ]" ]
+36  ["🌑 ", "🌒 ", "🌓 ", "🌔 ", "🌕 ", "🌖 ", "🌗 ", "🌘 "]
+.Ed
+.Pp
+Or, define an array of strings each consisting of a frame in the progress sequence indicator for a custom spinner:
+.Bl -tag -width 36n
+.It Ic interval_ms Ar u64
+.Pq Em optional
+Frame interval.
+.\" default value
+.Pq 50
+.It Ic frames Ar [String]
+The animation frames.
+.El
+.Pp
+Example:
+.Bd -literal
+progress_spinner_sequence = { interval_ms = 150, frames = [ "-", "=", "≡" ] }
+.Ed
 .El
 .Sh LOG
 .Bl -tag -width 36n
@@ -1012,25 +1175,62 @@ subsection
 .El
 .Ss SmtpAuth
 .Bl -tag -width 36n
-.It Ic type Ar "none" | "auto"
+.It Ic type Ar "none" | "auto" | "xoauth2"
 .El
 .Pp
 For type "auto":
 .Bl -tag -width 36n
 .It Ic username Ar String
-.It Ic password Ar String|SmtpPassword
+.It Ic password Ar SmtpPassword
 .It Ic require_auth Ar bool
 .Pq Em optional
 require authentication in every case
 .\" default value
 .Pq Em true
 .El
+.sp
+For type "xoauth2":
+.Bl -tag -width 36n
+.It Ic token_command Ar String
+Command to evaluate that returns an XOAUTH2 token.
+.It Ic require_auth Ar bool
+.Pq Em optional
+require authentication in every case
+.\" default value
+.Pq Em true
+.El
+.sp
+Examples:
+.Bd -literal
+auth = { type = "auto", username = "user", password = { type = "raw", value = "hunter2" } }
+.Ed
+.Bd -literal
+auth = { type = "auto", username = "user", password = "hunter2" }
+.Ed
+.Bd -literal
+auth = { type = "none" }
+.Ed
+.sp
+For Gmail (see
+.Sx Gmail OAUTH2
+for details on the authentication token command):
+.Bd -literal
+auth = { type = "xoauth2", token_command = "TOKEN=$(python3 oauth2.py --user=xxx@gmail.com --quiet --client_id=1038[...].apps.googleusercontent.com --client_secret=[..] --refresh_token=[..] && python3 oauth2.py --user=xxx@gmail.com --generate_oauth2_string --quiet --access_token=$TOKEN" }
+.Ed
 .Ss SmtpPassword
 .Bl -tag -width 36n
 .It Ic type Ar "raw" | "command_evaluation"
 .It Ic value Ar String
 Either a raw password string, or command to execute.
 .El
+.sp
+Examples:
+.Bd -literal
+password = { type = "raw", value = "hunter2" }
+.Ed
+.Bd -literal
+password = { type = "command_eval", value = "gpg2 --no-tty -q -d ~/.passwords/user.gpg" }
+.Ed
 .Ss SmtpSecurity
 Default security type is
 .Em auto Ns

docs/samples/sample-config.toml

@@ -12,7 +12,7 @@
 #root_mailbox = "/path/to/root/mailbox"
 #format = "Maildir"
 #index_style = "Conversations" # or [plain, threaded, compact]
-#identity="email@address.tld"
+#identity="email@example.com"
 #display_name = "Name"
 #subscribed_mailboxes = ["INBOX", "INBOX/Sent", "INBOX/Drafts", "INBOX/Junk"]
 #
@@ -33,14 +33,14 @@
 #[accounts."imap"]
 #root_mailbox = "INBOX"
 #format = "imap"
-#server_hostname="mail.server.tld"
+#server_hostname="mail.example.com"
 #server_password="pha2hiLohs2eeeish2phaii1We3ood4chakaiv0hien2ahie3m"
-#server_username="username@server.tld"
+#server_username="username@example.com"
 #server_port="993" # imaps
 #server_port="143" # STARTTLS
 #use_starttls=true #optional
 #index_style = "Conversations"
-#identity = "username@server.tld"
+#identity = "username@example.com"
 #display_name = "Name Name"
 ### match every mailbox:
 #subscribed_mailboxes = ["*" ]
@@ -52,14 +52,32 @@
 #root_mailbox = "/path/to/folder" # where .notmuch/ directory is located
 #format = "notmuch"
 #index_style = "conversations"
-#identity="username@server.tld"
+#identity="username@example.com"
 #display_name = "Name Name"
 #  # notmuch mailboxes are virtual, they are defined by their alias and the notmuch query that corresponds to their content.
 #  [accounts.notmuch.mailboxes]
 #  "INBOX" = {  query="tag:inbox", subscribe = true }
 #  "Drafts" = {  query="tag:draft", subscribe = true }
-#  "Sent" = {  query="from:username@server.tld from:username2@server.tld", subscribe = true }
-##
+#  "Sent" = {  query="from:username@example.com from:username2@example.com", subscribe = true }
+#
+## Setting up a Gmail account
+#[accounts."gmail"]
+#root_mailbox = '[Gmail]'
+#format = "imap"
+#server_hostname='imap.gmail.com'
+#server_password="password"
+#server_username="username@gmail.com"
+#server_port="993"
+#index_style = "Conversations"
+#identity = "username@gmail.com"
+#display_name = "Name Name"
+### match every mailbox:
+#subscribed_mailboxes = ["*" ]
+#composing.send_mail = 'msmtp --read-recipients --read-envelope-from'
+### Gmail auto saves sent mail to Sent folder, so don't duplicate the effort:
+#composing.store_sent_mail = false
+#
+#
 #[pager]
 #filter = "COLUMNS=72 /usr/local/bin/pygmentize -l email"
 #pager_context = 0 # default, optional
@@ -105,14 +123,13 @@
 #[composing]
 ##required for sending e-mail
 #send_mail = 'msmtp --read-recipients --read-envelope-from'
-##send_mail = { hostname = "smtp.mail.tld", port = 587, auth = { type = "auto", username = "user", password = { type = "command_eval", value = "gpg2 --no-tty -q -d ~/.passwords/user.gpg" } }, security = { type = "STARTTLS" } }
+##send_mail = { hostname = "smtp.example.com", port = 587, auth = { type = "auto", username = "user", password = { type = "command_eval", value = "gpg2 --no-tty -q -d ~/.passwords/user.gpg" } }, security = { type = "STARTTLS" } }
 #editor_command = 'vim +/^$' # optional, by default $EDITOR is used.
 #
 #
 #[pgp]
 #auto_sign = false # always sign sent messages
 #auto_verify_signatures = true # always verify signatures when reading signed e-mails
-#gpg_binary = "/usr/bin/gpg2" #optional
 #
 #[terminal]
 #theme = "dark" # or "light"

docs/samples/themes/nord.toml

@@ -0,0 +1,70 @@
+[terminal.themes.nord]
+"theme_default" = { fg = "$nord6", bg = "$nord0", attrs = "Default" }
+"mail.listing.compact.even" = { fg = "theme_default", bg = "$nord1", attrs = "theme_default" }
+"mail.listing.compact.odd" = { fg = "theme_default", bg = "$nord2", attrs = "theme_default" }
+"mail.listing.plain.even" = { fg = "theme_default", bg = "$nord1", attrs = "theme_default" }
+"mail.listing.plain.odd" = { fg = "theme_default", bg = "$nord2", attrs = "theme_default" }
+"mail.listing.compact.even_highlighted" = { fg = "$nord0", bg = "$focused_bg", attrs = "theme_default" }
+"mail.listing.compact.odd_highlighted" = { fg = "$nord0", bg = "$focused_bg", attrs = "theme_default" }
+"mail.listing.conversations.highlighted" = { fg = "$nord0", bg = "$focused_bg", attrs = "theme_default" }
+"mail.listing.conversations.selected" = { fg = "theme_default", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.conversations.subject" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.conversations.unseen" = { fg = "theme_default", bg = "$nord2", attrs = "theme_default" }
+"mail.listing.plain.even_highlighted" = { fg = "$nord0", bg = "$focused_bg", attrs = "theme_default" }
+"mail.listing.plain.odd_highlighted" = { fg = "$nord0", bg = "$focused_bg", attrs = "theme_default" }
+"mail.listing.tag_default" = { fg = "theme_default", bg = "$nord8", attrs = "theme_default" }
+"mail.sidebar_highlighted" = { fg = "$nord1", bg = "$focused_bg", attrs = "theme_default" }
+"mail.sidebar_highlighted_account" = { fg = "$nord5", bg = "$nord1", attrs = "theme_default" }
+"mail.sidebar_highlighted_account_name" = { fg = "mail.sidebar_highlighted_account", bg = "mail.sidebar_highlighted_account", attrs = "theme_default" }
+"mail.sidebar_highlighted_account_index" = { fg = "mail.sidebar_highlighted_account", bg = "mail.sidebar_highlighted_account", attrs = "theme_default" }
+"mail.sidebar_highlighted_account_unread_count" = { fg = "mail.sidebar_highlighted_account", bg = "mail.sidebar_highlighted_account", attrs = "theme_default" }
+"mail.sidebar_highlighted_index" = { fg = "mail.sidebar_index", bg = "mail.sidebar_highlighted", attrs = "theme_default" }
+"mail.sidebar_highlighted_unread_count" = { fg = "mail.sidebar_highlighted", bg = "mail.sidebar_highlighted", attrs = "theme_default" }
+"mail.sidebar" = { fg = "$nord5", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar_account_name" = { fg = "$nord5", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar_index" = { fg = "$nord1", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar_unread_count" = { fg = "$nord1", bg = "theme_default", attrs = "theme_default" }
+"mail.view.body" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.view.headers" = { fg = "$nord9", bg = "theme_default", attrs = "theme_default" }
+"mail.view.thread.indentation.a" = { fg = "theme_default", bg = "$nord11", attrs = "theme_default" }
+"mail.view.thread.indentation.b" = { fg = "theme_default", bg = "$nord12", attrs = "theme_default" }
+"mail.view.thread.indentation.c" = { fg = "theme_default", bg = "$nord13", attrs = "theme_default" }
+"mail.view.thread.indentation.d" = { fg = "theme_default", bg = "$nord14", attrs = "theme_default" }
+"mail.view.thread.indentation.e" = { fg = "theme_default", bg = "$nord15", attrs = "theme_default" }
+"mail.view.thread.indentation.f" = { fg = "theme_default", bg = "$nord13", attrs = "theme_default" }
+"pager.highlight_search" = { fg = "$nord5", bg = "$nord7", attrs = "Bold" }
+"pager.highlight_search_current" = { fg = "$nord7", bg = "$nord10", attrs = "Bold" }
+"status.bar" = { fg = "$nord5", bg = "$nord3", attrs = "theme_default" }
+"status.notification" = { fg = "$nord5", bg = "$nord3", attrs = "theme_default" }
+"tab.bar" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"tab.focused" = { fg = "$nord1", bg = "$focused_bg", attrs = "theme_default" }
+"tab.unfocused" = { fg = "$nord4", bg = "$unfocused_bg", attrs = "theme_default" }
+"widgets.form.field" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"widgets.form.highlighted" = { fg = "theme_default", bg = "$nord2", attrs = "theme_default" }
+"widgets.form.label" = { fg = "theme_default", bg = "theme_default", attrs = "Bold" }
+"widgets.list.header" = { fg = "theme_default", bg = "theme_default", attrs = "Bold" }
+"widgets.options.highlighted" = { fg = "theme_default", bg = "$nord2", attrs = "theme_default" }
+
+[terminal.themes.nord.color_aliases]
+nord0 = "#2e3440"
+nord1 = "#3b4252"
+nord2 = "#434c5e"
+nord3 = "#4c566a"
+# snow storm
+nord4 = "#d8dee9"
+nord5 = "#e5e9f0"
+nord6 = "#eceff4"
+# frost
+nord7 = "#8fbcbb"
+nord8 = "#88c0d0"
+nord9 = "#81a1c1"
+nord10 = "#5e81ac"
+# aurora
+nord11 = "#bf616a"
+nord12 = "#d08770"
+nord13 = "#ebcb8b"
+nord14 = "#a3be8c"
+nord15 = "#b48ead"
+# semantics
+focused_bg = "$nord8"
+unfocused_bg = "$nord3"

docs/samples/themes/orca.toml

@@ -51,7 +51,7 @@ color_aliases = { "neon_green" = "#6ef9d4",  "darkgrey" = "#4a4a4a",  "neon_purp
 "pager.highlight_search" = { fg = "White", bg = "Teal", attrs = "Bold" }
 "pager.highlight_search_current" = { fg = "White", bg = "NavyBlue", attrs = "Bold" }
 "status.bar" = { fg = "White", bg = "Black", attrs = "theme_default" }
-"status.notification" = { fg = "Plum1", bg = "DarkRed", attrs = "theme_default" }
+"status.notification" = { fg = "Plum1", bg = "theme_default", attrs = "theme_default" }
 "tab.bar" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
 "tab.focused" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
 "tab.unfocused" = { fg = "$darkgrey", bg = "Black", attrs = "theme_default" }

docs/samples/themes/sail.toml

@@ -0,0 +1,69 @@
+[terminal.themes.sail]
+color_aliases = { "unseen_fg" = "theme_default", "unseen_bg" = "theme_default", "sea" = "#91C7FF", "dimmed_text" = "#afbec5", "dimmed_bg" = "Grey78", "header" = "#edeff1" }
+"theme_default" = { fg = "#37474f", bg = "White", attrs = "Default" }
+"mail.listing.attachment_flag" = { fg = "Blue", bg = "theme_default", attrs = "theme_default" }
+
+"mail.listing.compact.even" = { fg = "$dimmed_text", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.compact.even_highlighted" = { fg = "theme_default", bg = "$dimmed_bg", attrs = "theme_default" }
+"mail.listing.compact.even_selected" = { fg = "$dimmed_text", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.compact.even_unseen" = { fg = "$unseen_fg", bg = "$unseen_bg", attrs = "theme_default" }
+
+"mail.listing.compact.odd" = { fg = "$dimmed_text", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.compact.odd_highlighted" = { fg = "theme_default", bg = "$dimmed_bg", attrs = "theme_default" }
+"mail.listing.compact.odd_selected" = { fg = "$dimmed_text", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.compact.odd_unseen" = { fg = "$unseen_fg", bg = "$unseen_bg", attrs = "theme_default" }
+
+"mail.listing.plain.even" = { fg = "mail.listing.compact.even", bg = "mail.listing.compact.even", attrs = "theme_default" }
+"mail.listing.plain.even_highlighted" = { fg = "mail.listing.compact.even_highlighted", bg = "mail.listing.compact.even_highlighted", attrs = "theme_default" }
+"mail.listing.plain.even_selected" = { fg = "mail.listing.compact.even_selected", bg = "mail.listing.compact.even_selected", attrs = "theme_default" }
+"mail.listing.plain.even_unseen" = { fg = "mail.listing.compact.even_unseen", bg = "mail.listing.compact.even_unseen", attrs = "theme_default" }
+"mail.listing.plain.odd" = { fg = "mail.listing.compact.odd", bg = "mail.listing.compact.odd", attrs = "theme_default" }
+"mail.listing.plain.odd_highlighted" = { fg = "mail.listing.compact.odd_highlighted", bg = "mail.listing.compact.odd_highlighted", attrs = "theme_default" }
+"mail.listing.plain.odd_selected" = { fg = "mail.listing.compact.odd_selected", bg = "mail.listing.compact.odd_selected", attrs = "theme_default" }
+"mail.listing.plain.odd_unseen" = { fg = "mail.listing.compact.odd_unseen", bg = "mail.listing.compact.odd_unseen", attrs = "theme_default" }
+
+"mail.listing.conversations" = { fg = "$dimmed_text", bg = "theme_default", attrs = "Default" }
+"mail.listing.conversations.date" = { fg = "mail.listing.conversations", bg = "mail.listing.conversations", attrs = "theme_default" }
+"mail.listing.conversations.from" = { fg = "mail.listing.conversations", bg = "mail.listing.conversations", attrs = "theme_default" }
+"mail.listing.conversations.subject" = { fg = "mail.listing.conversations", bg = "mail.listing.conversations", attrs = "theme_default" }
+"mail.listing.conversations.highlighted" = { fg = "theme_default", bg = "$dimmed_bg", attrs = "theme_default" }
+"mail.listing.conversations.selected" = { fg = "theme_default", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.conversations.unseen" = { fg = "$unseen_fg", bg = "$unseen_bg", attrs = "theme_default" }
+
+"mail.listing.tag_default" = { fg = "Black", bg = "$dimmed_text", attrs = "theme_default" }
+"mail.listing.thread_snooze_flag" = { fg = "Red", bg = "theme_default", attrs = "theme_default" }
+
+"mail.sidebar" = { fg = "$dimmed_text", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar_highlighted" = { fg = "theme_default", bg = "$dimmed_bg", attrs = "theme_default" }
+"mail.sidebar_highlighted_account" = { fg = "$dimmed_text", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar_highlighted_account_name" = { fg = "theme_default", bg = "$header", attrs = "Bold" }
+"mail.sidebar_account_name" = { fg = "mail.sidebar", bg = "mail.sidebar", attrs = "theme_default" }
+"mail.sidebar_highlighted_account_index" = { fg = "mail.sidebar_index", bg = "mail.sidebar_highlighted_account", attrs = "theme_default" }
+"mail.sidebar_highlighted_account_unread_count" = { fg = "mail.sidebar_unread_count", bg = "mail.sidebar_highlighted_account", attrs = "theme_default" }
+"mail.sidebar_highlighted_index" = { fg = "mail.sidebar_index", bg = "mail.sidebar_highlighted", attrs = "theme_default" }
+"mail.sidebar_highlighted_unread_count" = { fg = "mail.sidebar_highlighted", bg = "mail.sidebar_highlighted", attrs = "theme_default" }
+"mail.sidebar_index" = { fg = "mail.sidebar", bg = "mail.sidebar", attrs = "theme_default" }
+"mail.sidebar_unread_count" = { fg = "$dimmed_text", bg = "theme_default", attrs = "theme_default" }
+
+"mail.view.body" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.view.headers" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.view.headers_names" = { fg = "theme_default", bg = "theme_default", attrs = "Bold" }
+"mail.view.thread.indentation.a" = { fg = "theme_default", bg = "#EC633D", attrs = "theme_default" }
+"mail.view.thread.indentation.b" = { fg = "theme_default", bg = "#D347F9", attrs = "theme_default" }
+"mail.view.thread.indentation.c" = { fg = "theme_default", bg = "#317EFB", attrs = "theme_default" }
+"mail.view.thread.indentation.d" = { fg = "theme_default", bg = "#06B8CD", attrs = "theme_default" }
+"mail.view.thread.indentation.e" = { fg = "theme_default", bg = "#93DDB6", attrs = "theme_default" }
+"mail.view.thread.indentation.f" = { fg = "theme_default", bg = "#68A033", attrs = "theme_default" }
+
+"pager.highlight_search" = { fg = "White", bg = "Teal", attrs = "Bold" }
+"pager.highlight_search_current" = { fg = "White", bg = "NavyBlue", attrs = "Bold" }
+"status.bar" = { fg = "theme_default", bg = "$sea", attrs = "theme_default" }
+"status.notification" = { fg = "Plum1", bg = "theme_default", attrs = "theme_default" }
+"tab.bar" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"tab.focused" = { fg = "theme_default", bg = "theme_default", attrs = "Bold" }
+"tab.unfocused" = { fg = "White", bg = "$sea", attrs = "Bold" }
+"widgets.form.field" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"widgets.form.highlighted" = { fg = "theme_default", bg = "$dimmed_bg", attrs = "theme_default" }
+"widgets.form.label" = { fg = "theme_default", bg = "theme_default", attrs = "Bold" }
+"widgets.list.header" = { fg = "Black", bg = "White", attrs = "Bold" }
+"widgets.options.highlighted" = { fg = "theme_default", bg = "$dimmed_bg", attrs = "theme_default" }

melib/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "melib"
-version = "0.6.1"
+version = "0.6.2"
 authors = ["Manos Pitsidianakis <epilys@nessuent.xyz>"]
 workspace = ".."
 edition = "2018"
@@ -8,10 +8,11 @@ build = "build.rs"
 
 homepage = "https://meli.delivery"
 repository = "https://git.meli.delivery/meli/meli.git"
-description = "backend mail client library"
-keywords = ["mail", "mua", "maildir", "imap"]
-categories = [ "email"]
+description = "mail library"
+keywords = ["mail", "mua", "maildir", "imap", "jmap"]
+categories = [ "email", "parser-implementations"]
 license = "GPL-3.0-or-later"
+readme = "README.md"
 
 [lib]
 name = "melib"
@@ -19,10 +20,8 @@ path = "src/lib.rs"
 
 [dependencies]
 bitflags = "1.0"
-crossbeam = "0.7.2"
 data-encoding = "2.1.1"
 encoding = "0.2.33"
-memmap = { version = "0.5.2", optional = true }
 nom = { version = "5.1.1" }
 
 indexmap = { version = "^1.5", features = ["serde-1", ] }
@@ -31,39 +30,40 @@ xdg = "2.1.0"
 native-tls = { version ="0.2.3", optional=true }
 serde = { version = "1.0.71", features = ["rc", ] }
 serde_derive = "1.0.71"
-bincode = "1.2.0"
+bincode = "^1.3.0"
 uuid = { version = "0.8.1", features = ["serde", "v4", "v5"] }
 
 unicode-segmentation = { version = "1.2.1", optional = true }
 libc = {version = "0.2.59", features = ["extra_traits",]}
 isahc = { version = "0.9.7", optional = true, default-features = false, features = ["http2", "json", "text-decoding"]}
 serde_json = { version = "1.0", optional = true, features = ["raw_value",] }
-smallvec = { version = "^1.4.0", features = ["serde", ] }
+smallvec = { version = "^1.5.0", features = ["serde", ] }
 nix = "0.17.0"
 rusqlite = {version = "0.24.0", optional = true }
 
 libloading = "0.6.2"
 futures = "0.3.5"
-smol = "0.1.18"
+smol = "1.0.0"
 async-stream = "0.2.1"
 base64 = { version = "0.12.3", optional = true }
 flate2 = { version = "1.0.16", optional = true }
-xdg-utils = "0.3.0"
+xdg-utils = "^0.4.0"
 
 [features]
 default = ["unicode_algorithms", "imap_backend", "maildir_backend", "mbox_backend", "vcard", "sqlite3", "smtp", "deflate_compression"]
 
 debug-tracing = []
 deflate_compression = ["flate2", ]
+gpgme = []
 http = ["isahc"]
 http-static = ["isahc", "isahc/static-curl"]
-tls = ["native-tls"]
 imap_backend = ["tls"]
 jmap_backend = ["http", "serde_json"]
-maildir_backend = ["notify", "memmap"]
-mbox_backend = ["notify", "memmap"]
+maildir_backend = ["notify"]
+mbox_backend = ["notify"]
 notmuch_backend = []
 smtp = ["tls", "base64"]
 sqlite3 = ["rusqlite", ]
+tls = ["native-tls"]
 unicode_algorithms = ["unicode-segmentation"]
 vcard = []

melib/README.md

@@ -0,0 +1,88 @@
+# melib
+
+[![GitHub license](https://img.shields.io/github/license/meli/meli)](https://github.com/meli/meli/blob/master/COPYING) [![Crates.io](https://img.shields.io/crates/v/melib)](https://crates.io/crates/melib) [![docs.rs](https://docs.rs/melib/badge.svg)](https://docs.rs/melib)
+
+Library for handling mail.
+
+## optional features
+
+| feature flag           | dependencies                        | notes                    |
+| ---------------------- | ----------------------------------- | ------------------------ |
+| `imap_backend`         | `native-tls`                        |                          |
+| `deflate_compression`  | `flate2`                            | for use with IMAP        |
+| `jmap_backend`         | `isahc`, `native-tls`, `serde_json` |                          |
+| `maildir_backend`      | `notify`                            |                          |
+| `mbox_backend`         | `notify`                            |                          |
+| `notmuch_backend`      | `notify`                            |                          |
+| `sqlite`               | `rusqlite`                          | used in IMAP cache       |
+| `unicode_algorithms`   | `unicode-segmentation`              | linebreaking algo etc    |
+| `vcard`                |                                     | vcard parsing            |
+| `gpgme`                |                                     | GPG use with libgpgme    |
+| `smtp`                 | `native-tls`, `base64`              | async SMTP communication |
+
+## Example: Parsing bytes into an `Envelope`
+
+An `Envelope` represents the information you can get from an email's headers
+and body structure. Addresses in `To`, `From` fields etc are parsed into
+`Address` types.
+
+```rust
+use melib::{Attachment, Envelope};
+
+let raw_mail = r#"From: "some name" <some@example.com>
+To: "me" <myself@example.com>
+Cc:
+Subject: =?utf-8?Q?gratuitously_encoded_subject?=
+Message-ID: <h2g7f.z0gy2pgaen5m@example.com>
+MIME-Version: 1.0
+Content-Type: multipart/mixed; charset="utf-8";
+ boundary="bzz_bzz__bzz__"
+
+This is a MIME formatted message with attachments. Use a MIME-compliant client to view it properly.
+--bzz_bzz__bzz__
+
+hello world.
+--bzz_bzz__bzz__
+Content-Type: image/gif; name="test_image.gif"; charset="utf-8"
+Content-Disposition: attachment
+Content-Transfer-Encoding: base64
+
+R0lGODdhKAAXAOfZAAABzAADzQAEzgQFtBEAxAAGxBcAxwALvRcFwAAPwBcLugATuQEUuxoNuxYQ
+sxwOvAYVvBsStSAVtx8YsRUcuhwhth4iuCQsyDAwuDc1vTc3uDg4uT85rkc9ukJBvENCvURGukdF
+wUVKt0hLuUxPvVZSvFlYu1hbt2BZuFxdul5joGhqlnNuf3FvlnBvwXJyt3Jxw3N0oXx1gH12gV99
+z317f3N7spFxwHp5wH99gYB+goF/g25+26tziIOBhWqD3oiBjICAuudkjIN+zHeC2n6Bzc1vh4eF
+iYaBw8F0kImHi4KFxYyHmIWIvI2Lj4uIvYaJyY+IuJGMi5iJl4qKxZSMmIuLxpONnpGPk42NvI2M
+1LKGl46OvZePm5ORlZiQnJqSnpaUmLyJnJuTn5iVmZyUoJGVyZ2VoZSVw5iXoZmWrO18rJiUyp6W
+opuYnKaVnZ+Xo5yZncaMoaCYpJiaqo+Z2Z2annuf5qGZpa2WoJybpZmayZ2Z0KCZypydrZ6dp6Cd
+oZ6a0aGay5ucy5+eqKGeouWMgp+b0qKbzKCfqdqPnp2ezaGgqqOgpKafqrScpp+gz6ajqKujr62j
+qayksKmmq62lsaiosqqorOyWnaqqtKeqzLGptaurta2rr7Kqtq+ssLOrt6+uuLGusuqhfbWtubCv
+ubKvs7GwurOwtPSazbevu+ali7SxtbiwvOykjLOyvLWytuCmqOankrSzvbazuLmyvrW0vre0uba1
+wLi1ury0wLm2u721wbe3wbq3vMC2vLi4wr+3w7m5w8C4xLi6yry6vsG5xbu7xcC6zMK6xry8xry+
+u8O7x729x8C9wb++yMG+wsO+vMK/w8a+y8e/zMnBzcXH18nL2///////////////////////////
+////////////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////ywAAAAAKAAXAAAI/gBP4Cjh
+IYMLEh0w4EgBgsMLEyFGFBEB5cOFABgzatS4AVssZAOsLOHCxooVMzCyoNmzaBOkJlS0VEDyZMjG
+mxk3XOMF60CDBgsoPABK9KcDCRImPCiQYAECAgQCRMU4VSrGCjFarBgUSJCgQ10FBTrkNRCfPnz4
+dA3UNa1btnDZqgU7Ntqzu3ej2X2mFy9eaHuhNRtMGJrhwYYN930G2K7eaNIY34U2mfJkwpgzI9Yr
+GBqwR2KSvAlMOXHnw5pTNzPdLNoWIWtU9XjGjDEYS8LAlFm1SrVvzIKj5TH0KpORSZOryPgCZgqL
+Ob+jG0YVRBErUrOiiGJ8KxgtYsh27xWL/tswnTtEbsiRVYdJNMHk4yOGhswGjR88UKjQ9Ey+/8TL
+XKKGGn7Akph/8XX2WDTTcAYfguVt9hhrEPqmzIOJ3VUheb48WJiHG6amC4i+WVJKKCimqGIoYxyj
+WWK8kKjaJ9bA18sxvXjYhourmbbMMrjI+OIn1QymDCVXANGFK4S1gQw0PxozzC+33FLLKUJq9gk1
+gyWDhyNwrMLkYGUEM4wvuLRiCiieXIJJJVlmJskcZ9TZRht1lnFGGmTMkMoonVQSSSOFAGJHHI0w
+ouiijDaaCCGQRgrpH3q4QYYXWDihxBE+7KCDDjnUIEVAADs=
+--bzz_bzz__bzz__--"#;
+
+let envelope = Envelope::from_bytes(raw_mail.as_bytes(), None).expect("Could not parse mail");
+assert_eq!(envelope.subject().as_ref(), "gratuitously encoded subject");
+assert_eq!(envelope.message_id_display().as_ref(), "<h2g7f.z0gy2pgaen5m@example.com>");
+
+let body = envelope.body_bytes(raw_mail.as_bytes());
+assert_eq!(body.content_type().to_string().as_str(), "multipart/mixed");
+
+let body_text = body.text();
+assert_eq!(body_text.as_str(), "hello world.");
+
+let subattachments: Vec<Attachment> = body.attachments();
+assert_eq!(subattachments.len(), 3);
+assert_eq!(subattachments[2].content_type().name().unwrap(), "test_image.gif");
+```

melib/build.rs

@@ -25,15 +25,26 @@ include!("src/text_processing/types.rs");
 fn main() -> Result<(), std::io::Error> {
     #[cfg(feature = "unicode_algorithms")]
     {
+        const MOD_PATH: &str = "src/text_processing/tables.rs";
+        println!("cargo:rerun-if-changed=build.rs");
+        println!("cargo:rerun-if-changed={}", MOD_PATH);
+        /* Line break tables */
         use std::fs::File;
         use std::io::prelude::*;
         use std::io::BufReader;
-        use std::path::{Path, PathBuf};
-        use std::process::Command;
+        use std::path::Path;
+        use std::process::{Command, Stdio};
         const LINE_BREAK_TABLE_URL: &str =
             "http://www.unicode.org/Public/UCD/latest/ucd/LineBreak.txt";
+        /* Grapheme width tables */
+        const UNICODE_DATA_URL: &str =
+            "http://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt";
+        const EAW_URL: &str = "http://www.unicode.org/Public/UCD/latest/ucd/EastAsianWidth.txt";
+        const EMOJI_DATA_URL: &str =
+            "https://www.unicode.org/Public/UCD/latest/ucd/emoji/emoji-data.txt";
 
-        let mod_path = Path::new("src/text_processing/tables.rs");
+
+        let mod_path = Path::new(MOD_PATH);
         if mod_path.exists() {
             eprintln!(
                 "{} already exists, delete it if you want to replace it.",
@@ -41,18 +52,14 @@ fn main() -> Result<(), std::io::Error> {
             );
             std::process::exit(0);
         }
-        let mut tmpdir_path = PathBuf::from(
-            std::str::from_utf8(&Command::new("mktemp").arg("-d").output()?.stdout)
-                .unwrap()
-                .trim(),
-        );
-        tmpdir_path.push("LineBreak.txt");
-        Command::new("curl")
-            .args(&["-o", tmpdir_path.to_str().unwrap(), LINE_BREAK_TABLE_URL])
-            .output()?;
+        let mut child = Command::new("curl")
+            .args(&["-o", "-", LINE_BREAK_TABLE_URL])
+            .stdout(Stdio::piped())
+            .stdin(Stdio::null())
+            .stderr(Stdio::inherit())
+            .spawn()?;
 
-        let file = File::open(&tmpdir_path)?;
-        let buf_reader = BufReader::new(file);
+        let buf_reader = BufReader::new(child.stdout.take().unwrap());
 
         let mut line_break_table: Vec<(u32, u32, LineBreakClass)> = Vec::with_capacity(3800);
         for line in buf_reader.lines() {
@@ -69,31 +76,351 @@ fn main() -> Result<(), std::io::Error> {
             let mut codepoint_iter = chars_str.split("..");
 
             let first_codepoint: u32 =
-                u32::from_str_radix(std::dbg!(codepoint_iter.next().unwrap()), 16).unwrap();
+                u32::from_str_radix(codepoint_iter.next().unwrap(), 16).unwrap();
 
             let sec_codepoint: u32 = codepoint_iter
                 .next()
-                .map(|v| u32::from_str_radix(std::dbg!(v), 16).unwrap())
+                .map(|v| u32::from_str_radix(v, 16).unwrap())
                 .unwrap_or(first_codepoint);
             let class = &tokens[semicolon_idx + 1..semicolon_idx + 1 + 2];
             line_break_table.push((first_codepoint, sec_codepoint, LineBreakClass::from(class)));
         }
+        child.wait()?;
+
+        let child = Command::new("curl")
+            .args(&["-o", "-", UNICODE_DATA_URL])
+            .stdout(Stdio::piped())
+            .output()?;
+
+        let unicode_data = String::from_utf8_lossy(&child.stdout);
+
+        let child = Command::new("curl")
+            .args(&["-o", "-", EAW_URL])
+            .stdout(Stdio::piped())
+            .output()?;
+
+        let eaw_data = String::from_utf8_lossy(&child.stdout);
+
+        let child = Command::new("curl")
+            .args(&["-o", "-", EMOJI_DATA_URL])
+            .stdout(Stdio::piped())
+            .output()?;
+
+        let emoji_data = String::from_utf8_lossy(&child.stdout);
+
+        const MAX_CODEPOINT: usize = 0x110000;
+        // See https://www.unicode.org/L2/L1999/UnicodeData.html
+        const FIELD_CODEPOINT: usize = 0;
+        const FIELD_CATEGORY: usize = 2;
+        // Ambiguous East Asian characters
+        const WIDTH_AMBIGUOUS_EASTASIAN: isize = -3;
+
+        // Width changed from 1 to 2 in Unicode 9.0
+        const WIDTH_WIDENED_IN_9: isize = -6;
+        // Category for unassigned codepoints.
+        const CAT_UNASSIGNED: &str = "Cn";
+
+        // Category for private use codepoints.
+        const CAT_PRIVATE_USE: &str = "Co";
+
+        // Category for surrogates.
+        const CAT_SURROGATE: &str = "Cs";
+
+        struct Codepoint<'cat> {
+            raw: u32,
+            width: Option<isize>,
+            category: &'cat str,
+        }
+
+        let mut codepoints: Vec<Codepoint> = Vec::with_capacity(MAX_CODEPOINT + 1);
+        for i in 0..=MAX_CODEPOINT {
+            codepoints.push(Codepoint {
+                raw: i as u32,
+                width: None,
+                category: CAT_UNASSIGNED,
+            });
+        }
+
+        set_general_categories(&mut codepoints, &unicode_data);
+        set_eaw_widths(&mut codepoints, &eaw_data);
+        set_emoji_widths(&mut codepoints, &emoji_data);
+        set_hardcoded_ranges(&mut codepoints);
+        fn hexrange_to_range(hexrange: &str) -> std::ops::Range<usize> {
+            /* Given a string like 1F300..1F320 representing an inclusive range,
+            return the range of codepoints.
+            If the string is like 1F321, return a range of just that element.
+            */
+            let hexrange = hexrange.trim();
+            let fields = hexrange
+                .split("..")
+                .map(|h| usize::from_str_radix(h.trim(), 16).unwrap())
+                .collect::<Vec<usize>>();
+            if fields.len() == 1 {
+                fields[0]..(fields[0] + 1)
+            } else {
+                fields[0]..(fields[1] + 1)
+            }
+        }
+
+        fn set_general_categories<'u>(codepoints: &mut Vec<Codepoint<'u>>, unicode_data: &'u str) {
+            for line in unicode_data.lines() {
+                let fields = line.trim().split(";").collect::<Vec<_>>();
+                if fields.len() > FIELD_CATEGORY {
+                    for idx in hexrange_to_range(fields[FIELD_CODEPOINT]) {
+                        codepoints[idx].category = fields[FIELD_CATEGORY];
+                    }
+                }
+            }
+        }
+
+        fn set_eaw_widths(codepoints: &mut Vec<Codepoint<'_>>, eaw_data_lines: &str) {
+            //  Read from EastAsianWidth.txt, set width values on the codepoints
+            for line in eaw_data_lines.lines() {
+                let line = line.trim().split('#').next().unwrap_or(line);
+                let fields = line.trim().split(';').collect::<Vec<_>>();
+                if fields.len() != 2 {
+                    continue;
+                }
+                let hexrange = fields[0];
+                let width_type = fields[1];
+                // width_types:
+                //  A: ambiguous, F: fullwidth, H: halfwidth,
+                // . N: neutral, Na: east-asian Narrow
+                let width: isize = if width_type == "A" {
+                    WIDTH_AMBIGUOUS_EASTASIAN
+                } else if width_type == "F" || width_type == "W" {
+                    2
+                } else {
+                    1
+                };
+                for cp in hexrange_to_range(hexrange) {
+                    codepoints[cp].width = Some(width);
+                }
+            }
+            // Apply the following special cases:
+            //  - The unassigned code points in the following blocks default to "W":
+            //         CJK Unified Ideographs Extension A: U+3400..U+4DBF
+            //         CJK Unified Ideographs:             U+4E00..U+9FFF
+            //         CJK Compatibility Ideographs:       U+F900..U+FAFF
+            //  - All undesignated code points in Planes 2 and 3, whether inside or
+            //      outside of allocated blocks, default to "W":
+            //         Plane 2:                            U+20000..U+2FFFD
+            //         Plane 3:                            U+30000..U+3FFFD
+            const WIDE_RANGES: [(usize, usize); 5] = [
+                (0x3400, 0x4DBF),
+                (0x4E00, 0x9FFF),
+                (0xF900, 0xFAFF),
+                (0x20000, 0x2FFFD),
+                (0x30000, 0x3FFFD),
+            ];
+            for &wr in WIDE_RANGES.iter() {
+                for cp in wr.0..(wr.1 + 1) {
+                    if codepoints[cp].width.is_none() {
+                        codepoints[cp].width = Some(2);
+                    }
+                }
+            }
+        }
+        fn set_emoji_widths(codepoints: &mut Vec<Codepoint<'_>>, emoji_data_lines: &str) {
+            // Read from emoji-data.txt, set codepoint widths
+            for line in emoji_data_lines.lines() {
+                if !line.contains("#") || line.trim().starts_with("#") {
+                    continue;
+                }
+                let mut fields = line.trim().split('#').collect::<Vec<_>>();
+                if fields.len() != 2 {
+                    continue;
+                }
+                let comment = fields.pop().unwrap();
+                let fields = fields.pop().unwrap();
+
+                let hexrange = fields.split(";").next().unwrap();
+
+                // In later versions of emoji-data.txt there are some "reserved"
+                // entries that have "NA" instead of a Unicode version number
+                // of first use, they will now return a zero version instead of
+                // crashing the script
+                if comment.trim().starts_with("NA") {
+                    continue;
+                }
+
+                use std::str::FromStr;
+                let mut v = comment.trim().split_whitespace().next().unwrap();
+                if v.starts_with("E") {
+                    v = &v[1..];
+                }
+                if v.as_bytes()
+                    .get(0)
+                    .map(|c| !c.is_ascii_digit())
+                    .unwrap_or(true)
+                {
+                    continue;
+                }
+                let mut idx = 1;
+                while v
+                    .as_bytes()
+                    .get(idx)
+                    .map(|c| c.is_ascii_digit())
+                    .unwrap_or(false)
+                {
+                    idx += 1;
+                }
+                if v.as_bytes().get(idx).map(|&c| c != b'.').unwrap_or(true) {
+                    continue;
+                }
+                idx += 1;
+                while v
+                    .as_bytes()
+                    .get(idx)
+                    .map(|c| c.is_ascii_digit())
+                    .unwrap_or(false)
+                {
+                    idx += 1;
+                }
+                v = &v[0..idx];
+
+                let version = f32::from_str(v).unwrap();
+                for cp in hexrange_to_range(hexrange) {
+                    // Don't consider <=1F000 values as emoji. These can only be made
+                    // emoji through the variation selector which interacts terribly
+                    // with wcwidth().
+                    if cp < 0x1F000 {
+                        continue;
+                    }
+                    // Skip codepoints that are explicitly not wide.
+                    // For example U+1F336 ("Hot Pepper") renders like any emoji but is
+                    // marked as neutral in EAW so has width 1 for some reason.
+                    //if codepoints[cp].width == Some(1) {
+                    //    continue;
+                    //}
+
+                    // If this emoji was introduced before Unicode 9, then it was widened in 9.
+                    codepoints[cp].width = if version >= 9.0 {
+                        Some(2)
+                    } else {
+                        Some(WIDTH_WIDENED_IN_9)
+                    };
+                }
+            }
+        }
+        fn set_hardcoded_ranges(codepoints: &mut Vec<Codepoint<'_>>) {
+            // Mark private use and surrogate codepoints
+            // Private use can be determined awkwardly from UnicodeData.txt,
+            // but we just hard-code them.
+            // We do not treat "private use high surrogate" as private use
+            // so as to match wcwidth9().
+            const PRIVATE_RANGES: [(usize, usize); 3] =
+                [(0xE000, 0xF8FF), (0xF0000, 0xFFFFD), (0x100000, 0x10FFFD)];
+            for &(first, last) in PRIVATE_RANGES.iter() {
+                for idx in first..=last {
+                    codepoints[idx].category = CAT_PRIVATE_USE;
+                }
+            }
+
+            const SURROGATE_RANGES: [(usize, usize); 2] = [(0xD800, 0xDBFF), (0xDC00, 0xDFFF)];
+            for &(first, last) in SURROGATE_RANGES.iter() {
+                for idx in first..=last {
+                    codepoints[idx].category = CAT_SURROGATE;
+                }
+            }
+        }
 
         let mut file = File::create(&mod_path)?;
-        file.write_all(b"use crate::types::LineBreakClass::*;\n")
-            .unwrap();
-        file.write_all(b"use crate::types::LineBreakClass;\n\n")
-            .unwrap();
-        file.write_all(b"const LINE_BREAK_RULES: &[(u32, u32, LineBreakClass)] = &[\n")
-            .unwrap();
+        file.write_all(
+            br#"/*
+ * meli - text_processing crate.
+ *
+ * Copyright 2017-2020 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::types::LineBreakClass::{self, *};
+
+pub const LINE_BREAK_RULES: &[(u32, u32, LineBreakClass)] = &[
+"#,
+        )
+        .unwrap();
         for l in &line_break_table {
             file.write_all(format!("    (0x{:X}, 0x{:X}, {:?}),\n", l.0, l.1, l.2).as_bytes())
                 .unwrap();
         }
-        file.write_all(b"];").unwrap();
-        std::fs::remove_file(&tmpdir_path).unwrap();
-        tmpdir_path.pop();
-        std::fs::remove_dir(&tmpdir_path).unwrap();
+        file.write_all(b"];\n").unwrap();
+
+        for (name, filter) in [
+            (
+                "ASCII",
+                Box::new(|c: &&Codepoint| c.raw < 0x7f && c.raw >= 0x20)
+                    as Box<dyn Fn(&&Codepoint) -> bool>,
+            ),
+            (
+                "PRIVATE",
+                Box::new(|c: &&Codepoint| c.category == CAT_PRIVATE_USE),
+            ),
+            (
+                "NONPRINT",
+                Box::new(|c: &&Codepoint| {
+                    ["Cc", "Cf", "Zl", "Zp", CAT_SURROGATE].contains(&c.category)
+                }),
+            ),
+            (
+                "COMBINING",
+                Box::new(|c: &&Codepoint| ["Mn", "Mc", "Me"].contains(&c.category)),
+            ),
+            ("DOUBLEWIDE", Box::new(|c: &&Codepoint| c.width == Some(2))),
+            (
+                "UNASSIGNED",
+                Box::new(|c: &&Codepoint| c.category == CAT_UNASSIGNED),
+            ),
+            (
+                "AMBIGUOUS",
+                Box::new(|c: &&Codepoint| c.width == Some(WIDTH_AMBIGUOUS_EASTASIAN)),
+            ),
+            (
+                "WIDENEDIN9",
+                Box::new(|c: &&Codepoint| c.width == Some(WIDTH_WIDENED_IN_9)),
+            ),
+        ]
+        .iter()
+        {
+            file.write_all(
+                format!(
+                    r#"
+pub const {}: &[(u32, u32)] = &[
+"#,
+                    name
+                )
+                .as_bytes(),
+            )
+            .unwrap();
+            let mut iter = codepoints.iter().filter(filter);
+            let mut prev = iter.next().unwrap().raw;
+            let mut a = prev;
+            for cp in iter {
+                if prev + 1 != cp.raw {
+                    file.write_all(format!("    (0x{:X}, 0x{:X}),\n", a, prev).as_bytes())
+                        .unwrap();
+                    a = cp.raw;
+                }
+                prev = cp.raw;
+            }
+            file.write_all(format!("    (0x{:X}, 0x{:X}),\n", a, prev).as_bytes())
+                .unwrap();
+            file.write_all(b"];\n").unwrap();
+        }
     }
     Ok(())
 }

melib/src/addressbook.rs

@@ -192,7 +192,7 @@ impl Card {
         self.key.as_str()
     }
     pub fn last_edited(&self) -> String {
-        datetime::timestamp_to_string(self.last_edited, None)
+        datetime::timestamp_to_string(self.last_edited, None, false)
     }
 
     pub fn set_id(&mut self, new_val: CardId) {

melib/src/addressbook/vcard.rs

@@ -201,7 +201,7 @@ impl<V: VCardVersion> TryInto<Card> for VCard<V> {
                       T102200Z
                       T102200-0800
                       */
-            card.birthday = crate::datetime::timestamp_from_string(val.value.as_str(), "%Y%m%d")
+            card.birthday = crate::datetime::timestamp_from_string(val.value.as_str(), "%Y%m%d\0")
                 .unwrap_or_default();
         }
         if let Some(val) = self.0.remove("EMAIL") {

melib/src/backends.rs

@@ -58,15 +58,15 @@ use self::maildir::MaildirType;
 use self::mbox::MboxType;
 use super::email::{Envelope, EnvelopeHash, Flag};
 use std::any::Any;
-use std::collections::BTreeMap;
+use std::collections::BTreeSet;
 use std::fmt;
 use std::fmt::Debug;
 use std::ops::Deref;
 use std::sync::{Arc, RwLock};
 
-pub use futures::stream::Stream;
+use futures::stream::Stream;
 use std::future::Future;
-pub use std::pin::Pin;
+use std::pin::Pin;
 
 use std::collections::HashMap;
 
@@ -246,6 +246,14 @@ pub enum RefreshEventKind {
     NewFlags(EnvelopeHash, (Flag, Vec<String>)),
     Rescan,
     Failure(MeliError),
+    MailboxCreate(Mailbox),
+    MailboxDelete(MailboxHash),
+    MailboxRename {
+        old_mailbox_hash: MailboxHash,
+        new_mailbox: Mailbox,
+    },
+    MailboxSubscribe(MailboxHash),
+    MailboxUnsubscribe(MailboxHash),
 }
 
 #[derive(Debug, Clone)]
@@ -335,18 +343,12 @@ pub trait MailBackend: ::std::fmt::Debug + Send + Sync {
     ) -> ResultFuture<()>;
 
     fn delete_messages(
-        &self,
-        _env_hashes: EnvelopeHashBatch,
-        _mailbox_hash: MailboxHash,
-    ) -> ResultFuture<()> {
-        Err(MeliError::new("Unimplemented."))
-    }
-    fn delete(&self, _env_hash: EnvelopeHash, _mailbox_hash: MailboxHash) -> ResultFuture<()> {
-        Err(MeliError::new("Unimplemented."))
-    }
-    fn tags(&self) -> Option<Arc<RwLock<BTreeMap<u64, String>>>> {
-        None
-    }
+        &mut self,
+        env_hashes: EnvelopeHashBatch,
+        mailbox_hash: MailboxHash,
+    ) -> ResultFuture<()>;
+
+    fn collection(&self) -> crate::Collection;
     fn as_any(&self) -> &dyn Any;
     fn as_any_mut(&mut self) -> &mut dyn Any;
 
@@ -617,3 +619,95 @@ impl EnvelopeHashBatch {
         1 + self.rest.len()
     }
 }
+
+#[derive(Default, Clone)]
+pub struct LazyCountSet {
+    not_yet_seen: usize,
+    set: BTreeSet<EnvelopeHash>,
+}
+
+impl fmt::Debug for LazyCountSet {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.debug_struct("LazyCountSet")
+            .field("not_yet_seen", &self.not_yet_seen)
+            .field("set", &self.set.len())
+            .field("total_len", &self.len())
+            .finish()
+    }
+}
+
+impl LazyCountSet {
+    pub fn set_not_yet_seen(&mut self, new_val: usize) {
+        self.not_yet_seen = new_val;
+    }
+
+    pub fn insert_existing(&mut self, new_val: EnvelopeHash) -> bool {
+        if self.not_yet_seen == 0 {
+            false
+        } else {
+            if !self.set.contains(&new_val) {
+                self.not_yet_seen -= 1;
+            }
+            self.set.insert(new_val);
+            true
+        }
+    }
+
+    pub fn insert_existing_set(&mut self, set: BTreeSet<EnvelopeHash>) {
+        let old_len = self.set.len();
+        self.set.extend(set.into_iter());
+        self.not_yet_seen = self.not_yet_seen.saturating_sub(self.set.len() - old_len);
+    }
+
+    #[inline(always)]
+    pub fn len(&self) -> usize {
+        self.set.len() + self.not_yet_seen
+    }
+
+    #[inline(always)]
+    pub fn clear(&mut self) {
+        self.set.clear();
+        self.not_yet_seen = 0;
+    }
+
+    pub fn insert_new(&mut self, new_val: EnvelopeHash) {
+        self.set.insert(new_val);
+    }
+
+    pub fn insert_set(&mut self, set: BTreeSet<EnvelopeHash>) {
+        self.set.extend(set.into_iter());
+    }
+
+    pub fn remove(&mut self, env_hash: EnvelopeHash) -> bool {
+        self.set.remove(&env_hash)
+    }
+}
+
+#[test]
+fn test_lazy_count_set() {
+    let mut new = LazyCountSet::default();
+    assert_eq!(new.len(), 0);
+    new.set_not_yet_seen(10);
+    assert_eq!(new.len(), 10);
+    for i in 0..10 {
+        assert!(new.insert_existing(i));
+    }
+    assert_eq!(new.len(), 10);
+    assert!(!new.insert_existing(10));
+    assert_eq!(new.len(), 10);
+}
+
+pub struct IsSubscribedFn(Box<dyn Fn(&str) -> bool + Send + Sync>);
+
+impl std::fmt::Debug for IsSubscribedFn {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "IsSubscribedFn Box")
+    }
+}
+
+impl std::ops::Deref for IsSubscribedFn {
+    type Target = Box<dyn Fn(&str) -> bool + Send + Sync>;
+    fn deref(&self) -> &Box<dyn Fn(&str) -> bool + Send + Sync> {
+        &self.0
+    }
+}

melib/src/backends/imap.rs

@@ -42,20 +42,21 @@ use crate::backends::{
     *,
 };
 
+use crate::collection::Collection;
 use crate::conf::AccountSettings;
 use crate::connections::timeout;
 use crate::email::{parser::BytesExt, *};
 use crate::error::{MeliError, Result, ResultIntoMeliError};
 use futures::lock::Mutex as FutureMutex;
 use futures::stream::Stream;
-use std::collections::{hash_map::DefaultHasher, BTreeMap};
+use std::collections::hash_map::DefaultHasher;
 use std::collections::{BTreeSet, HashMap, HashSet};
 use std::convert::TryFrom;
 use std::convert::TryInto;
 use std::hash::Hasher;
 use std::pin::Pin;
 use std::str::FromStr;
-use std::sync::{Arc, Mutex, RwLock};
+use std::sync::{Arc, Mutex};
 use std::time::{Duration, SystemTime};
 
 pub type ImapNum = usize;
@@ -64,6 +65,7 @@ pub type UIDVALIDITY = UID;
 pub type MessageSequenceNumber = ImapNum;
 
 pub static SUPPORTED_CAPABILITIES: &[&str] = &[
+    "AUTH=OAUTH2",
     #[cfg(feature = "deflate_compression")]
     "COMPRESS=DEFLATE",
     "CONDSTORE",
@@ -82,9 +84,7 @@ pub static SUPPORTED_CAPABILITIES: &[&str] = &[
 
 #[derive(Debug, Default)]
 pub struct EnvelopeCache {
-    bytes: Option<String>,
-    headers: Option<String>,
-    body: Option<String>,
+    bytes: Option<Vec<u8>>,
     flags: Option<Flag>,
 }
 
@@ -101,20 +101,6 @@ pub struct ImapServerConf {
     pub timeout: Option<Duration>,
 }
 
-struct IsSubscribedFn(Box<dyn Fn(&str) -> bool + Send + Sync>);
-
-impl std::fmt::Debug for IsSubscribedFn {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(f, "IsSubscribedFn Box")
-    }
-}
-
-impl std::ops::Deref for IsSubscribedFn {
-    type Target = Box<dyn Fn(&str) -> bool + Send + Sync>;
-    fn deref(&self) -> &Box<dyn Fn(&str) -> bool + Send + Sync> {
-        &self.0
-    }
-}
 type Capabilities = HashSet<Vec<u8>>;
 
 #[macro_export]
@@ -157,14 +143,13 @@ pub struct UIDStore {
     msn_index: Arc<Mutex<HashMap<MailboxHash, Vec<UID>>>>,
 
     byte_cache: Arc<Mutex<HashMap<UID, EnvelopeCache>>>,
-    tag_index: Arc<RwLock<BTreeMap<u64, String>>>,
+    collection: Collection,
 
     /* Offline caching */
     uidvalidity: Arc<Mutex<HashMap<MailboxHash, UID>>>,
     envelopes: Arc<Mutex<HashMap<EnvelopeHash, cache::CachedEnvelope>>>,
     max_uids: Arc<Mutex<HashMap<MailboxHash, UID>>>,
     modseq: Arc<Mutex<HashMap<EnvelopeHash, ModSequence>>>,
-    reverse_modseq: Arc<Mutex<HashMap<MailboxHash, BTreeMap<ModSequence, EnvelopeHash>>>>,
     highestmodseqs: Arc<Mutex<HashMap<MailboxHash, std::result::Result<ModSequence, ()>>>>,
     mailboxes: Arc<FutureMutex<HashMap<MailboxHash, ImapMailbox>>>,
     is_online: Arc<Mutex<(SystemTime, Result<()>)>>,
@@ -188,14 +173,13 @@ impl UIDStore {
             envelopes: Default::default(),
             max_uids: Default::default(),
             modseq: Default::default(),
-            reverse_modseq: Default::default(),
             highestmodseqs: Default::default(),
             hash_index: Default::default(),
             uid_index: Default::default(),
             msn_index: Default::default(),
             byte_cache: Default::default(),
             mailboxes: Arc::new(FutureMutex::new(Default::default())),
-            tag_index: Arc::new(RwLock::new(Default::default())),
+            collection: Default::default(),
             is_online: Arc::new(Mutex::new((
                 SystemTime::now(),
                 Err(MeliError::new("Account is uninitialised.")),
@@ -236,6 +220,7 @@ impl MailBackend for ImapType {
                     #[cfg(feature = "deflate_compression")]
                     deflate,
                     condstore,
+                    oauth2,
                 },
         } = self.server_conf.protocol
         {
@@ -277,6 +262,15 @@ impl MailBackend for ImapType {
                             };
                         }
                     }
+                    "AUTH=OAUTH2" => {
+                        if oauth2 {
+                            *status = MailBackendExtensionStatus::Enabled { comment: None };
+                        } else {
+                            *status = MailBackendExtensionStatus::Supported {
+                                comment: Some("Disabled by user configuration"),
+                            };
+                        }
+                    }
                     _ => {
                         if SUPPORTED_CAPABILITIES
                             .iter()
@@ -325,7 +319,7 @@ impl MailBackend for ImapType {
             None
         };
         let mut state = FetchState {
-            stage: if self.uid_store.keep_offline_cache {
+            stage: if self.uid_store.keep_offline_cache && cache_handle.is_some() {
                 FetchStage::InitialCache
             } else {
                 FetchStage::InitialFresh
@@ -336,11 +330,24 @@ impl MailBackend for ImapType {
             cache_handle,
         };
 
+        /* do this in a closure to prevent recursion limit error in async_stream macro */
+        let prepare_cl = |f: &ImapMailbox| {
+            f.set_warm(true);
+            if let Ok(mut exists) = f.exists.lock() {
+                let total = exists.len();
+                exists.clear();
+                exists.set_not_yet_seen(total);
+            }
+            if let Ok(mut unseen) = f.unseen.lock() {
+                let total = unseen.len();
+                unseen.clear();
+                unseen.set_not_yet_seen(total);
+            }
+        };
         Ok(Box::pin(async_stream::try_stream! {
             {
                 let f = &state.uid_store.mailboxes.lock().await[&mailbox_hash];
-                f.exists.lock().unwrap().clear();
-                f.unseen.lock().unwrap().clear();
+                prepare_cl(f);
                 if f.no_select {
                     yield vec![];
                     return;
@@ -430,11 +437,11 @@ impl MailBackend for ImapType {
             match timeout(timeout_dur, connection.lock()).await {
                 Ok(mut conn) => {
                     debug!("is_online");
-                    match debug!(timeout(timeout_dur, conn.connect()).await) {
+                    match timeout(timeout_dur, conn.connect()).await {
                         Ok(Ok(())) => Ok(()),
                         Err(err) | Ok(Err(err)) => {
                             conn.stream = Err(err.clone());
-                            debug!(conn.connect().await)
+                            conn.connect().await
                         }
                     }
                 }
@@ -447,21 +454,20 @@ impl MailBackend for ImapType {
         let server_conf = self.server_conf.clone();
         let main_conn = self.connection.clone();
         let uid_store = self.uid_store.clone();
-        let has_idle: bool = match self.server_conf.protocol {
-            ImapProtocol::IMAP {
-                extension_use: ImapExtensionUse { idle, .. },
-            } => {
-                idle && uid_store
-                    .capabilities
-                    .lock()
-                    .unwrap()
-                    .iter()
-                    .any(|cap| cap.eq_ignore_ascii_case(b"IDLE"))
-            }
-            _ => false,
-        };
         Ok(Box::pin(async move {
-            debug!(has_idle);
+            let has_idle: bool = match server_conf.protocol {
+                ImapProtocol::IMAP {
+                    extension_use: ImapExtensionUse { idle, .. },
+                } => {
+                    idle && uid_store
+                        .capabilities
+                        .lock()
+                        .unwrap()
+                        .iter()
+                        .any(|cap| cap.eq_ignore_ascii_case(b"IDLE"))
+                }
+                _ => false,
+            };
             while let Err(err) = if has_idle {
                 idle(ImapWatchKit {
                     conn: ImapConnection::new_connection(&server_conf, uid_store.clone()),
@@ -480,17 +486,19 @@ impl MailBackend for ImapType {
                 let mut main_conn_lck = timeout(uid_store.timeout, main_conn.lock()).await?;
                 if err.kind.is_network() {
                     uid_store.is_online.lock().unwrap().1 = Err(err.clone());
+                } else {
+                    return Err(err);
                 }
-                debug!("failure: {}", err.to_string());
+                debug!("Watch failure: {}", err.to_string());
                 match timeout(uid_store.timeout, main_conn_lck.connect())
                     .await
                     .and_then(|res| res)
                 {
                     Err(err2) => {
-                        debug!("reconnect attempt failed: {}", err2.to_string());
+                        debug!("Watch reconnect attempt failed: {}", err2.to_string());
                     }
                     Ok(()) => {
-                        debug!("reconnect attempt succesful");
+                        debug!("Watch reconnect attempt succesful");
                         continue;
                     }
                 }
@@ -624,25 +632,9 @@ impl MailBackend for ImapType {
             }
             let dest_path = {
                 let mailboxes = uid_store.mailboxes.lock().await;
-                let mailbox = mailboxes
-                    .get(&source_mailbox_hash)
-                    .ok_or_else(|| MeliError::new("Source mailbox not found"))?;
-                if move_ && !mailbox.permissions.lock().unwrap().delete_messages {
-                    return Err(MeliError::new(format!(
-                        "You are not allowed to delete messages from mailbox {}",
-                        mailbox.path()
-                    )));
-                }
                 let mailbox = mailboxes
                     .get(&destination_mailbox_hash)
                     .ok_or_else(|| MeliError::new("Destination mailbox not found"))?;
-                if !mailbox.permissions.lock().unwrap().create_messages {
-                    return Err(MeliError::new(format!(
-                        "You are not allowed to create messages in mailbox {}",
-                        mailbox.path()
-                    )));
-                }
-
                 mailbox.imap_path().to_string()
             };
             let mut response = Vec::with_capacity(8 * 1024);
@@ -717,8 +709,9 @@ impl MailBackend for ImapType {
                 .await?;
             if flags.iter().any(|(_, b)| *b) {
                 /* Set flags/tags to true */
+                let mut set_seen = false;
                 let command = {
-                    let mut tag_lck = uid_store.tag_index.write().unwrap();
+                    let mut tag_lck = uid_store.collection.tag_index.write().unwrap();
                     let mut cmd = format!("UID STORE {}", uids[0]);
                     for uid in uids.iter().skip(1) {
                         cmd = format!("{},{}", cmd, uid);
@@ -740,6 +733,7 @@ impl MailBackend for ImapType {
                             }
                             Ok(flag) if *flag == Flag::SEEN => {
                                 cmd.push_str("\\Seen ");
+                                set_seen = true;
                             }
                             Ok(flag) if *flag == Flag::DRAFT => {
                                 cmd.push_str("\\Draft ");
@@ -766,8 +760,17 @@ impl MailBackend for ImapType {
                 conn.send_command(command.as_bytes()).await?;
                 conn.read_response(&mut response, RequiredResponses::empty())
                     .await?;
+                if set_seen {
+                    let f = &uid_store.mailboxes.lock().await[&mailbox_hash];
+                    if let Ok(mut unseen) = f.unseen.lock() {
+                        for env_hash in env_hashes.iter() {
+                            unseen.remove(env_hash);
+                        }
+                    };
+                }
             }
             if flags.iter().any(|(_, b)| !*b) {
+                let mut set_unseen = false;
                 /* Set flags/tags to false */
                 let command = {
                     let mut cmd = format!("UID STORE {}", uids[0]);
@@ -791,6 +794,7 @@ impl MailBackend for ImapType {
                             }
                             Ok(flag) if *flag == Flag::SEEN => {
                                 cmd.push_str("\\Seen ");
+                                set_unseen = true;
                             }
                             Ok(flag) if *flag == Flag::DRAFT => {
                                 cmd.push_str("\\Draft ");
@@ -820,13 +824,40 @@ impl MailBackend for ImapType {
                 conn.send_command(command.as_bytes()).await?;
                 conn.read_response(&mut response, RequiredResponses::empty())
                     .await?;
+                if set_unseen {
+                    let f = &uid_store.mailboxes.lock().await[&mailbox_hash];
+                    if let Ok(mut unseen) = f.unseen.lock() {
+                        for env_hash in env_hashes.iter() {
+                            unseen.insert_new(env_hash);
+                        }
+                    };
+                }
             }
             Ok(())
         }))
     }
 
-    fn tags(&self) -> Option<Arc<RwLock<BTreeMap<u64, String>>>> {
-        Some(self.uid_store.tag_index.clone())
+    fn delete_messages(
+        &mut self,
+        env_hashes: EnvelopeHashBatch,
+        mailbox_hash: MailboxHash,
+    ) -> ResultFuture<()> {
+        let flag_future = self.set_flags(
+            env_hashes,
+            mailbox_hash,
+            smallvec::smallvec![(Ok(Flag::TRASHED), true)],
+        )?;
+        let connection = self.connection.clone();
+        Ok(Box::pin(async move {
+            flag_future.await?;
+            let mut response = Vec::with_capacity(8 * 1024);
+            let mut conn = connection.lock().await;
+            conn.send_command("EXPUNGE".as_bytes()).await?;
+            conn.read_response(&mut response, RequiredResponses::empty())
+                .await?;
+            debug!("EXPUNGE response: {}", &String::from_utf8_lossy(&response));
+            Ok(())
+        }))
     }
 
     fn as_any(&self) -> &dyn Any {
@@ -837,6 +868,10 @@ impl MailBackend for ImapType {
         self
     }
 
+    fn collection(&self) -> Collection {
+        self.uid_store.collection.clone()
+    }
+
     fn create_mailbox(
         &mut self,
         mut path: String,
@@ -1176,7 +1211,7 @@ impl MailBackend for ImapType {
             let mut conn = connection.lock().await;
             conn.examine_mailbox(mailbox_hash, &mut response, false)
                 .await?;
-            conn.send_command(format!("UID SEARCH CHARSET UTF-8 {}", query_str).as_bytes())
+            conn.send_command(format!("UID SEARCH CHARSET UTF-8 {}", query_str.trim()).as_bytes())
                 .await?;
             conn.read_response(&mut response, RequiredResponses::SEARCH)
                 .await?;
@@ -1215,7 +1250,14 @@ impl ImapType {
     ) -> Result<Box<dyn MailBackend>> {
         let server_hostname = get_conf_val!(s["server_hostname"])?;
         let server_username = get_conf_val!(s["server_username"])?;
+        let use_oauth2: bool = get_conf_val!(s["use_oauth2"], false)?;
         let server_password = if !s.extra.contains_key("server_password_command") {
+            if use_oauth2 {
+                return Err(MeliError::new(format!(
+                    "({}) `use_oauth2` use requires `server_password_command` set with a command that returns an OAUTH2 token. Consult documentation for guidance.",
+                    s.name,
+                )));
+            }
             get_conf_val!(s["server_password"])?.to_string()
         } else {
             let invocation = get_conf_val!(s["server_password_command"])?;
@@ -1272,6 +1314,7 @@ impl ImapType {
                     condstore: get_conf_val!(s["use_condstore"], true)?,
                     #[cfg(feature = "deflate_compression")]
                     deflate: get_conf_val!(s["use_deflate"], true)?,
+                    oauth2: use_oauth2,
                 },
             },
             timeout,
@@ -1385,11 +1428,11 @@ impl ImapType {
             conn.read_response(&mut res, RequiredResponses::LIST_REQUIRED)
                 .await?;
         }
-        debug!("out: {}", String::from_utf8_lossy(&res));
-        let mut lines = res.split_rn();
-        /* Remove "M__ OK .." line */
-        lines.next_back();
-        for l in lines {
+        debug!("LIST reply: {}", String::from_utf8_lossy(&res));
+        for l in res.split_rn() {
+            if !l.starts_with(b"*") {
+                continue;
+            }
             if let Ok(mut mailbox) = protocol_parser::list_mailbox_result(&l).map(|(_, v)| v) {
                 if let Some(parent) = mailbox.parent {
                     if mailboxes.contains_key(&parent) {
@@ -1436,26 +1479,38 @@ impl ImapType {
         conn.send_command(b"LSUB \"\" \"*\"").await?;
         conn.read_response(&mut res, RequiredResponses::LSUB_REQUIRED)
             .await?;
-        let mut lines = res.split_rn();
-        debug!("out: {}", String::from_utf8_lossy(&res));
-        /* Remove "M__ OK .." line */
-        lines.next_back();
-        for l in lines {
+        debug!("LSUB reply: {}", String::from_utf8_lossy(&res));
+        for l in res.split_rn() {
+            if !l.starts_with(b"*") {
+                continue;
+            }
             if let Ok(subscription) = protocol_parser::list_mailbox_result(&l).map(|(_, v)| v) {
                 if let Some(f) = mailboxes.get_mut(&subscription.hash()) {
+                    if f.special_usage() == SpecialUsageMailbox::Normal
+                        && subscription.special_usage() != SpecialUsageMailbox::Normal
+                    {
+                        f.set_special_usage(subscription.special_usage())?;
+                    }
                     f.is_subscribed = true;
                 }
             } else {
                 debug!("parse error for {:?}", l);
             }
         }
-        Ok(debug!(mailboxes))
+        Ok(mailboxes)
     }
 
     pub fn validate_config(s: &AccountSettings) -> Result<()> {
         get_conf_val!(s["server_hostname"])?;
         get_conf_val!(s["server_username"])?;
+        let use_oauth2: bool = get_conf_val!(s["use_oauth2"], false)?;
         if !s.extra.contains_key("server_password_command") {
+            if use_oauth2 {
+                return Err(MeliError::new(format!(
+                    "({}) `use_oauth2` use requires `server_password_command` set with a command that returns an OAUTH2 token. Consult documentation for guidance.",
+                    s.name,
+                )));
+            }
             get_conf_val!(s["server_password"])?;
         } else if s.extra.contains_key("server_password") {
             return Err(MeliError::new(format!(
@@ -1616,7 +1671,7 @@ async fn fetch_hlpr(state: &mut FetchState) -> Result<Vec<Envelope>> {
             FetchStage::ResyncCache => {
                 let mailbox_hash = state.mailbox_hash;
                 let mut conn = state.connection.lock().await;
-                let res = debug!(conn.resync(mailbox_hash).await);
+                let res = conn.resync(mailbox_hash).await;
                 if let Ok(Some(payload)) = res {
                     state.stage = FetchStage::Finished;
                     return Ok(payload);
@@ -1648,26 +1703,21 @@ async fn fetch_hlpr(state: &mut FetchState) -> Result<Vec<Envelope>> {
                     return Ok(Vec::new());
                 }
                 let mut conn = connection.lock().await;
-                debug!("locked for fetch {}", mailbox_path);
                 let mut response = Vec::with_capacity(8 * 1024);
                 let max_uid_left = max_uid;
                 let chunk_size = 250;
 
-                let mut payload = vec![];
+                let mut envelopes = Vec::with_capacity(chunk_size);
                 conn.examine_mailbox(mailbox_hash, &mut response, false)
                     .await?;
                 if max_uid_left > 0 {
-                    let mut envelopes = vec![];
                     debug!("{} max_uid_left= {}", mailbox_hash, max_uid_left);
                     let command = if max_uid_left == 1 {
-                        "UID FETCH 1 (UID FLAGS ENVELOPE BODYSTRUCTURE)".to_string()
+                        "UID FETCH 1 (UID FLAGS ENVELOPE BODY.PEEK[HEADER.FIELDS (REFERENCES)] BODYSTRUCTURE)".to_string()
                     } else {
                         format!(
-                            "UID FETCH {}:{} (UID FLAGS ENVELOPE BODYSTRUCTURE)",
-                            std::cmp::max(
+                            "UID FETCH {}:{} (UID FLAGS ENVELOPE BODY.PEEK[HEADER.FIELDS (REFERENCES)] BODYSTRUCTURE)",
                                 std::cmp::max(max_uid_left.saturating_sub(chunk_size), 1),
-                                1
-                            ),
                             max_uid_left
                         )
                     };
@@ -1681,29 +1731,43 @@ async fn fetch_hlpr(state: &mut FetchState) -> Result<Vec<Envelope>> {
                                 mailbox_path
                             )
                         })?;
-                    debug!(
-                        "fetch response is {} bytes and {} lines",
-                        response.len(),
-                        String::from_utf8_lossy(&response).lines().count()
-                    );
                     let (_, mut v, _) = protocol_parser::fetch_responses(&response)?;
-                    debug!("responses len is {}", v.len());
+                    debug!(
+                        "fetch response is {} bytes and {} lines and has {} parsed Envelopes",
+                        response.len(),
+                        String::from_utf8_lossy(&response).lines().count(),
+                        v.len()
+                    );
                     for FetchResponse {
                         ref uid,
                         ref mut envelope,
                         ref mut flags,
+                        ref raw_fetch_value,
+                        ref references,
                         ..
                     } in v.iter_mut()
                     {
+                        if uid.is_none() || envelope.is_none() || flags.is_none() {
+                            debug!("BUG? in fetch is none");
+                            debug!(uid);
+                            debug!(envelope);
+                            debug!(flags);
+                            debug!("response was: {}", String::from_utf8_lossy(&response));
+                            debug!(conn.process_untagged(raw_fetch_value).await)?;
+                            continue;
+                        }
                         let uid = uid.unwrap();
                         let env = envelope.as_mut().unwrap();
                         env.set_hash(generate_envelope_hash(&mailbox_path, &uid));
-                        let mut tag_lck = uid_store.tag_index.write().unwrap();
+                        if let Some(value) = references {
+                            env.set_references(value);
+                        }
+                        let mut tag_lck = uid_store.collection.tag_index.write().unwrap();
                         if let Some((flags, keywords)) = flags {
-                            if !flags.intersects(Flag::SEEN) {
+                            env.set_flags(*flags);
+                            if !env.is_seen() {
                                 our_unseen.insert(env.hash());
                             }
-                            env.set_flags(*flags);
                             for f in keywords {
                                 let hash = tag_hash!(f);
                                 if !tag_lck.contains_key(&hash) {
@@ -1714,14 +1778,14 @@ async fn fetch_hlpr(state: &mut FetchState) -> Result<Vec<Envelope>> {
                         }
                     }
                     if let Some(ref mut cache_handle) = cache_handle {
-                        if let Err(err) = debug!(cache_handle
+                        if let Err(err) = cache_handle
                             .insert_envelopes(mailbox_hash, &v)
                             .chain_err_summary(|| {
                                 format!(
                                     "Could not save envelopes in cache for mailbox {}",
                                     mailbox_path
                                 )
-                            }))
+                            })
                         {
                             (state.uid_store.event_consumer)(
                                 state.uid_store.account_hash,
@@ -1765,30 +1829,25 @@ async fn fetch_hlpr(state: &mut FetchState) -> Result<Vec<Envelope>> {
                             .lock()
                             .unwrap()
                             .insert((mailbox_hash, uid), env.hash());
-                        envelopes.push((uid, env));
+                        envelopes.push(env);
                     }
-                    debug!("sending payload for {}", mailbox_hash);
-                    unseen
+                    unseen.lock().unwrap().insert_existing_set(our_unseen);
+                    mailbox_exists
                         .lock()
                         .unwrap()
-                        .insert_existing_set(our_unseen.iter().cloned().collect());
-                    mailbox_exists.lock().unwrap().insert_existing_set(
-                        envelopes.iter().map(|(_, env)| env.hash()).collect::<_>(),
-                    );
+                        .insert_existing_set(envelopes.iter().map(|env| env.hash()).collect::<_>());
                     drop(conn);
-                    payload.extend(envelopes.into_iter().map(|(_, env)| env));
                 }
                 if max_uid_left <= 1 {
+                    unseen.lock().unwrap().set_not_yet_seen(0);
+                    mailbox_exists.lock().unwrap().set_not_yet_seen(0);
                     *stage = FetchStage::Finished;
                 } else {
                     *stage = FetchStage::FreshFetch {
-                        max_uid: std::cmp::max(
-                            std::cmp::max(max_uid_left.saturating_sub(chunk_size), 1),
-                            1,
-                        ),
+                        max_uid: std::cmp::max(max_uid_left.saturating_sub(chunk_size + 1), 1),
                     };
                 }
-                return Ok(payload);
+                return Ok(envelopes);
             }
             FetchStage::Finished => {
                 return Ok(vec![]);

melib/src/backends/imap/cache.rs

@@ -140,7 +140,7 @@ mod sqlite3_m {
     CREATE INDEX IF NOT EXISTS envelope_idx ON envelopes(hash);
     CREATE INDEX IF NOT EXISTS mailbox_idx ON mailbox(mailbox_hash);",
         ),
-        version: 1,
+        version: 2,
     };
 
     impl ToSql for ModSequence {
@@ -239,7 +239,7 @@ mod sqlite3_m {
                     .entry(mailbox_hash)
                     .and_modify(|entry| *entry = uidvalidity)
                     .or_insert(uidvalidity);
-                let mut tag_lck = self.uid_store.tag_index.write().unwrap();
+                let mut tag_lck = self.uid_store.collection.tag_index.write().unwrap();
                 for f in to_str!(&flags).split('\0') {
                     let hash = tag_hash!(f);
                     //debug!("hash {} flag {}", hash, &f);
@@ -365,7 +365,7 @@ mod sqlite3_m {
 
         fn envelopes(&mut self, mailbox_hash: MailboxHash) -> Result<Option<Vec<EnvelopeHash>>> {
             debug!("envelopes mailbox_hash {}", mailbox_hash);
-            if debug!(self.mailbox_state(mailbox_hash)?.is_none()) {
+            if self.mailbox_state(mailbox_hash)?.is_none() {
                 return Ok(None);
             }
 
@@ -429,7 +429,6 @@ mod sqlite3_m {
                 .cloned()
                 .unwrap_or_default();
             if self.mailbox_state(mailbox_hash)?.is_none() {
-                debug!(self.mailbox_state(mailbox_hash)?.is_none());
                 return Err(MeliError::new("Mailbox is not in cache").set_kind(ErrorKind::Bug));
             }
             let Self {
@@ -445,7 +444,9 @@ mod sqlite3_m {
                     modseq,
                     flags: _,
                     body: _,
+                    references: _,
                     envelope: Some(envelope),
+                    raw_fetch_value: _,
                 } = item
                 {
                     max_uid = std::cmp::max(max_uid, *uid);
@@ -469,13 +470,7 @@ mod sqlite3_m {
             mailbox_hash: MailboxHash,
             refresh_events: &[(UID, RefreshEvent)],
         ) -> Result<()> {
-            debug!(
-                "update with refresh_events mailbox_hash {} len {}",
-                mailbox_hash,
-                refresh_events.len()
-            );
             if self.mailbox_state(mailbox_hash)?.is_none() {
-                debug!(self.mailbox_state(mailbox_hash)?.is_none());
                 return Err(MeliError::new("Mailbox is not in cache").set_kind(ErrorKind::Bug));
             }
             let Self {
@@ -486,7 +481,7 @@ mod sqlite3_m {
             let tx = connection.transaction()?;
             let mut hash_index_lck = uid_store.hash_index.lock().unwrap();
             for (uid, event) in refresh_events {
-                match debug!(&event.kind) {
+                match &event.kind {
                     RefreshEventKind::Remove(env_hash) => {
                         hash_index_lck.remove(&env_hash);
                         tx.execute(
@@ -653,43 +648,16 @@ pub(super) async fn fetch_cached_envs(state: &mut FetchState) -> Result<Option<V
         ref uid_store,
         cache_handle: _,
     } = state;
-    debug!(uid_store.keep_offline_cache);
     let mailbox_hash = *mailbox_hash;
     if !uid_store.keep_offline_cache {
         return Ok(None);
     }
     {
         let mut conn = connection.lock().await;
-        match debug!(conn.load_cache(mailbox_hash).await) {
+        match conn.load_cache(mailbox_hash).await {
             None => return Ok(None),
             Some(Ok(env_hashes)) => {
-                uid_store
-                    .mailboxes
-                    .lock()
-                    .await
-                    .entry(mailbox_hash)
-                    .and_modify(|entry| {
-                        entry
-                            .exists
-                            .lock()
-                            .unwrap()
-                            .insert_set(env_hashes.iter().cloned().collect());
-                        let env_lck = uid_store.envelopes.lock().unwrap();
-                        entry.unseen.lock().unwrap().insert_set(
-                            env_hashes
-                                .iter()
-                                .filter_map(|h| {
-                                    if !env_lck[h].inner.is_seen() {
-                                        Some(*h)
-                                    } else {
-                                        None
-                                    }
-                                })
-                                .collect(),
-                        );
-                    });
                 let env_lck = uid_store.envelopes.lock().unwrap();
-
                 return Ok(Some(
                     env_hashes
                         .into_iter()
@@ -699,7 +667,7 @@ pub(super) async fn fetch_cached_envs(state: &mut FetchState) -> Result<Option<V
                         .collect::<Vec<Envelope>>(),
                 ));
             }
-            Some(Err(err)) => return debug!(Err(err)),
+            Some(Err(err)) => return Err(err),
         }
     }
 }

melib/src/backends/imap/cache/sync.rs

@@ -63,54 +63,20 @@ impl ImapConnection {
             Ok(v) => v,
             Err(err) => return Some(Err(err)),
         };
-        match debug!(cache_handle.mailbox_state(mailbox_hash)) {
+        match cache_handle.mailbox_state(mailbox_hash) {
             Err(err) => return Some(Err(err)),
             Ok(Some(())) => {}
             Ok(None) => {
                 return None;
             }
         };
-        match debug!(cache_handle.envelopes(mailbox_hash)) {
+        match cache_handle.envelopes(mailbox_hash) {
             Ok(Some(envs)) => Some(Ok(envs)),
             Ok(None) => None,
             Err(err) => Some(Err(err)),
         }
     }
 
-    pub async fn build_cache(
-        &mut self,
-        cache_handle: &mut Box<dyn ImapCache>,
-        mailbox_hash: MailboxHash,
-    ) -> Result<()> {
-        debug!("build_cache {}", mailbox_hash);
-        let mut response = Vec::with_capacity(8 * 1024);
-        // 1 get uidvalidity, highestmodseq
-        let select_response = self
-            .select_mailbox(mailbox_hash, &mut response, true)
-            .await?
-            .unwrap();
-        self.uid_store
-            .uidvalidity
-            .lock()
-            .unwrap()
-            .insert(mailbox_hash, select_response.uidvalidity);
-        if let Some(v) = select_response.highestmodseq {
-            self.uid_store
-                .highestmodseqs
-                .lock()
-                .unwrap()
-                .insert(mailbox_hash, v);
-        }
-        cache_handle.clear(mailbox_hash, &select_response)?;
-        self.send_command(b"UID FETCH 1:* (UID FLAGS ENVELOPE BODYSTRUCTURE)")
-            .await?;
-        self.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
-            .await?;
-        let fetches = protocol_parser::fetch_responses(&response)?.1;
-        cache_handle.insert_envelopes(mailbox_hash, &fetches)?;
-        Ok(())
-    }
-
     //rfc4549_Synchronization_Operations_for_Disconnected_IMAP4_Clients
     pub async fn resync_basic(
         &mut self,
@@ -150,16 +116,10 @@ impl ImapConnection {
             )
         };
         let mut new_unseen = BTreeSet::default();
-        debug!("current_uidvalidity is {}", current_uidvalidity);
-        debug!("max_uid is {}", max_uid);
         let select_response = self
             .select_mailbox(mailbox_hash, &mut response, true)
             .await?
             .unwrap();
-        debug!(
-            "select_response.uidvalidity is {}",
-            select_response.uidvalidity
-        );
         // 1. check UIDVALIDITY. If fail, discard cache and rebuild
         if select_response.uidvalidity != current_uidvalidity {
             cache_handle.clear(mailbox_hash, &select_response)?;
@@ -170,7 +130,7 @@ impl ImapConnection {
         // 2.  tag1 UID FETCH <lastseenuid+1>:* <descriptors>
         self.send_command(
             format!(
-                "UID FETCH {}:* (UID FLAGS ENVELOPE BODYSTRUCTURE)",
+                "UID FETCH {}:* (UID FLAGS ENVELOPE BODY.PEEK[HEADER.FIELDS (REFERENCES)] BODYSTRUCTURE)",
                 max_uid + 1
             )
             .as_bytes(),
@@ -189,18 +149,22 @@ impl ImapConnection {
             ref uid,
             ref mut envelope,
             ref mut flags,
+            ref references,
             ..
         } in v.iter_mut()
         {
             let uid = uid.unwrap();
             let env = envelope.as_mut().unwrap();
             env.set_hash(generate_envelope_hash(&mailbox_path, &uid));
-            let mut tag_lck = self.uid_store.tag_index.write().unwrap();
+            if let Some(value) = references {
+                env.set_references(value);
+            }
+            let mut tag_lck = self.uid_store.collection.tag_index.write().unwrap();
             if let Some((flags, keywords)) = flags {
-                if !flags.intersects(Flag::SEEN) {
+                env.set_flags(*flags);
+                if !env.is_seen() {
                     new_unseen.insert(env.hash());
                 }
-                env.set_flags(*flags);
                 for f in keywords {
                     let hash = tag_hash!(f);
                     if !tag_lck.contains_key(&hash) {
@@ -211,14 +175,14 @@ impl ImapConnection {
             }
         }
         {
-            debug!(cache_handle
+            cache_handle
                 .insert_envelopes(mailbox_hash, &v)
                 .chain_err_summary(|| {
                     format!(
                         "Could not save envelopes in cache for mailbox {}",
                         mailbox_path
                     )
-                }))?;
+                })?;
         }
 
         for FetchResponse {
@@ -252,14 +216,17 @@ impl ImapConnection {
             payload.push((uid, env));
         }
         debug!("sending payload for {}", mailbox_hash);
-        unseen
-            .lock()
-            .unwrap()
-            .insert_existing_set(new_unseen.iter().cloned().collect());
-        mailbox_exists
-            .lock()
-            .unwrap()
-            .insert_existing_set(payload.iter().map(|(_, env)| env.hash()).collect::<_>());
+        let payload_hash_set: BTreeSet<_> =
+            payload.iter().map(|(_, env)| env.hash()).collect::<_>();
+        {
+            let mut unseen_lck = unseen.lock().unwrap();
+            for &seen_env_hash in payload_hash_set.difference(&new_unseen) {
+                unseen_lck.remove(seen_env_hash);
+            }
+
+            unseen_lck.insert_set(new_unseen);
+        }
+        mailbox_exists.lock().unwrap().insert_set(payload_hash_set);
         // 3.  tag2 UID FETCH 1:<lastseenuid> FLAGS
         if max_uid == 0 {
             self.send_command("UID FETCH 1:* FLAGS".as_bytes()).await?;
@@ -370,9 +337,6 @@ impl ImapConnection {
             .unwrap()
             .get(&mailbox_hash)
             .cloned();
-        debug!(&cached_uidvalidity);
-        debug!(&cached_max_uid);
-        debug!(&cached_highestmodseq);
         if cached_uidvalidity.is_none()
             || cached_max_uid.is_none()
             || cached_highestmodseq.is_none()
@@ -399,17 +363,11 @@ impl ImapConnection {
             )
         };
         let mut new_unseen = BTreeSet::default();
-        debug!("current_uidvalidity is {}", cached_uidvalidity);
-        debug!("max_uid is {}", cached_max_uid);
         // 1. check UIDVALIDITY. If fail, discard cache and rebuild
         let select_response = self
             .select_mailbox(mailbox_hash, &mut response, true)
             .await?
             .unwrap();
-        debug!(
-            "select_response.uidvalidity is {}",
-            select_response.uidvalidity
-        );
         if select_response.uidvalidity != cached_uidvalidity {
             // 1a) Check the mailbox UIDVALIDITY (see section 4.1 for more
             //details) with SELECT/EXAMINE/STATUS.
@@ -457,7 +415,7 @@ impl ImapConnection {
             // 2.  tag1 UID FETCH <lastseenuid+1>:* <descriptors>
             self.send_command(
                 format!(
-                    "UID FETCH {}:* (UID FLAGS ENVELOPE BODYSTRUCTURE) (CHANGEDSINCE {})",
+                    "UID FETCH {}:* (UID FLAGS ENVELOPE BODY.PEEK[HEADER.FIELDS (REFERENCES)] BODYSTRUCTURE) (CHANGEDSINCE {})",
                     cached_max_uid + 1,
                     cached_highestmodseq,
                 )
@@ -477,18 +435,22 @@ impl ImapConnection {
                 ref uid,
                 ref mut envelope,
                 ref mut flags,
+                ref references,
                 ..
             } in v.iter_mut()
             {
                 let uid = uid.unwrap();
                 let env = envelope.as_mut().unwrap();
                 env.set_hash(generate_envelope_hash(&mailbox_path, &uid));
-                let mut tag_lck = self.uid_store.tag_index.write().unwrap();
+                if let Some(value) = references {
+                    env.set_references(value);
+                }
+                let mut tag_lck = self.uid_store.collection.tag_index.write().unwrap();
                 if let Some((flags, keywords)) = flags {
-                    if !flags.intersects(Flag::SEEN) {
+                    env.set_flags(*flags);
+                    if !env.is_seen() {
                         new_unseen.insert(env.hash());
                     }
-                    env.set_flags(*flags);
                     for f in keywords {
                         let hash = tag_hash!(f);
                         if !tag_lck.contains_key(&hash) {
@@ -499,14 +461,14 @@ impl ImapConnection {
                 }
             }
             {
-                debug!(cache_handle
+                cache_handle
                     .insert_envelopes(mailbox_hash, &v)
                     .chain_err_summary(|| {
                         format!(
                             "Could not save envelopes in cache for mailbox {}",
                             mailbox_path
                         )
-                    }))?;
+                    })?;
             }
 
             for FetchResponse { uid, envelope, .. } in v {
@@ -534,14 +496,17 @@ impl ImapConnection {
                 payload.push((uid, env));
             }
             debug!("sending payload for {}", mailbox_hash);
-            unseen
-                .lock()
-                .unwrap()
-                .insert_existing_set(new_unseen.iter().cloned().collect());
-            mailbox_exists
-                .lock()
-                .unwrap()
-                .insert_existing_set(payload.iter().map(|(_, env)| env.hash()).collect::<_>());
+            let payload_hash_set: BTreeSet<_> =
+                payload.iter().map(|(_, env)| env.hash()).collect::<_>();
+            {
+                let mut unseen_lck = unseen.lock().unwrap();
+                for &seen_env_hash in payload_hash_set.difference(&new_unseen) {
+                    unseen_lck.remove(seen_env_hash);
+                }
+
+                unseen_lck.insert_set(new_unseen);
+            }
+            mailbox_exists.lock().unwrap().insert_set(payload_hash_set);
             // 3.  tag2 UID FETCH 1:<lastseenuid> FLAGS
             if cached_max_uid == 0 {
                 self.send_command(
@@ -660,12 +625,11 @@ impl ImapConnection {
 
     pub async fn init_mailbox(&mut self, mailbox_hash: MailboxHash) -> Result<SelectResponse> {
         let mut response = Vec::with_capacity(8 * 1024);
-        let (mailbox_path, mailbox_exists, unseen, permissions) = {
+        let (mailbox_path, mailbox_exists, permissions) = {
             let f = &self.uid_store.mailboxes.lock().await[&mailbox_hash];
             (
                 f.imap_path().to_string(),
                 f.exists.clone(),
-                f.unseen.clone(),
                 f.permissions.clone(),
             )
         };
@@ -702,14 +666,11 @@ impl ImapConnection {
             permissions.set_flags = !select_response.read_only;
             permissions.rename_messages = !select_response.read_only;
             permissions.delete_messages = !select_response.read_only;
-            mailbox_exists
-                .lock()
-                .unwrap()
-                .set_not_yet_seen(select_response.exists);
-            unseen
-                .lock()
-                .unwrap()
-                .set_not_yet_seen(select_response.unseen);
+            {
+                let mut mailbox_exists_lck = mailbox_exists.lock().unwrap();
+                mailbox_exists_lck.clear();
+                mailbox_exists_lck.set_not_yet_seen(select_response.exists);
+            }
         }
         if select_response.exists == 0 {
             return Ok(select_response);

melib/src/backends/imap/connection.rs

@@ -63,6 +63,7 @@ pub struct ImapExtensionUse {
     pub idle: bool,
     #[cfg(feature = "deflate_compression")]
     pub deflate: bool,
+    pub oauth2: bool,
 }
 
 impl Default for ImapExtensionUse {
@@ -72,6 +73,7 @@ impl Default for ImapExtensionUse {
             idle: true,
             #[cfg(feature = "deflate_compression")]
             deflate: true,
+            oauth2: false,
         }
     }
 }
@@ -146,26 +148,37 @@ impl ImapStream {
             ))
             .chain_err_kind(crate::error::ErrorKind::Network)?;
             if server_conf.use_starttls {
+                let err_fn = || {
+                    if server_conf.server_port == 993 {
+                        "STARTTLS failed. Server port is set to 993, which normally uses TLS. Maybe try disabling use_starttls."
+                    } else {
+                        "STARTTLS failed. Is the connection already encrypted?"
+                    }
+                };
                 let mut buf = vec![0; Connection::IO_BUF_SIZE];
                 match server_conf.protocol {
                     ImapProtocol::IMAP { .. } => socket
                         .write_all(format!("M{} STARTTLS\r\n", cmd_id).as_bytes())
                         .await
+                        .chain_err_summary(err_fn)
                         .chain_err_kind(crate::error::ErrorKind::Network)?,
                     ImapProtocol::ManageSieve => {
                         socket
                             .read(&mut buf)
                             .await
+                            .chain_err_summary(err_fn)
                             .chain_err_kind(crate::error::ErrorKind::Network)?;
                         socket
                             .write_all(b"STARTTLS\r\n")
                             .await
+                            .chain_err_summary(err_fn)
                             .chain_err_kind(crate::error::ErrorKind::Network)?;
                     }
                 }
                 socket
                     .flush()
                     .await
+                    .chain_err_summary(err_fn)
                     .chain_err_kind(crate::error::ErrorKind::Network)?;
                 let mut response = Vec::with_capacity(1024);
                 let mut broken = false;
@@ -175,6 +188,7 @@ impl ImapStream {
                     let len = socket
                         .read(&mut buf)
                         .await
+                        .chain_err_summary(err_fn)
                         .chain_err_kind(crate::error::ErrorKind::Network)?;
                     response.extend_from_slice(&buf[0..len]);
                     match server_conf.protocol {
@@ -200,7 +214,7 @@ impl ImapStream {
                 }
                 if !broken {
                     return Err(MeliError::new(format!(
-                        "Could not initiate TLS negotiation to {}.",
+                        "Could not initiate STARTTLS negotiation to {}.",
                         path
                     )));
                 }
@@ -232,8 +246,13 @@ impl ImapStream {
                     }
                 }
                 AsyncWrapper::new(Connection::Tls(
-                    conn_result.chain_err_kind(crate::error::ErrorKind::Network)?,
+                    conn_result
+                        .chain_err_summary(|| {
+                            format!("Could not initiate TLS negotiation to {}.", path)
+                        })
+                        .chain_err_kind(crate::error::ErrorKind::Network)?,
                 ))
+                .chain_err_summary(|| format!("Could not initiate TLS negotiation to {}.", path))
                 .chain_err_kind(crate::error::ErrorKind::Network)?
             }
         } else {
@@ -334,16 +353,39 @@ impl ImapStream {
             .set_err_kind(crate::error::ErrorKind::Authentication));
         }
 
-        let mut capabilities = None;
-        ret.send_command(
-            format!(
-                "LOGIN \"{}\" \"{}\"",
-                &server_conf.server_username, &server_conf.server_password
-            )
-            .as_bytes(),
-        )
-        .await?;
+        match server_conf.protocol {
+            ImapProtocol::IMAP {
+                extension_use: ImapExtensionUse { oauth2, .. },
+            } if oauth2 => {
+                if !capabilities
+                    .iter()
+                    .any(|cap| cap.eq_ignore_ascii_case(b"AUTH=XOAUTH2"))
+                {
+                    return Err(MeliError::new(format!(
+                                "Could not connect to {}: OAUTH2 is enabled but server did not return AUTH=XOAUTH2 capability. Returned capabilities were: {}",
+                                &server_conf.server_hostname,
+                                capabilities.iter().map(|capability|
+                                    String::from_utf8_lossy(capability).to_string()).collect::<Vec<String>>().join(" ")
+                    )));
+                }
+                ret.send_command(
+                    format!("AUTHENTICATE XOAUTH2 {}", &server_conf.server_password).as_bytes(),
+                )
+                .await?;
+            }
+            _ => {
+                ret.send_command(
+                    format!(
+                        "LOGIN \"{}\" \"{}\"",
+                        &server_conf.server_username, &server_conf.server_password
+                    )
+                    .as_bytes(),
+                )
+                .await?;
+            }
+        }
         let tag_start = format!("M{} ", (ret.cmd_id - 1));
+        let mut capabilities = None;
 
         loop {
             ret.read_lines(&mut res, &[], false).await?;
@@ -425,7 +467,6 @@ impl ImapStream {
                             if !termination_string.is_empty()
                                 && ret[last_line_idx..].starts_with(termination_string)
                             {
-                                debug!(&ret[last_line_idx..]);
                                 if !keep_termination_string {
                                     ret.splice(last_line_idx.., std::iter::empty::<u8>());
                                 }
@@ -475,9 +516,13 @@ impl ImapStream {
                 self.stream.flush().await?;
                 match self.protocol {
                     ImapProtocol::IMAP { .. } => {
-                        debug!("sent: M{} {}", self.cmd_id - 1, unsafe {
-                            std::str::from_utf8_unchecked(command)
-                        });
+                        if !command.starts_with(b"LOGIN") {
+                            debug!("sent: M{} {}", self.cmd_id - 1, unsafe {
+                                std::str::from_utf8_unchecked(command)
+                            });
+                        } else {
+                            debug!("sent: M{} LOGIN ..", self.cmd_id - 1);
+                        }
                     }
                     ImapProtocol::ManageSieve => {}
                 }
@@ -549,7 +594,7 @@ impl ImapConnection {
                     self.stream = Err(err);
                 }
             }
-            if debug!(self.stream.is_ok()) {
+            if self.stream.is_ok() {
                 let mut ret = Vec::new();
                 if let Err(err) = try_await(async {
                     self.send_command(b"NOOP").await?;
@@ -562,12 +607,12 @@ impl ImapConnection {
                 } else {
                     debug!(
                         "connect(): connection is probably alive, NOOP returned {:?}",
-                        &ret
+                        &String::from_utf8_lossy(&ret)
                     );
                     return Ok(());
                 }
             }
-            let new_stream = debug!(ImapStream::new_connection(&self.server_conf).await);
+            let new_stream = ImapStream::new_connection(&self.server_conf).await;
             if let Err(err) = new_stream.as_ref() {
                 self.uid_store.is_online.lock().unwrap().1 = Err(err.clone());
             } else {
@@ -583,6 +628,7 @@ impl ImapConnection {
                             #[cfg(feature = "deflate_compression")]
                             deflate,
                             idle: _idle,
+                            oauth2: _,
                         },
                 } => {
                     if capabilities.contains(&b"CONDSTORE"[..]) && condstore {
@@ -675,12 +721,16 @@ impl ImapConnection {
                         {
                             debug!(
                                 "Received expected NO response: {:?} {:?}",
-                                response_code, response
+                                response_code,
+                                String::from_utf8_lossy(&response)
                             );
                         }
                         ImapResponse::No(ref response_code) => {
-                            //FIXME return error
-                            debug!("Received NO response: {:?} {:?}", response_code, response);
+                            debug!(
+                                "Received NO response: {:?} {:?}",
+                                response_code,
+                                String::from_utf8_lossy(&response)
+                            );
                             (self.uid_store.event_consumer)(
                                 self.uid_store.account_hash,
                                 crate::backends::BackendEvent::Notice {
@@ -693,8 +743,11 @@ impl ImapConnection {
                             return r.into();
                         }
                         ImapResponse::Bad(ref response_code) => {
-                            //FIXME return error
-                            debug!("Received BAD response: {:?} {:?}", response_code, response);
+                            debug!(
+                                "Received BAD response: {:?} {:?}",
+                                response_code,
+                                String::from_utf8_lossy(&response)
+                            );
                             (self.uid_store.event_consumer)(
                                 self.uid_store.account_hash,
                                 crate::backends::BackendEvent::Notice {
@@ -987,15 +1040,10 @@ impl ImapConnection {
             .await?;
         self.read_response(&mut response, RequiredResponses::SEARCH)
             .await?;
-        debug!("uid search response {:?}", &response);
         let mut msn_index_lck = self.uid_store.msn_index.lock().unwrap();
         let msn_index = msn_index_lck.entry(mailbox_hash).or_default();
         let _ = msn_index.drain(low - 1..);
-        msn_index.extend(
-            debug!(protocol_parser::search_results(&response))?
-                .1
-                .into_iter(),
-        );
+        msn_index.extend(protocol_parser::search_results(&response)?.1.into_iter());
         Ok(())
     }
 }
@@ -1036,7 +1084,6 @@ impl ImapBlockingConnection {
         let mut prev_failure = None;
         async move {
             if self.conn.stream.is_err() {
-                debug!(&self.conn.stream);
                 return None;
             }
             loop {
@@ -1070,7 +1117,6 @@ async fn read(
         }
         Ok(b) => {
             result.extend_from_slice(&buf[0..b]);
-            debug!(unsafe { std::str::from_utf8_unchecked(result) });
             if let Some(pos) = result.find(b"\r\n") {
                 *prev_res_length = pos + b"\r\n".len();
                 return Some(result[0..*prev_res_length].to_vec());
@@ -1078,8 +1124,6 @@ async fn read(
             *prev_failure = None;
         }
         Err(_err) => {
-            debug!(&conn.stream);
-            debug!(&_err);
             *err = Some(Into::<MeliError>::into(_err).set_kind(crate::error::ErrorKind::Network));
             *break_flag = true;
             *prev_failure = Some(SystemTime::now());

melib/src/backends/imap/mailbox.rs

@@ -21,80 +21,11 @@
 
 use super::protocol_parser::SelectResponse;
 use crate::backends::{
-    BackendMailbox, Mailbox, MailboxHash, MailboxPermissions, SpecialUsageMailbox,
+    BackendMailbox, LazyCountSet, Mailbox, MailboxHash, MailboxPermissions, SpecialUsageMailbox,
 };
-use crate::email::EnvelopeHash;
 use crate::error::*;
-use std::collections::BTreeSet;
 use std::sync::{Arc, Mutex, RwLock};
 
-#[derive(Debug, Default, Clone)]
-pub struct LazyCountSet {
-    not_yet_seen: usize,
-    set: BTreeSet<EnvelopeHash>,
-}
-
-impl LazyCountSet {
-    pub fn set_not_yet_seen(&mut self, new_val: usize) {
-        self.not_yet_seen = new_val;
-    }
-
-    pub fn insert_existing(&mut self, new_val: EnvelopeHash) -> bool {
-        if self.not_yet_seen == 0 {
-            false
-        } else {
-            self.not_yet_seen -= 1;
-            self.set.insert(new_val);
-            true
-        }
-    }
-
-    pub fn insert_existing_set(&mut self, set: BTreeSet<EnvelopeHash>) -> bool {
-        debug!("insert_existing_set {:?}", &set);
-        if self.not_yet_seen < set.len() {
-            false
-        } else {
-            self.not_yet_seen -= set.len();
-            self.set.extend(set.into_iter());
-            true
-        }
-    }
-
-    #[inline(always)]
-    pub fn len(&self) -> usize {
-        self.set.len() + self.not_yet_seen
-    }
-
-    #[inline(always)]
-    pub fn clear(&mut self) {
-        self.set.clear();
-        self.not_yet_seen = 0;
-    }
-
-    pub fn insert_new(&mut self, new_val: EnvelopeHash) {
-        self.set.insert(new_val);
-    }
-
-    pub fn insert_set(&mut self, set: BTreeSet<EnvelopeHash>) {
-        debug!("insert__set {:?}", &set);
-        self.set.extend(set.into_iter());
-    }
-
-    pub fn remove(&mut self, new_val: EnvelopeHash) -> bool {
-        self.set.remove(&new_val)
-    }
-}
-
-#[test]
-fn test_lazy_count_set() {
-    let mut new = LazyCountSet::default();
-    new.set_not_yet_seen(10);
-    for i in 0..10 {
-        assert!(new.insert_existing(i));
-    }
-    assert!(!new.insert_existing(10));
-}
-
 #[derive(Debug, Default, Clone)]
 pub struct ImapMailbox {
     pub hash: MailboxHash,
@@ -112,12 +43,31 @@ pub struct ImapMailbox {
     pub permissions: Arc<Mutex<MailboxPermissions>>,
     pub exists: Arc<Mutex<LazyCountSet>>,
     pub unseen: Arc<Mutex<LazyCountSet>>,
+    pub warm: Arc<Mutex<bool>>,
 }
 
 impl ImapMailbox {
     pub fn imap_path(&self) -> &str {
         &self.imap_path
     }
+
+    /// Establish that mailbox contents have been fetched at least once during this execution
+    #[inline(always)]
+    pub fn set_warm(&self, new_value: bool) {
+        *self.warm.lock().unwrap() = new_value;
+    }
+
+    /// Mailbox contents have been fetched at least once during this execution
+    #[inline(always)]
+    pub fn is_warm(&self) -> bool {
+        *self.warm.lock().unwrap()
+    }
+
+    /// Mailbox contents have not been fetched at all during this execution
+    #[inline(always)]
+    pub fn is_cold(&self) -> bool {
+        !self.is_warm()
+    }
 }
 
 impl BackendMailbox for ImapMailbox {

melib/src/backends/imap/operations.rs

@@ -103,12 +103,11 @@ impl BackendOp for ImapOp {
                     //flags.lock().await.set(Some(_flags));
                     cache.flags = Some(_flags);
                 }
-                cache.bytes =
-                    Some(unsafe { std::str::from_utf8_unchecked(body.unwrap()).to_string() });
+                cache.bytes = Some(body.unwrap().to_vec());
             }
             let mut bytes_cache = uid_store.byte_cache.lock()?;
             let cache = bytes_cache.entry(uid).or_default();
-            let ret = cache.bytes.clone().unwrap().into_bytes();
+            let ret = cache.bytes.clone().unwrap();
             Ok(ret)
         }))
     }
@@ -145,9 +144,9 @@ impl BackendOp for ImapOp {
                     .map_err(MeliError::from)?;
                 if v.len() != 1 {
                     debug!("responses len is {}", v.len());
-                    debug!(&response);
+                    debug!(String::from_utf8_lossy(&response));
                     /* TODO: Trigger cache invalidation here. */
-                    debug!(format!("message with UID {} was not found", uid));
+                    debug!("message with UID {} was not found", uid);
                     return Err(MeliError::new(format!(
                         "Invalid/unexpected response: {:?}",
                         response

melib/src/backends/imap/protocol_parser.rs

@@ -314,46 +314,30 @@ fn test_imap_response() {
     assert_eq!(ImapResponse::try_from(&b"M12 NO [CANNOT] Invalid mailbox name: Name must not have \'/\' characters (0.000 + 0.098 + 0.097 secs).\r\n"[..]).unwrap(), ImapResponse::No(ResponseCode::Alert("Invalid mailbox name: Name must not have '/' characters".to_string())));
 }
 
-impl<'a> std::iter::DoubleEndedIterator for ImapLineIterator<'a> {
-    fn next_back(&mut self) -> Option<Self::Item> {
-        if self.slice.is_empty() {
-            None
-        } else if let Some(pos) = self.slice.rfind(b"\r\n") {
-            if self.slice.get(..pos).unwrap_or_default().is_empty() {
-                self.slice = self.slice.get(..pos).unwrap_or_default();
-                None
-            } else if let Some(prev_pos) = self.slice.get(..pos).unwrap_or_default().rfind(b"\r\n")
-            {
-                let ret = self.slice.get(prev_pos + 2..pos + 2).unwrap_or_default();
-                self.slice = self.slice.get(..prev_pos + 2).unwrap_or_default();
-                Some(ret)
-            } else {
-                let ret = self.slice;
-                self.slice = self.slice.get(ret.len()..).unwrap_or_default();
-                Some(ret)
-            }
-        } else {
-            let ret = self.slice;
-            self.slice = self.slice.get(ret.len()..).unwrap_or_default();
-            Some(ret)
-        }
-    }
-}
-
 impl<'a> Iterator for ImapLineIterator<'a> {
     type Item = &'a [u8];
 
     fn next(&mut self) -> Option<&'a [u8]> {
         if self.slice.is_empty() {
-            None
-        } else if let Some(pos) = self.slice.find(b"\r\n") {
-            let ret = self.slice.get(..pos + 2).unwrap_or_default();
-            self.slice = self.slice.get(pos + 2..).unwrap_or_default();
-            Some(ret)
-        } else {
-            let ret = self.slice;
-            self.slice = self.slice.get(ret.len()..).unwrap_or_default();
-            Some(ret)
+            return None;
+        }
+        let mut i = 0;
+        loop {
+            let cur_slice = &self.slice[i..];
+            if let Some(pos) = cur_slice.find(b"\r\n") {
+                /* Skip literal continuation line */
+                if cur_slice.get(pos.saturating_sub(1)) == Some(&b'}') {
+                    i += pos + 2;
+                    continue;
+                }
+                let ret = self.slice.get(..i + pos + 2).unwrap_or_default();
+                self.slice = self.slice.get(i + pos + 2..).unwrap_or_default();
+                return Some(ret);
+            } else {
+                let ret = self.slice;
+                self.slice = self.slice.get(ret.len()..).unwrap_or_default();
+                return Some(ret);
+            }
         }
     }
 }
@@ -372,6 +356,57 @@ macro_rules! to_str (
     ($v:expr) => (unsafe{ std::str::from_utf8_unchecked($v) })
 );
 
+#[test]
+fn test_imap_line_iterator() {
+    {
+        let s = b"* 1429 FETCH (UID 1505 FLAGS (\\Seen) RFC822 {26}\r\nReturn-Path: <blah blah...\r\n* 1430 FETCH (UID 1506 FLAGS (\\Seen)\r\n* 1431 FETCH (UID 1507 FLAGS (\\Seen)\r\n* 1432 FETCH (UID 1500 FLAGS (\\Seen) RFC822 {4}\r\nnull\r\n";
+        let line_a =
+            b"* 1429 FETCH (UID 1505 FLAGS (\\Seen) RFC822 {26}\r\nReturn-Path: <blah blah...\r\n";
+        let line_b = b"* 1430 FETCH (UID 1506 FLAGS (\\Seen)\r\n";
+        let line_c = b"* 1431 FETCH (UID 1507 FLAGS (\\Seen)\r\n";
+        let line_d = b"* 1432 FETCH (UID 1500 FLAGS (\\Seen) RFC822 {4}\r\nnull\r\n";
+
+        let mut iter = s.split_rn();
+
+        assert_eq!(to_str!(iter.next().unwrap()), to_str!(line_a));
+        assert_eq!(to_str!(iter.next().unwrap()), to_str!(line_b));
+        assert_eq!(to_str!(iter.next().unwrap()), to_str!(line_c));
+        assert_eq!(to_str!(iter.next().unwrap()), to_str!(line_d));
+        assert!(iter.next().is_none());
+    }
+
+    {
+        let s = b"* 23 FETCH (FLAGS (\\Seen) RFC822.SIZE 44827)\r\n";
+        let mut iter = s.split_rn();
+        assert_eq!(to_str!(iter.next().unwrap()), to_str!(s));
+        assert!(iter.next().is_none());
+    }
+
+    {
+        let s = b"";
+        let mut iter = s.split_rn();
+        assert!(iter.next().is_none());
+    }
+    {
+        let s = b"* 172 EXISTS\r\n* 1 RECENT\r\n* OK [UNSEEN 12] Message 12 is first unseen\r\n* OK [UIDVALIDITY 3857529045] UIDs valid\r\n* OK [UIDNEXT 4392] Predicted next UID\r\n* FLAGS (\\Answered \\Flagged \\Deleted \\Seen \\Draft)\r\n* OK [PERMANENTFLAGS (\\Deleted \\Seen \\*)] Limited\r\n* OK [NOMODSEQ] Sorry, this mailbox format doesn't support modsequences\r\n* A142 OK [READ-WRITE] SELECT completed\r\n";
+        let mut iter = s.split_rn();
+        for l in &[
+            &b"* 172 EXISTS\r\n"[..],
+            &b"* 1 RECENT\r\n"[..],
+            &b"* OK [UNSEEN 12] Message 12 is first unseen\r\n"[..],
+            &b"* OK [UIDVALIDITY 3857529045] UIDs valid\r\n"[..],
+            &b"* OK [UIDNEXT 4392] Predicted next UID\r\n"[..],
+            &b"* FLAGS (\\Answered \\Flagged \\Deleted \\Seen \\Draft)\r\n"[..],
+            &b"* OK [PERMANENTFLAGS (\\Deleted \\Seen \\*)] Limited\r\n"[..],
+            &b"* OK [NOMODSEQ] Sorry, this mailbox format doesn't support modsequences\r\n"[..],
+            &b"* A142 OK [READ-WRITE] SELECT completed\r\n"[..],
+        ] {
+            assert_eq!(to_str!(iter.next().unwrap()), to_str!(l));
+        }
+        assert!(iter.next().is_none());
+    }
+}
+
 /*macro_rules! dbg_dmp (
   ($i: expr, $submac:ident!( $($args:tt)* )) => (
     {
@@ -415,7 +450,13 @@ pub fn list_mailbox_result(input: &[u8]) -> IResult<&[u8], ImapMailbox> {
             let separator: u8 = separator[0];
             let mut f = ImapMailbox::default();
             f.no_select = false;
-            f.is_subscribed = path.eq_ignore_ascii_case("INBOX");
+            f.is_subscribed = false;
+
+            if path.eq_ignore_ascii_case("INBOX") {
+                f.is_subscribed = true;
+                let _ = f.set_special_usage(SpecialUsageMailbox::Inbox);
+            }
+
             for p in properties.split(|&b| b == b' ') {
                 if p.eq_ignore_ascii_case(b"\\NoSelect") || p.eq_ignore_ascii_case(b"\\NonExistent")
                 {
@@ -425,9 +466,15 @@ pub fn list_mailbox_result(input: &[u8]) -> IResult<&[u8], ImapMailbox> {
                 } else if p.eq_ignore_ascii_case(b"\\Sent") {
                     let _ = f.set_special_usage(SpecialUsageMailbox::Sent);
                 } else if p.eq_ignore_ascii_case(b"\\Junk") {
+                    let _ = f.set_special_usage(SpecialUsageMailbox::Junk);
+                } else if p.eq_ignore_ascii_case(b"\\Trash") {
                     let _ = f.set_special_usage(SpecialUsageMailbox::Trash);
                 } else if p.eq_ignore_ascii_case(b"\\Drafts") {
                     let _ = f.set_special_usage(SpecialUsageMailbox::Drafts);
+                } else if p.eq_ignore_ascii_case(b"\\Flagged") {
+                    let _ = f.set_special_usage(SpecialUsageMailbox::Flagged);
+                } else if p.eq_ignore_ascii_case(b"\\Archive") {
+                    let _ = f.set_special_usage(SpecialUsageMailbox::Archive);
                 }
             }
             f.imap_path = path.to_string();
@@ -458,7 +505,9 @@ pub struct FetchResponse<'a> {
     pub modseq: Option<ModSequence>,
     pub flags: Option<(Flag, Vec<String>)>,
     pub body: Option<&'a [u8]>,
+    pub references: Option<&'a [u8]>,
     pub envelope: Option<Envelope>,
+    pub raw_fetch_value: &'a [u8],
 }
 
 pub fn fetch_response(input: &[u8]) -> ImapParseResult<FetchResponse<'_>> {
@@ -513,7 +562,9 @@ pub fn fetch_response(input: &[u8]) -> ImapParseResult<FetchResponse<'_>> {
         modseq: None,
         flags: None,
         body: None,
+        references: None,
         envelope: None,
+        raw_fetch_value: &[],
     };
 
     while input[i].is_ascii_digit() {
@@ -609,6 +660,22 @@ pub fn fetch_response(input: &[u8]) -> ImapParseResult<FetchResponse<'_>> {
             let (rest, _has_attachments) = bodystructure_has_attachments(&input[i..])?;
             has_attachments = _has_attachments;
             i += input[i..].len() - rest.len();
+        } else if input[i..].starts_with(b"BODY[HEADER.FIELDS (REFERENCES)] ") {
+            i += b"BODY[HEADER.FIELDS (REFERENCES)] ".len();
+            if let Ok((rest, mut references)) = astring_token(&input[i..]) {
+                if !references.trim().is_empty() {
+                    if let Ok((_, (_, v))) = crate::email::parser::headers::header(&references) {
+                        references = v;
+                    }
+                    ret.references = Some(references);
+                }
+                i += input.len() - i - rest.len();
+            } else {
+                return debug!(Err(MeliError::new(format!(
+                    "Unexpected input while parsing UID FETCH response. Got: `{:.40}`",
+                    String::from_utf8_lossy(&input[i..])
+                ))));
+            }
         } else if input[i..].starts_with(b")\r\n") {
             i += b")\r\n".len();
             break;
@@ -623,6 +690,7 @@ pub fn fetch_response(input: &[u8]) -> ImapParseResult<FetchResponse<'_>> {
             ))));
         }
     }
+    ret.raw_fetch_value = &input[..i];
 
     if let Some(env) = ret.envelope.as_mut() {
         env.set_has_attachments(has_attachments);
@@ -823,7 +891,10 @@ pub fn untagged_responses(input: &[u8]) -> ImapParseResult<Option<UntaggedRespon
     let (input, _tag) =
         take_until::<_, &[u8], (&[u8], nom::error::ErrorKind)>(&b"\r\n"[..])(input)?;
     let (input, _) = tag::<_, &[u8], (&[u8], nom::error::ErrorKind)>(b"\r\n")(input)?;
-    debug!("Parse untagged response from {:?}", orig_input);
+    debug!(
+        "Parse untagged response from {:?}",
+        String::from_utf8_lossy(&orig_input)
+    );
     Ok((
         input,
         {
@@ -848,7 +919,6 @@ pub fn untagged_responses(input: &[u8]) -> ImapParseResult<Option<UntaggedRespon
 
 #[test]
 fn test_untagged_responses() {
-    use std::convert::TryInto;
     use UntaggedResponse::*;
     assert_eq!(
         untagged_responses(b"* 2 EXISTS\r\n")
@@ -868,7 +938,9 @@ fn test_untagged_responses() {
             modseq: Some(ModSequence(std::num::NonZeroU64::new(1365_u64).unwrap())),
             flags: Some((Flag::SEEN, vec![])),
             body: None,
-            envelope: None
+            references: None,
+            envelope: None,
+            raw_fetch_value: &b"* 1079 FETCH (UID 1103 MODSEQ (1365) FLAGS (\\Seen))\r\n"[..],
         })
     );
     assert_eq!(
@@ -882,7 +954,9 @@ fn test_untagged_responses() {
             modseq: None,
             flags: Some((Flag::SEEN, vec![])),
             body: None,
-            envelope: None
+            references: None,
+            envelope: None,
+            raw_fetch_value: &b"* 1 FETCH (FLAGS (\\Seen))\r\n"[..],
         })
     );
 }
@@ -944,7 +1018,7 @@ pub struct SelectResponse {
     pub exists: ImapNum,
     pub recent: ImapNum,
     pub flags: (Flag, Vec<String>),
-    pub unseen: MessageSequenceNumber,
+    pub first_unseen: MessageSequenceNumber,
     pub uidvalidity: UIDVALIDITY,
     pub uidnext: UID,
     pub permanentflags: (Flag, Vec<String>),
@@ -991,7 +1065,7 @@ pub fn select_response(input: &[u8]) -> Result<SelectResponse> {
             } else if l.starts_with(b"* FLAGS (") {
                 ret.flags = flags(&l[b"* FLAGS (".len()..l.len() - b")".len()]).map(|(_, v)| v)?;
             } else if l.starts_with(b"* OK [UNSEEN ") {
-                ret.unseen = MessageSequenceNumber::from_str(&String::from_utf8_lossy(
+                ret.first_unseen = MessageSequenceNumber::from_str(&String::from_utf8_lossy(
                     &l[b"* OK [UNSEEN ".len()..l.find(b"]").unwrap()],
                 ))?;
             } else if l.starts_with(b"* OK [UIDVALIDITY ") {
@@ -1038,7 +1112,6 @@ pub fn select_response(input: &[u8]) -> Result<SelectResponse> {
 
 #[test]
 fn test_select_response() {
-    use std::convert::TryInto;
     let r = b"* FLAGS (\\Answered \\Flagged \\Deleted \\Seen \\Draft)\r\n* OK [PERMANENTFLAGS (\\Answered \\Flagged \\Deleted \\Seen \\Draft \\*)] Flags permitted.\r\n* 45 EXISTS\r\n* 0 RECENT\r\n* OK [UNSEEN 16] First unseen.\r\n* OK [UIDVALIDITY 1554422056] UIDs valid\r\n* OK [UIDNEXT 50] Predicted next UID\r\n";
 
     assert_eq!(
@@ -1050,7 +1123,7 @@ fn test_select_response() {
                 Flag::REPLIED | Flag::SEEN | Flag::TRASHED | Flag::DRAFT | Flag::FLAGGED,
                 Vec::new()
             ),
-            unseen: 16,
+            first_unseen: 16,
             uidvalidity: 1554422056,
             uidnext: 50,
             permanentflags: (
@@ -1073,7 +1146,7 @@ fn test_select_response() {
                 Flag::REPLIED | Flag::SEEN | Flag::TRASHED | Flag::DRAFT | Flag::FLAGGED,
                 Vec::new()
             ),
-            unseen: 12,
+            first_unseen: 12,
             uidvalidity: 3857529045,
             uidnext: 4392,
             permanentflags: (Flag::SEEN | Flag::TRASHED, vec!["*".into()]),
@@ -1095,7 +1168,7 @@ fn test_select_response() {
                 Flag::REPLIED | Flag::SEEN | Flag::TRASHED | Flag::DRAFT | Flag::FLAGGED,
                 Vec::new()
             ),
-            unseen: 12,
+            first_unseen: 12,
             uidvalidity: 3857529045,
             uidnext: 4392,
             permanentflags: (Flag::SEEN | Flag::TRASHED, vec!["*".into()]),
@@ -1112,7 +1185,8 @@ pub fn flags(input: &[u8]) -> IResult<&[u8], (Flag, Vec<String>)> {
 
     let mut input = input;
     while !input.starts_with(b")") && !input.is_empty() {
-        if input.starts_with(b"\\") {
+        let is_system_flag = input.starts_with(b"\\");
+        if is_system_flag {
             input = &input[1..];
         }
         let mut match_end = 0;
@@ -1123,23 +1197,24 @@ pub fn flags(input: &[u8]) -> IResult<&[u8], (Flag, Vec<String>)> {
             match_end += 1;
         }
 
-        match &input[..match_end] {
-            b"Answered" => {
+        match (is_system_flag, &input[..match_end]) {
+            (true, t) if t.eq_ignore_ascii_case(b"Answered") => {
                 ret.set(Flag::REPLIED, true);
             }
-            b"Flagged" => {
+            (true, t) if t.eq_ignore_ascii_case(b"Flagged") => {
                 ret.set(Flag::FLAGGED, true);
             }
-            b"Deleted" => {
+            (true, t) if t.eq_ignore_ascii_case(b"Deleted") => {
                 ret.set(Flag::TRASHED, true);
             }
-            b"Seen" => {
+            (true, t) if t.eq_ignore_ascii_case(b"Seen") => {
                 ret.set(Flag::SEEN, true);
             }
-            b"Draft" => {
+            (true, t) if t.eq_ignore_ascii_case(b"Draft") => {
                 ret.set(Flag::DRAFT, true);
             }
-            f => {
+            (true, t) if t.eq_ignore_ascii_case(b"Recent") => { /* ignore */ }
+            (_, f) => {
                 keywords.push(String::from_utf8_lossy(&f).into());
             }
         }
@@ -1243,7 +1318,9 @@ pub fn envelope(input: &[u8]) -> IResult<&[u8], Envelope> {
             }
             if let Some(in_reply_to) = in_reply_to {
                 env.set_in_reply_to(&in_reply_to);
-                env.push_references(env.in_reply_to().unwrap().clone());
+                if let Some(in_reply_to) = env.in_reply_to().cloned() {
+                    env.push_references(in_reply_to);
+                }
             }
 
             if let Some(message_id) = message_id {
@@ -1420,7 +1497,7 @@ pub fn bodystructure_has_attachments(input: &[u8]) -> IResult<&[u8], bool> {
     let mut has_attachments = false;
     let mut first_in_line = true;
     while !input.is_empty() && !input.starts_with(b")") {
-        if input.starts_with(b"\"") || input[0].is_ascii_alphanumeric() {
+        if input.starts_with(b"\"") || input[0].is_ascii_alphanumeric() || input[0] == b'{' {
             let (_input, token) = astring_token(input)?;
             input = _input;
             if first_in_line {

melib/src/backends/imap/untagged.rs

@@ -28,14 +28,13 @@ use crate::backends::{
     RefreshEvent,
     RefreshEventKind::{self, *},
 };
-use crate::email::Envelope;
 use crate::error::*;
 use std::convert::TryInto;
 
 impl ImapConnection {
     pub async fn process_untagged(&mut self, line: &[u8]) -> Result<bool> {
         macro_rules! try_fail {
-            ($mailbox_hash: expr, $($result:expr)+) => {
+            ($mailbox_hash: expr, $($result:expr $(,)*)+) => {
                 $(if let Err(err) = $result {
                     self.uid_store.is_online.lock().unwrap().1 = Err(err.clone());
                     debug!("failure: {}", err.to_string());
@@ -78,7 +77,7 @@ impl ImapConnection {
                     .lock()
                     .unwrap()
                     .get(&mailbox_hash)
-                    .map(|i| i.len() < n.try_into().unwrap())
+                    .map(|i| i.len() < TryInto::<usize>::try_into(n).unwrap())
                     .unwrap_or(true)
                 {
                     debug!(
@@ -86,6 +85,64 @@ impl ImapConnection {
                         n,
                         self.uid_store.msn_index.lock().unwrap().get(&mailbox_hash)
                     );
+                    self.send_command("UID SEARCH 1:*".as_bytes()).await?;
+                    self.read_response(&mut response, RequiredResponses::SEARCH)
+                        .await?;
+                    let results = super::protocol_parser::search_results(&response)?
+                        .1
+                        .into_iter()
+                        .collect::<std::collections::BTreeSet<UID>>();
+                    {
+                        let mut lck = self.uid_store.msn_index.lock().unwrap();
+                        let msn_index = lck.entry(mailbox_hash).or_default();
+                        msn_index.clear();
+                        msn_index.extend(
+                            super::protocol_parser::search_results(&response)?
+                                .1
+                                .into_iter(),
+                        );
+                    }
+                    let mut events = vec![];
+                    for (deleted_uid, deleted_hash) in self
+                        .uid_store
+                        .uid_index
+                        .lock()
+                        .unwrap()
+                        .iter()
+                        .filter(|((mailbox_hash_, u), _)| {
+                            *mailbox_hash_ == mailbox_hash && !results.contains(u)
+                        })
+                        .map(|((_, uid), hash)| (*uid, *hash))
+                        .collect::<Vec<(UID, crate::email::EnvelopeHash)>>()
+                    {
+                        mailbox.exists.lock().unwrap().remove(deleted_hash);
+                        mailbox.unseen.lock().unwrap().remove(deleted_hash);
+                        self.uid_store
+                            .uid_index
+                            .lock()
+                            .unwrap()
+                            .remove(&(mailbox_hash, deleted_uid));
+                        self.uid_store
+                            .hash_index
+                            .lock()
+                            .unwrap()
+                            .remove(&deleted_hash);
+                        events.push((
+                            deleted_uid,
+                            RefreshEvent {
+                                account_hash: self.uid_store.account_hash,
+                                mailbox_hash,
+                                kind: Remove(deleted_hash),
+                            },
+                        ));
+                    }
+                    if self.uid_store.keep_offline_cache {
+                        cache_handle.update(mailbox_hash, &events)?;
+                    }
+                    for (_, event) in events {
+                        self.add_refresh_event(event);
+                    }
+
                     return Ok(true);
                 }
                 let deleted_uid = self
@@ -95,7 +152,7 @@ impl ImapConnection {
                     .unwrap()
                     .entry(mailbox_hash)
                     .or_default()
-                    .remove(n.try_into().unwrap());
+                    .remove(TryInto::<usize>::try_into(n).unwrap().saturating_sub(1));
                 debug!("expunge {}, UID = {}", n, deleted_uid);
                 let deleted_hash: crate::email::EnvelopeHash = match self
                     .uid_store
@@ -107,6 +164,8 @@ impl ImapConnection {
                     Some(v) => v,
                     None => return Ok(true),
                 };
+                mailbox.exists.lock().unwrap().remove(deleted_hash);
+                mailbox.unseen.lock().unwrap().remove(deleted_hash);
                 self.uid_store
                     .hash_index
                     .lock()
@@ -136,94 +195,108 @@ impl ImapConnection {
                 debug!("exists {}", n);
                 try_fail!(
                     mailbox_hash,
-                    self.send_command(format!("FETCH {} (UID FLAGS RFC822)", n).as_bytes()).await
+                    self.send_command(format!("FETCH {} (UID FLAGS ENVELOPE BODY.PEEK[HEADER.FIELDS (REFERENCES)] BODYSTRUCTURE)", n).as_bytes()).await
                     self.read_response(&mut response, RequiredResponses::FETCH_REQUIRED).await
                 );
-                match super::protocol_parser::fetch_responses(&response) {
-                    Ok((_, v, _)) => {
-                        'fetch_responses: for FetchResponse {
-                            uid, flags, body, ..
-                        } in v
-                        {
-                            if uid.is_none() || flags.is_none() || body.is_none() {
-                                continue;
-                            }
-                            let uid = uid.unwrap();
-                            if self
-                                .uid_store
-                                .uid_index
-                                .lock()
-                                .unwrap()
-                                .contains_key(&(mailbox_hash, uid))
-                            {
-                                continue 'fetch_responses;
-                            }
-                            let env_hash = generate_envelope_hash(&mailbox.imap_path(), &uid);
-                            self.uid_store
-                                .msn_index
-                                .lock()
-                                .unwrap()
-                                .entry(mailbox_hash)
-                                .or_default()
-                                .push(uid);
-                            if let Ok(mut env) =
-                                Envelope::from_bytes(body.unwrap(), flags.as_ref().map(|&(f, _)| f))
-                            {
-                                env.set_hash(env_hash);
-                                self.uid_store
-                                    .hash_index
-                                    .lock()
-                                    .unwrap()
-                                    .insert(env_hash, (uid, mailbox_hash));
-                                self.uid_store
-                                    .uid_index
-                                    .lock()
-                                    .unwrap()
-                                    .insert((mailbox_hash, uid), env_hash);
-                                if let Some((_, keywords)) = flags {
-                                    let mut tag_lck = self.uid_store.tag_index.write().unwrap();
-                                    for f in keywords {
-                                        let hash = tag_hash!(f);
-                                        if !tag_lck.contains_key(&hash) {
-                                            tag_lck.insert(hash, f);
-                                        }
-                                        env.labels_mut().push(hash);
-                                    }
-                                }
-                                debug!(
-                                    "Create event {} {} {}",
-                                    env.hash(),
-                                    env.subject(),
-                                    mailbox.path(),
-                                );
-                                if !env.is_seen() {
-                                    mailbox.unseen.lock().unwrap().insert_new(env.hash());
-                                }
-                                mailbox.exists.lock().unwrap().insert_new(env.hash());
-                                let mut event: [(UID, RefreshEvent); 1] = [(
-                                    uid,
-                                    RefreshEvent {
-                                        account_hash: self.uid_store.account_hash,
-                                        mailbox_hash,
-                                        kind: Create(Box::new(env)),
-                                    },
-                                )];
-                                if self.uid_store.keep_offline_cache {
-                                    cache_handle.update(mailbox_hash, &event)?;
-                                }
-                                self.add_refresh_event(std::mem::replace(
-                                    &mut event[0].1,
-                                    RefreshEvent {
-                                        account_hash: self.uid_store.account_hash,
-                                        mailbox_hash,
-                                        kind: Rescan,
-                                    },
-                                ));
+                let mut v = match super::protocol_parser::fetch_responses(&response) {
+                    Ok((_, v, _)) => v,
+                    Err(err) => {
+                        debug!(
+                            "Error when parsing FETCH response after untagged exists {:?}",
+                            err
+                        );
+                        return Ok(true);
+                    }
+                };
+                debug!("responses len is {}", v.len());
+                for FetchResponse {
+                    ref uid,
+                    ref mut envelope,
+                    ref mut flags,
+                    ref references,
+                    ..
+                } in &mut v
+                {
+                    if uid.is_none() || flags.is_none() || envelope.is_none() {
+                        continue;
+                    }
+                    let uid = uid.unwrap();
+                    let env = envelope.as_mut().unwrap();
+                    env.set_hash(generate_envelope_hash(&mailbox.imap_path(), &uid));
+                    if let Some(value) = references {
+                        env.set_references(value);
+                    }
+                    let mut tag_lck = self.uid_store.collection.tag_index.write().unwrap();
+                    if let Some((flags, keywords)) = flags {
+                        env.set_flags(*flags);
+                        if !env.is_seen() {
+                            mailbox.unseen.lock().unwrap().insert_new(env.hash());
+                        }
+                        for f in keywords {
+                            let hash = tag_hash!(f);
+                            if !tag_lck.contains_key(&hash) {
+                                tag_lck.insert(hash, f.to_string());
                             }
+                            env.labels_mut().push(hash);
                         }
                     }
-                    Err(e) => {
-                        debug!(e);
+                    mailbox.exists.lock().unwrap().insert_new(env.hash());
+                    if !self
+                        .uid_store
+                        .uid_index
+                        .lock()
+                        .unwrap()
+                        .contains_key(&(mailbox_hash, uid))
+                    {
+                        self.uid_store
+                            .msn_index
+                            .lock()
+                            .unwrap()
+                            .entry(mailbox_hash)
+                            .or_default()
+                            .push(uid);
+                    }
+                    self.uid_store
+                        .hash_index
+                        .lock()
+                        .unwrap()
+                        .insert(env.hash(), (uid, mailbox_hash));
+                    self.uid_store
+                        .uid_index
+                        .lock()
+                        .unwrap()
+                        .insert((mailbox_hash, uid), env.hash());
+                    debug!(
+                        "Create event {} {} {}",
+                        env.hash(),
+                        env.subject(),
+                        mailbox.path(),
+                    );
+                }
+                if self.uid_store.keep_offline_cache {
+                    if let Err(err) = cache_handle
+                        .insert_envelopes(mailbox_hash, &v)
+                        .chain_err_summary(|| {
+                            format!(
+                                "Could not save envelopes in cache for mailbox {}",
+                                &mailbox.imap_path()
+                            )
+                        })
+                    {
+                        crate::log(err.to_string(), crate::INFO);
+                    }
+                }
+                for response in v {
+                    if let FetchResponse {
+                        envelope: Some(envelope),
+                        ..
+                    } = response
+                    {
+                        self.add_refresh_event(RefreshEvent {
+                            account_hash: self.uid_store.account_hash,
+                            mailbox_hash,
+                            kind: Create(Box::new(envelope)),
+                        });
                     }
                 }
             }
@@ -241,97 +314,120 @@ impl ImapConnection {
                         debug!("UID SEARCH RECENT returned no results");
                     }
                     Ok(v) => {
+                        let command = {
+                            let mut iter = v.split(u8::is_ascii_whitespace);
+                            let first = iter.next().unwrap_or(v);
+                            let mut accum = format!("{}", to_str!(first).trim());
+                            for ms in iter {
+                                accum = format!("{},{}", accum, to_str!(ms).trim());
+                            }
+                            format!("UID FETCH {} (UID FLAGS ENVELOPE BODY.PEEK[HEADER.FIELDS (REFERENCES)] BODYSTRUCTURE)", accum)
+                        };
                         try_fail!(
                             mailbox_hash,
-                            self.send_command(
-                                &[b"UID FETCH", v, b"(FLAGS RFC822)"]
-                                .join(&b' '),
-                                ).await
+                            self.send_command(command.as_bytes()).await
                             self.read_response(&mut response, RequiredResponses::FETCH_REQUIRED).await
                         );
-                        debug!(&response);
-                        match super::protocol_parser::fetch_responses(&response) {
-                            Ok((_, v, _)) => {
-                                for FetchResponse {
-                                    uid, flags, body, ..
-                                } in v
-                                {
-                                    if uid.is_none() || flags.is_none() || body.is_none() {
-                                        continue;
-                                    }
-                                    let uid = uid.unwrap();
-                                    if !self
-                                        .uid_store
-                                        .uid_index
-                                        .lock()
-                                        .unwrap()
-                                        .contains_key(&(mailbox_hash, uid))
-                                    {
-                                        if let Ok(mut env) = Envelope::from_bytes(
-                                            body.unwrap(),
-                                            flags.as_ref().map(|&(f, _)| f),
-                                        ) {
-                                            self.uid_store
-                                                .hash_index
-                                                .lock()
-                                                .unwrap()
-                                                .insert(env.hash(), (uid, mailbox_hash));
-                                            self.uid_store
-                                                .uid_index
-                                                .lock()
-                                                .unwrap()
-                                                .insert((mailbox_hash, uid), env.hash());
-                                            debug!(
-                                                "Create event {} {} {}",
-                                                env.hash(),
-                                                env.subject(),
-                                                mailbox.path(),
-                                            );
-                                            if let Some((_, keywords)) = flags {
-                                                let mut tag_lck =
-                                                    self.uid_store.tag_index.write().unwrap();
-                                                for f in keywords {
-                                                    let hash = tag_hash!(f);
-                                                    if !tag_lck.contains_key(&hash) {
-                                                        tag_lck.insert(hash, f);
-                                                    }
-                                                    env.labels_mut().push(hash);
-                                                }
-                                            }
-                                            if !env.is_seen() {
-                                                mailbox
-                                                    .unseen
-                                                    .lock()
-                                                    .unwrap()
-                                                    .insert_new(env.hash());
-                                            }
-
-                                            mailbox.exists.lock().unwrap().insert_new(env.hash());
-                                            let mut event: [(UID, RefreshEvent); 1] = [(
-                                                uid,
-                                                RefreshEvent {
-                                                    account_hash: self.uid_store.account_hash,
-                                                    mailbox_hash,
-                                                    kind: Create(Box::new(env)),
-                                                },
-                                            )];
-                                            if self.uid_store.keep_offline_cache {
-                                                cache_handle.update(mailbox_hash, &event)?;
-                                            }
-                                            self.add_refresh_event(std::mem::replace(
-                                                &mut event[0].1,
-                                                RefreshEvent {
-                                                    account_hash: self.uid_store.account_hash,
-                                                    mailbox_hash,
-                                                    kind: Rescan,
-                                                },
-                                            ));
-                                        }
+                        let mut v = match super::protocol_parser::fetch_responses(&response) {
+                            Ok((_, v, _)) => v,
+                            Err(err) => {
+                                debug!(
+                                    "Error when parsing FETCH response after untagged recent {:?}",
+                                    err
+                                );
+                                return Ok(true);
+                            }
+                        };
+                        debug!("responses len is {}", v.len());
+                        for FetchResponse {
+                            ref uid,
+                            ref mut envelope,
+                            ref mut flags,
+                            ref references,
+                            ..
+                        } in &mut v
+                        {
+                            if uid.is_none() || flags.is_none() || envelope.is_none() {
+                                continue;
+                            }
+                            let uid = uid.unwrap();
+                            let env = envelope.as_mut().unwrap();
+                            env.set_hash(generate_envelope_hash(&mailbox.imap_path(), &uid));
+                            if let Some(value) = references {
+                                env.set_references(value);
+                            }
+                            let mut tag_lck = self.uid_store.collection.tag_index.write().unwrap();
+                            if let Some((flags, keywords)) = flags {
+                                env.set_flags(*flags);
+                                if !env.is_seen() {
+                                    mailbox.unseen.lock().unwrap().insert_new(env.hash());
+                                }
+                                for f in keywords {
+                                    let hash = tag_hash!(f);
+                                    if !tag_lck.contains_key(&hash) {
+                                        tag_lck.insert(hash, f.to_string());
                                     }
+                                    env.labels_mut().push(hash);
                                 }
                             }
-                            Err(e) => {
-                                debug!(e);
+                            mailbox.exists.lock().unwrap().insert_new(env.hash());
+                        }
+                        if self.uid_store.keep_offline_cache {
+                            if let Err(err) = cache_handle
+                                .insert_envelopes(mailbox_hash, &v)
+                                .chain_err_summary(|| {
+                                    format!(
+                                        "Could not save envelopes in cache for mailbox {}",
+                                        &mailbox.imap_path()
+                                    )
+                                })
+                            {
+                                crate::log(err.to_string(), crate::INFO);
+                            }
+                        }
+                        for response in v {
+                            if let FetchResponse {
+                                envelope: Some(envelope),
+                                uid: Some(uid),
+                                ..
+                            } = response
+                            {
+                                if !self
+                                    .uid_store
+                                    .uid_index
+                                    .lock()
+                                    .unwrap()
+                                    .contains_key(&(mailbox_hash, uid))
+                                {
+                                    self.uid_store
+                                        .msn_index
+                                        .lock()
+                                        .unwrap()
+                                        .entry(mailbox_hash)
+                                        .or_default()
+                                        .push(uid);
+                                }
+                                self.uid_store
+                                    .hash_index
+                                    .lock()
+                                    .unwrap()
+                                    .insert(envelope.hash(), (uid, mailbox_hash));
+                                self.uid_store
+                                    .uid_index
+                                    .lock()
+                                    .unwrap()
+                                    .insert((mailbox_hash, uid), envelope.hash());
+                                debug!(
+                                    "Create event {} {} {}",
+                                    envelope.hash(),
+                                    envelope.subject(),
+                                    mailbox.path(),
+                                );
+                                self.add_refresh_event(RefreshEvent {
+                                    account_hash: self.uid_store.account_hash,
+                                    mailbox_hash,
+                                    kind: Create(Box::new(envelope)),
+                                });
                             }
                         }
                     }
@@ -350,38 +446,21 @@ impl ImapConnection {
                 modseq,
                 flags,
                 body: _,
+                references: _,
                 envelope: _,
+                raw_fetch_value: _,
             }) => {
-                if let Some(modseq) = modseq {
-                    if self
-                        .uid_store
-                        .reverse_modseq
-                        .lock()
-                        .unwrap()
-                        .entry(mailbox_hash)
-                        .or_default()
-                        .contains_key(&modseq)
-                    {
-                        return Ok(true);
-                    }
-                }
-
                 if let Some(flags) = flags {
                     let uid = if let Some(uid) = uid {
                         uid
                     } else {
                         try_fail!(
-                                mailbox_hash,
-                        self.send_command(
-                            &[
-                            b"UID SEARCH",
-                            format!("{}", msg_seq).as_bytes(),
-                            ]
-                            .join(&b' '),
-                        ).await
-                                self.read_response(&mut response, RequiredResponses::SEARCH).await
-                            );
-                        debug!(to_str!(&response));
+                            mailbox_hash,
+                            self.send_command(format!("UID SEARCH {}", msg_seq).as_bytes())
+                                .await,
+                            self.read_response(&mut response, RequiredResponses::SEARCH)
+                                .await,
+                        );
                         match super::protocol_parser::search_results(
                             response.split_rn().next().unwrap_or(b""),
                         )
@@ -392,34 +471,30 @@ impl ImapConnection {
                                 return Ok(false);
                             }
                             Err(e) => {
+                                debug!("SEARCH error failed: {}", e);
                                 debug!(to_str!(&response));
-                                debug!(e);
                                 return Ok(false);
                             }
                         }
                     };
                     debug!("fetch uid {} {:?}", uid, flags);
-                    let env_hash = self
-                        .uid_store
-                        .uid_index
-                        .lock()
-                        .unwrap()
-                        .get(&(mailbox_hash, uid))
-                        .copied();
-                    if let Some(env_hash) = env_hash {
+                    if let Some(env_hash) = {
+                        let temp = self
+                            .uid_store
+                            .uid_index
+                            .lock()
+                            .unwrap()
+                            .get(&(mailbox_hash, uid))
+                            .copied();
+                        temp
+                    } {
                         if !flags.0.intersects(crate::email::Flag::SEEN) {
                             mailbox.unseen.lock().unwrap().insert_new(env_hash);
                         } else {
                             mailbox.unseen.lock().unwrap().remove(env_hash);
                         }
+                        mailbox.exists.lock().unwrap().insert_new(env_hash);
                         if let Some(modseq) = modseq {
-                            self.uid_store
-                                .reverse_modseq
-                                .lock()
-                                .unwrap()
-                                .entry(mailbox_hash)
-                                .or_default()
-                                .insert(modseq, env_hash);
                             self.uid_store
                                 .modseq
                                 .lock()

melib/src/backends/imap/watch.rs

@@ -76,10 +76,9 @@ pub async fn idle(kit: ImapWatchKit) -> Result<()> {
     let mailbox_hash = mailbox.hash();
     let mut response = Vec::with_capacity(8 * 1024);
     let select_response = conn
-        .select_mailbox(mailbox_hash, &mut response, true)
+        .examine_mailbox(mailbox_hash, &mut response, true)
         .await?
         .unwrap();
-    debug!("select response {}", String::from_utf8_lossy(&response));
     {
         let mut uidvalidities = uid_store.uidvalidity.lock().unwrap();
 
@@ -111,6 +110,12 @@ pub async fn idle(kit: ImapWatchKit) -> Result<()> {
         let mailboxes_lck = timeout(uid_store.timeout, uid_store.mailboxes.lock()).await?;
         mailboxes_lck.clone()
     };
+    for (h, mailbox) in mailboxes.clone() {
+        if mailbox_hash == h {
+            continue;
+        }
+        examine_updates(mailbox, &mut conn, &uid_store).await?;
+    }
     conn.send_command(b"IDLE").await?;
     let mut blockn = ImapBlockingConnection::from(conn);
     let mut watch = std::time::Instant::now();
@@ -145,10 +150,7 @@ pub async fn idle(kit: ImapWatchKit) -> Result<()> {
         if now.duration_since(watch) >= _5_MINS {
             /* Time to poll all inboxes */
             let mut conn = timeout(uid_store.timeout, main_conn.lock()).await?;
-            for (h, mailbox) in mailboxes.clone() {
-                if mailbox_hash == h {
-                    continue;
-                }
+            for (_h, mailbox) in mailboxes.clone() {
                 examine_updates(mailbox, &mut conn, &uid_store).await?;
             }
             watch = now;
@@ -173,7 +175,7 @@ pub async fn idle(kit: ImapWatchKit) -> Result<()> {
                 .conn
                 .read_response(&mut response, RequiredResponses::empty())
                 .await?;
-            for l in line.split_rn() {
+            for l in line.split_rn().chain(response.split_rn()) {
                 debug!("process_untagged {:?}", &l);
                 if l.starts_with(b"+ ")
                     || l.starts_with(b"* ok")
@@ -219,7 +221,6 @@ pub async fn examine_updates(
             .examine_mailbox(mailbox_hash, &mut response, true)
             .await?
             .unwrap();
-        debug!(&select_response);
         {
             let mut uidvalidities = uid_store.uidvalidity.lock().unwrap();
 
@@ -244,7 +245,74 @@ pub async fn examine_updates(
                 uidvalidities.insert(mailbox_hash, select_response.uidvalidity);
             }
         }
-        if debug!(select_response.recent > 0) {
+        if mailbox.is_cold() {
+            /* Mailbox hasn't been loaded yet */
+            let has_list_status: bool = conn
+                .uid_store
+                .capabilities
+                .lock()
+                .unwrap()
+                .iter()
+                .any(|cap| cap.eq_ignore_ascii_case(b"LIST-STATUS"));
+            if has_list_status {
+                conn.send_command(
+                    format!(
+                        "LIST \"{}\" \"\" RETURN (STATUS (MESSAGES UNSEEN))",
+                        mailbox.imap_path()
+                    )
+                    .as_bytes(),
+                )
+                .await?;
+                conn.read_response(
+                    &mut response,
+                    RequiredResponses::LIST_REQUIRED | RequiredResponses::STATUS,
+                )
+                .await?;
+                debug!(
+                    "list return status out: {}",
+                    String::from_utf8_lossy(&response)
+                );
+                for l in response.split_rn() {
+                    if !l.starts_with(b"*") {
+                        continue;
+                    }
+                    if let Ok(status) = protocol_parser::status_response(&l).map(|(_, v)| v) {
+                        if Some(mailbox_hash) == status.mailbox {
+                            if let Some(total) = status.messages {
+                                if let Ok(mut exists_lck) = mailbox.exists.lock() {
+                                    exists_lck.clear();
+                                    exists_lck.set_not_yet_seen(total);
+                                }
+                            }
+                            if let Some(total) = status.unseen {
+                                if let Ok(mut unseen_lck) = mailbox.unseen.lock() {
+                                    unseen_lck.clear();
+                                    unseen_lck.set_not_yet_seen(total);
+                                }
+                            }
+                            break;
+                        }
+                    }
+                }
+            } else {
+                conn.send_command(b"SEARCH UNSEEN").await?;
+                conn.read_response(&mut response, RequiredResponses::SEARCH)
+                    .await?;
+                let unseen_count = protocol_parser::search_results(&response)?.1.len();
+                if let Ok(mut exists_lck) = mailbox.exists.lock() {
+                    exists_lck.clear();
+                    exists_lck.set_not_yet_seen(select_response.exists);
+                }
+                if let Ok(mut unseen_lck) = mailbox.unseen.lock() {
+                    unseen_lck.clear();
+                    unseen_lck.set_not_yet_seen(unseen_count);
+                }
+            }
+            mailbox.set_warm(true);
+            return Ok(());
+        }
+
+        if select_response.recent > 0 {
             /* UID SEARCH RECENT */
             conn.send_command(b"UID SEARCH RECENT").await?;
             conn.read_response(&mut response, RequiredResponses::SEARCH)
@@ -267,15 +335,17 @@ pub async fn examine_updates(
                     cmd.push_str(&n.to_string());
                 }
             }
-            cmd.push_str(" (UID FLAGS RFC822)");
+            cmd.push_str(
+                " (UID FLAGS ENVELOPE BODY.PEEK[HEADER.FIELDS (REFERENCES)] BODYSTRUCTURE)",
+            );
             conn.send_command(cmd.as_bytes()).await?;
             conn.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
                 .await?;
-        } else if debug!(select_response.exists > mailbox.exists.lock().unwrap().len()) {
+        } else if select_response.exists > mailbox.exists.lock().unwrap().len() {
             conn.send_command(
                 format!(
-                    "FETCH {}:* (UID FLAGS RFC822)",
-                    mailbox.exists.lock().unwrap().len()
+                    "FETCH {}:* (UID FLAGS ENVELOPE BODY.PEEK[HEADER.FIELDS (REFERENCES)] BODYSTRUCTURE)",
+                    std::cmp::max(mailbox.exists.lock().unwrap().len(), 1)
                 )
                 .as_bytes(),
             )
@@ -285,42 +355,60 @@ pub async fn examine_updates(
         } else {
             return Ok(());
         }
-        debug!(&response);
+        debug!(
+            "fetch response is {} bytes and {} lines",
+            response.len(),
+            String::from_utf8_lossy(&response).lines().count()
+        );
         let (_, mut v, _) = protocol_parser::fetch_responses(&response)?;
+        debug!("responses len is {}", v.len());
         for FetchResponse {
             ref uid,
-            ref mut flags,
-            ref mut body,
             ref mut envelope,
+            ref mut flags,
+            ref references,
             ..
         } in v.iter_mut()
         {
             let uid = uid.unwrap();
-            *envelope = Envelope::from_bytes(body.take().unwrap(), flags.as_ref().map(|&(f, _)| f))
-                .map(|mut env| {
-                    env.set_hash(generate_envelope_hash(&mailbox.imap_path(), &uid));
-                    if let Some((_, keywords)) = flags.take() {
-                        let mut tag_lck = uid_store.tag_index.write().unwrap();
-                        for f in keywords {
-                            let hash = tag_hash!(f);
-                            if !tag_lck.contains_key(&hash) {
-                                tag_lck.insert(hash, f);
-                            }
-                            env.labels_mut().push(hash);
-                        }
+            let env = envelope.as_mut().unwrap();
+            env.set_hash(generate_envelope_hash(&mailbox.imap_path(), &uid));
+            if let Some(value) = references {
+                env.set_references(value);
+            }
+            let mut tag_lck = uid_store.collection.tag_index.write().unwrap();
+            if let Some((flags, keywords)) = flags {
+                env.set_flags(*flags);
+                if !env.is_seen() {
+                    mailbox.unseen.lock().unwrap().insert_new(env.hash());
+                }
+                mailbox.exists.lock().unwrap().insert_new(env.hash());
+                for f in keywords {
+                    let hash = tag_hash!(f);
+                    if !tag_lck.contains_key(&hash) {
+                        tag_lck.insert(hash, f.to_string());
                     }
-                    env
-                })
-                .map_err(|err| {
-                    debug!("uid {} envelope parse error {}", uid, &err);
-                    err
-                })
-                .ok();
+                    env.labels_mut().push(hash);
+                }
+            }
         }
         if uid_store.keep_offline_cache {
-            cache_handle.insert_envelopes(mailbox_hash, &v)?;
+            if !cache_handle.mailbox_state(mailbox_hash)?.is_none() {
+                cache_handle
+                    .insert_envelopes(mailbox_hash, &v)
+                    .chain_err_summary(|| {
+                        format!(
+                            "Could not save envelopes in cache for mailbox {}",
+                            mailbox.imap_path()
+                        )
+                    })?;
+            }
         }
-        'fetch_responses_c: for FetchResponse { uid, envelope, .. } in v {
+
+        for FetchResponse { uid, envelope, .. } in v {
+            if uid.is_none() || envelope.is_none() {
+                continue;
+            }
             let uid = uid.unwrap();
             if uid_store
                 .uid_index
@@ -328,35 +416,37 @@ pub async fn examine_updates(
                 .unwrap()
                 .contains_key(&(mailbox_hash, uid))
             {
-                continue 'fetch_responses_c;
-            }
-            if let Some(env) = envelope {
-                uid_store
-                    .hash_index
-                    .lock()
-                    .unwrap()
-                    .insert(env.hash(), (uid, mailbox_hash));
-                uid_store
-                    .uid_index
-                    .lock()
-                    .unwrap()
-                    .insert((mailbox_hash, uid), env.hash());
-                debug!(
-                    "Create event {} {} {}",
-                    env.hash(),
-                    env.subject(),
-                    mailbox.path(),
-                );
-                if !env.is_seen() {
-                    mailbox.unseen.lock().unwrap().insert_new(env.hash());
-                }
-                mailbox.exists.lock().unwrap().insert_new(env.hash());
-                conn.add_refresh_event(RefreshEvent {
-                    account_hash: uid_store.account_hash,
-                    mailbox_hash,
-                    kind: Create(Box::new(env)),
-                });
+                continue;
             }
+            let env = envelope.unwrap();
+            debug!(
+                "Create event {} {} {}",
+                env.hash(),
+                env.subject(),
+                mailbox.path(),
+            );
+            uid_store
+                .msn_index
+                .lock()
+                .unwrap()
+                .entry(mailbox_hash)
+                .or_default()
+                .push(uid);
+            uid_store
+                .hash_index
+                .lock()
+                .unwrap()
+                .insert(env.hash(), (uid, mailbox_hash));
+            uid_store
+                .uid_index
+                .lock()
+                .unwrap()
+                .insert((mailbox_hash, uid), env.hash());
+            conn.add_refresh_event(RefreshEvent {
+                account_hash: uid_store.account_hash,
+                mailbox_hash,
+                kind: Create(Box::new(env)),
+            });
         }
     }
     Ok(())

melib/src/backends/jmap.rs

@@ -23,16 +23,32 @@ use crate::backends::*;
 use crate::conf::AccountSettings;
 use crate::email::*;
 use crate::error::{MeliError, Result};
+use crate::Collection;
 use futures::lock::Mutex as FutureMutex;
+use isahc::config::RedirectPolicy;
 use isahc::prelude::HttpClient;
 use isahc::ResponseExt;
 use serde_json::Value;
-use std::collections::{BTreeMap, HashMap};
+use std::collections::{hash_map::DefaultHasher, HashMap, HashSet};
 use std::convert::TryFrom;
+use std::hash::{Hash, Hasher};
 use std::str::FromStr;
 use std::sync::{Arc, Mutex, RwLock};
 use std::time::Instant;
 
+macro_rules! tag_hash {
+    ($t:ident) => {{
+        let mut hasher = DefaultHasher::default();
+        $t.hash(&mut hasher);
+        hasher.finish()
+    }};
+    ($t:literal) => {{
+        let mut hasher = DefaultHasher::default();
+        $t.hash(&mut hasher);
+        hasher.finish()
+    }};
+}
+
 #[macro_export]
 macro_rules! _impl {
         ($(#[$outer:meta])*$field:ident : $t:ty) => {
@@ -131,20 +147,6 @@ impl JmapServerConf {
     }
 }
 
-struct IsSubscribedFn(Box<dyn Fn(&str) -> bool + Send + Sync>);
-
-impl std::fmt::Debug for IsSubscribedFn {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(f, "IsSubscribedFn Box")
-    }
-}
-
-impl std::ops::Deref for IsSubscribedFn {
-    type Target = Box<dyn Fn(&str) -> bool + Send + Sync>;
-    fn deref(&self) -> &Box<dyn Fn(&str) -> bool + Send + Sync> {
-        &self.0
-    }
-}
 macro_rules! get_conf_val {
     ($s:ident[$var:literal]) => {
         $s.extra.get($var).ok_or_else(|| {
@@ -173,24 +175,115 @@ macro_rules! get_conf_val {
     };
 }
 
-#[derive(Debug, Default)]
+#[derive(Debug)]
 pub struct Store {
-    byte_cache: HashMap<EnvelopeHash, EnvelopeCache>,
-    id_store: HashMap<EnvelopeHash, Id>,
-    blob_id_store: HashMap<EnvelopeHash, Id>,
+    pub account_name: Arc<String>,
+    pub account_hash: AccountHash,
+    pub account_id: Arc<Mutex<Id<Account>>>,
+    pub byte_cache: Arc<Mutex<HashMap<EnvelopeHash, EnvelopeCache>>>,
+    pub id_store: Arc<Mutex<HashMap<EnvelopeHash, Id<EmailObject>>>>,
+    pub reverse_id_store: Arc<Mutex<HashMap<Id<EmailObject>, EnvelopeHash>>>,
+    pub blob_id_store: Arc<Mutex<HashMap<EnvelopeHash, Id<BlobObject>>>>,
+    pub collection: Collection,
+    pub mailboxes: Arc<RwLock<HashMap<MailboxHash, JmapMailbox>>>,
+    pub mailboxes_index: Arc<RwLock<HashMap<MailboxHash, HashSet<EnvelopeHash>>>>,
+    pub mailbox_state: Arc<Mutex<State<MailboxObject>>>,
+    pub online_status: Arc<FutureMutex<(Instant, Result<()>)>>,
+    pub is_subscribed: Arc<IsSubscribedFn>,
+    pub event_consumer: BackendEventConsumer,
+}
+
+impl Store {
+    pub fn add_envelope(&self, obj: EmailObject) -> Envelope {
+        let mut tag_lck = self.collection.tag_index.write().unwrap();
+        let tags = obj
+            .keywords()
+            .keys()
+            .map(|tag| {
+                let tag_hash = {
+                    let mut hasher = DefaultHasher::default();
+                    tag.hash(&mut hasher);
+                    hasher.finish()
+                };
+                if !tag_lck.contains_key(&tag_hash) {
+                    tag_lck.insert(tag_hash, tag.to_string());
+                }
+                tag_hash
+            })
+            .collect::<SmallVec<[u64; 1024]>>();
+        let id = obj.id.clone();
+        let mailbox_ids = obj.mailbox_ids.clone();
+        let blob_id = obj.blob_id.clone();
+        drop(tag_lck);
+        let mut ret: Envelope = obj.into();
+
+        debug_assert_eq!(tag_hash!("$draft"), 6613915297903591176);
+        debug_assert_eq!(tag_hash!("$seen"), 1683863812294339685);
+        debug_assert_eq!(tag_hash!("$flagged"), 2714010747478170100);
+        debug_assert_eq!(tag_hash!("$answered"), 8940855303929342213);
+        debug_assert_eq!(tag_hash!("$junk"), 2656839745430720464);
+        debug_assert_eq!(tag_hash!("$notjunk"), 4091323799684325059);
+        let mut id_store_lck = self.id_store.lock().unwrap();
+        let mut reverse_id_store_lck = self.reverse_id_store.lock().unwrap();
+        let mut blob_id_store_lck = self.blob_id_store.lock().unwrap();
+        let mailboxes_lck = self.mailboxes.read().unwrap();
+        let mut mailboxes_index_lck = self.mailboxes_index.write().unwrap();
+        for (mailbox_id, _) in mailbox_ids {
+            if let Some((mailbox_hash, _)) = mailboxes_lck.iter().find(|(_, m)| m.id == mailbox_id)
+            {
+                mailboxes_index_lck
+                    .entry(*mailbox_hash)
+                    .or_default()
+                    .insert(ret.hash());
+            }
+        }
+        reverse_id_store_lck.insert(id.clone(), ret.hash());
+        id_store_lck.insert(ret.hash(), id);
+        blob_id_store_lck.insert(ret.hash(), blob_id);
+        for t in tags {
+            match t {
+                6613915297903591176 => {
+                    ret.set_flags(ret.flags() | Flag::DRAFT);
+                }
+                1683863812294339685 => {
+                    ret.set_flags(ret.flags() | Flag::SEEN);
+                }
+                2714010747478170100 => {
+                    ret.set_flags(ret.flags() | Flag::FLAGGED);
+                }
+                8940855303929342213 => {
+                    ret.set_flags(ret.flags() | Flag::REPLIED);
+                }
+                2656839745430720464 | 4091323799684325059 => { /* ignore */ }
+                _ => ret.labels_mut().push(t),
+            }
+        }
+        ret
+    }
+
+    pub fn remove_envelope(
+        &self,
+        obj_id: Id<EmailObject>,
+    ) -> Option<(EnvelopeHash, SmallVec<[MailboxHash; 8]>)> {
+        let env_hash = self.reverse_id_store.lock().unwrap().remove(&obj_id)?;
+        self.id_store.lock().unwrap().remove(&env_hash);
+        self.blob_id_store.lock().unwrap().remove(&env_hash);
+        self.byte_cache.lock().unwrap().remove(&env_hash);
+        let mut mailbox_hashes = SmallVec::new();
+        for (k, set) in self.mailboxes_index.write().unwrap().iter_mut() {
+            if set.remove(&env_hash) {
+                mailbox_hashes.push(*k);
+            }
+        }
+        Some((env_hash, mailbox_hashes))
+    }
 }
 
 #[derive(Debug)]
 pub struct JmapType {
-    account_name: String,
-    account_hash: AccountHash,
-    online: Arc<FutureMutex<(Instant, Result<()>)>>,
-    is_subscribed: Arc<IsSubscribedFn>,
     server_conf: JmapServerConf,
     connection: Arc<FutureMutex<JmapConnection>>,
-    store: Arc<RwLock<Store>>,
-    tag_index: Arc<RwLock<BTreeMap<u64, String>>>,
-    mailboxes: Arc<RwLock<HashMap<MailboxHash, JmapMailbox>>>,
+    store: Arc<Store>,
 }
 
 impl MailBackend for JmapType {
@@ -207,7 +300,7 @@ impl MailBackend for JmapType {
     }
 
     fn is_online(&self) -> ResultFuture<()> {
-        let online = self.online.clone();
+        let online = self.store.online_status.clone();
         Ok(Box::pin(async move {
             //match timeout(std::time::Duration::from_secs(3), connection.lock()).await {
             let online_lck = online.lock().await;
@@ -224,9 +317,7 @@ impl MailBackend for JmapType {
         &mut self,
         mailbox_hash: MailboxHash,
     ) -> Result<Pin<Box<dyn Stream<Item = Result<Vec<Envelope>>> + Send + 'static>>> {
-        let mailboxes = self.mailboxes.clone();
         let store = self.store.clone();
-        let tag_index = self.tag_index.clone();
         let connection = self.connection.clone();
         Ok(Box::pin(async_stream::try_stream! {
             let mut conn = connection.lock().await;
@@ -234,34 +325,64 @@ impl MailBackend for JmapType {
             let res = protocol::fetch(
                 &conn,
                 &store,
-                &tag_index,
-                &mailboxes,
                 mailbox_hash,
             ).await?;
             yield res;
         }))
     }
 
-    fn refresh(&mut self, _mailbox_hash: MailboxHash) -> ResultFuture<()> {
-        Err(MeliError::new("Unimplemented."))
-    }
-
-    fn watch(&self) -> ResultFuture<()> {
-        Err(MeliError::from("JMAP watch for updates is unimplemented"))
-    }
-
-    fn mailboxes(&self) -> ResultFuture<HashMap<MailboxHash, Mailbox>> {
-        let mailboxes = self.mailboxes.clone();
+    fn refresh(&mut self, mailbox_hash: MailboxHash) -> ResultFuture<()> {
         let connection = self.connection.clone();
         Ok(Box::pin(async move {
             let mut conn = connection.lock().await;
             conn.connect().await?;
-            if mailboxes.read().unwrap().is_empty() {
+            conn.email_changes(mailbox_hash).await?;
+            Ok(())
+        }))
+    }
+
+    fn watch(&self) -> ResultFuture<()> {
+        let connection = self.connection.clone();
+        let store = self.store.clone();
+        Ok(Box::pin(async move {
+            {
+                let mut conn = connection.lock().await;
+                conn.connect().await?;
+            }
+            loop {
+                {
+                    let mailbox_hashes = {
+                        store
+                            .mailboxes
+                            .read()
+                            .unwrap()
+                            .keys()
+                            .cloned()
+                            .collect::<SmallVec<[MailboxHash; 16]>>()
+                    };
+                    let conn = connection.lock().await;
+                    for mailbox_hash in mailbox_hashes {
+                        conn.email_changes(mailbox_hash).await?;
+                    }
+                }
+                crate::connections::sleep(std::time::Duration::from_secs(60)).await;
+            }
+        }))
+    }
+
+    fn mailboxes(&self) -> ResultFuture<HashMap<MailboxHash, Mailbox>> {
+        let store = self.store.clone();
+        let connection = self.connection.clone();
+        Ok(Box::pin(async move {
+            let mut conn = connection.lock().await;
+            conn.connect().await?;
+            if store.mailboxes.read().unwrap().is_empty() {
                 let new_mailboxes = debug!(protocol::get_mailboxes(&conn).await)?;
-                *mailboxes.write().unwrap() = new_mailboxes;
+                *store.mailboxes.write().unwrap() = new_mailboxes;
             }
 
-            let ret = mailboxes
+            let ret = store
+                .mailboxes
                 .read()
                 .unwrap()
                 .iter()
@@ -283,15 +404,84 @@ impl MailBackend for JmapType {
 
     fn save(
         &self,
-        _bytes: Vec<u8>,
-        _mailbox_hash: MailboxHash,
+        bytes: Vec<u8>,
+        mailbox_hash: MailboxHash,
         _flags: Option<Flag>,
     ) -> ResultFuture<()> {
-        Err(MeliError::new("Unimplemented."))
-    }
+        let store = self.store.clone();
+        let connection = self.connection.clone();
+        Ok(Box::pin(async move {
+            let mut conn = connection.lock().await;
+            conn.connect().await?;
+            /*
+             * 1. upload binary blob, get blobId
+             * 2. Email/import
+             */
+            let (api_url, upload_url) = {
+                let lck = conn.session.lock().unwrap();
+                (lck.api_url.clone(), lck.upload_url.clone())
+            };
+            let mut res = conn
+                .client
+                .post_async(
+                    &upload_request_format(upload_url.as_str(), &conn.mail_account_id()),
+                    bytes,
+                )
+                .await?;
 
-    fn tags(&self) -> Option<Arc<RwLock<BTreeMap<u64, String>>>> {
-        Some(self.tag_index.clone())
+            let mailbox_id: Id<MailboxObject> = {
+                let mailboxes_lck = store.mailboxes.read().unwrap();
+                if let Some(mailbox) = mailboxes_lck.get(&mailbox_hash) {
+                    mailbox.id.clone()
+                } else {
+                    return Err(MeliError::new(format!(
+                        "Mailbox with hash {} not found",
+                        mailbox_hash
+                    )));
+                }
+            };
+            let res_text = res.text_async().await?;
+
+            let upload_response: UploadResponse = serde_json::from_str(&res_text)?;
+            let mut req = Request::new(conn.request_no.clone());
+            let creation_id: Id<EmailObject> = "1".to_string().into();
+            let mut email_imports = HashMap::default();
+            let mut mailbox_ids = HashMap::default();
+            mailbox_ids.insert(mailbox_id, true);
+            email_imports.insert(
+                creation_id.clone(),
+                EmailImport::new()
+                    .blob_id(upload_response.blob_id)
+                    .mailbox_ids(mailbox_ids),
+            );
+
+            let import_call: ImportCall = ImportCall::new()
+                .account_id(conn.mail_account_id().clone())
+                .emails(email_imports);
+
+            req.add_call(&import_call);
+            let mut res = conn
+                .client
+                .post_async(api_url.as_str(), serde_json::to_string(&req)?)
+                .await?;
+            let res_text = res.text_async().await?;
+
+            let mut v: MethodResponse = serde_json::from_str(&res_text)?;
+            let m = ImportResponse::try_from(v.method_responses.remove(0)).or_else(|err| {
+                let ierr: Result<ImportError> =
+                    serde_json::from_str(&res_text).map_err(|err| err.into());
+                if let Ok(err) = ierr {
+                    Err(MeliError::new(format!("Could not save message: {:?}", err)))
+                } else {
+                    Err(err.into())
+                }
+            })?;
+
+            if let Some(err) = m.not_created.get(&creation_id) {
+                return Err(MeliError::new(format!("Could not save message: {:?}", err)));
+            }
+            Ok(())
+        }))
     }
 
     fn as_any(&self) -> &dyn Any {
@@ -302,14 +492,21 @@ impl MailBackend for JmapType {
         self
     }
 
+    fn collection(&self) -> Collection {
+        self.store.collection.clone()
+    }
+
     fn search(
         &self,
         q: crate::search::Query,
         mailbox_hash: Option<MailboxHash>,
     ) -> ResultFuture<SmallVec<[EnvelopeHash; 512]>> {
+        let store = self.store.clone();
         let connection = self.connection.clone();
         let filter = if let Some(mailbox_hash) = mailbox_hash {
-            let mailbox_id = self.mailboxes.read().unwrap()[&mailbox_hash].id.clone();
+            let mailbox_id = self.store.mailboxes.read().unwrap()[&mailbox_hash]
+                .id
+                .clone();
 
             let mut f = Filter::Condition(
                 EmailFilterCondition::new()
@@ -327,7 +524,7 @@ impl MailBackend for JmapType {
             conn.connect().await?;
             let email_call: EmailQuery = EmailQuery::new(
                 Query::new()
-                    .account_id(conn.mail_account_id().to_string())
+                    .account_id(conn.mail_account_id().clone())
                     .filter(Some(filter))
                     .position(0),
             )
@@ -335,26 +532,19 @@ impl MailBackend for JmapType {
 
             let mut req = Request::new(conn.request_no.clone());
             req.add_call(&email_call);
+            let api_url = conn.session.lock().unwrap().api_url.clone();
 
             let mut res = conn
                 .client
-                .post_async(&conn.session.api_url, serde_json::to_string(&req)?)
+                .post_async(api_url.as_str(), serde_json::to_string(&req)?)
                 .await?;
 
             let res_text = res.text_async().await?;
             let mut v: MethodResponse = serde_json::from_str(&res_text).unwrap();
-            *conn.online_status.lock().await = (std::time::Instant::now(), Ok(()));
+            *store.online_status.lock().await = (std::time::Instant::now(), Ok(()));
             let m = QueryResponse::<EmailObject>::try_from(v.method_responses.remove(0))?;
             let QueryResponse::<EmailObject> { ids, .. } = m;
-            let ret = ids
-                .into_iter()
-                .map(|id| {
-                    use std::hash::Hasher;
-                    let mut h = std::collections::hash_map::DefaultHasher::new();
-                    h.write(id.as_bytes());
-                    h.finish()
-                })
-                .collect();
+            let ret = ids.into_iter().map(|id| id.into_hash()).collect();
             Ok(ret)
         }))
     }
@@ -376,12 +566,92 @@ impl MailBackend for JmapType {
 
     fn copy_messages(
         &mut self,
-        _env_hashes: EnvelopeHashBatch,
-        _source_mailbox_hash: MailboxHash,
-        _destination_mailbox_hash: MailboxHash,
-        _move_: bool,
+        env_hashes: EnvelopeHashBatch,
+        source_mailbox_hash: MailboxHash,
+        destination_mailbox_hash: MailboxHash,
+        move_: bool,
     ) -> ResultFuture<()> {
-        Err(MeliError::new("Unimplemented."))
+        let store = self.store.clone();
+        let connection = self.connection.clone();
+        Ok(Box::pin(async move {
+            let (source_mailbox_id, destination_mailbox_id) = {
+                let mailboxes_lck = store.mailboxes.read().unwrap();
+                if !mailboxes_lck.contains_key(&source_mailbox_hash) {
+                    return Err(MeliError::new(format!(
+                        "Could not find source mailbox with hash {}",
+                        source_mailbox_hash
+                    )));
+                }
+                if !mailboxes_lck.contains_key(&destination_mailbox_hash) {
+                    return Err(MeliError::new(format!(
+                        "Could not find destination mailbox with hash {}",
+                        destination_mailbox_hash
+                    )));
+                }
+
+                (
+                    mailboxes_lck[&source_mailbox_hash].id.clone(),
+                    mailboxes_lck[&destination_mailbox_hash].id.clone(),
+                )
+            };
+            let mut update_map: HashMap<Id<EmailObject>, Value> = HashMap::default();
+            let mut ids: Vec<Id<EmailObject>> = Vec::with_capacity(env_hashes.rest.len() + 1);
+            let mut id_map: HashMap<Id<EmailObject>, EnvelopeHash> = HashMap::default();
+            let mut update_keywords: HashMap<String, Value> = HashMap::default();
+            update_keywords.insert(
+                format!("mailboxIds/{}", &destination_mailbox_id),
+                serde_json::json!(true),
+            );
+            if move_ {
+                update_keywords.insert(
+                    format!("mailboxIds/{}", &source_mailbox_id),
+                    serde_json::json!(null),
+                );
+            }
+            {
+                for env_hash in env_hashes.iter() {
+                    if let Some(id) = store.id_store.lock().unwrap().get(&env_hash) {
+                        ids.push(id.clone());
+                        id_map.insert(id.clone(), env_hash);
+                        update_map.insert(id.clone(), serde_json::json!(update_keywords.clone()));
+                    }
+                }
+            }
+            let conn = connection.lock().await;
+            let api_url = conn.session.lock().unwrap().api_url.clone();
+
+            let email_set_call: EmailSet = EmailSet::new(
+                Set::<EmailObject>::new()
+                    .account_id(conn.mail_account_id().clone())
+                    .update(Some(update_map)),
+            );
+
+            let mut req = Request::new(conn.request_no.clone());
+            let _prev_seq = req.add_call(&email_set_call);
+
+            let mut res = conn
+                .client
+                .post_async(api_url.as_str(), serde_json::to_string(&req)?)
+                .await?;
+
+            let res_text = res.text_async().await?;
+
+            let mut v: MethodResponse = serde_json::from_str(&res_text).unwrap();
+            *store.online_status.lock().await = (std::time::Instant::now(), Ok(()));
+            let m = SetResponse::<EmailObject>::try_from(v.method_responses.remove(0))?;
+            if let Some(ids) = m.not_updated {
+                if !ids.is_empty() {
+                    return Err(MeliError::new(format!(
+                        "Could not update ids: {}",
+                        ids.into_iter()
+                            .map(|err| err.to_string())
+                            .collect::<Vec<String>>()
+                            .join(",")
+                    )));
+                }
+            }
+            Ok(())
+        }))
     }
 
     fn set_flags(
@@ -390,16 +660,12 @@ impl MailBackend for JmapType {
         mailbox_hash: MailboxHash,
         flags: SmallVec<[(std::result::Result<Flag, String>, bool); 8]>,
     ) -> ResultFuture<()> {
-        let mailboxes = self.mailboxes.clone();
         let store = self.store.clone();
-        let account_hash = self.account_hash;
-        let tag_index = self.tag_index.clone();
         let connection = self.connection.clone();
         Ok(Box::pin(async move {
-            let mailbox_id = mailboxes.read().unwrap()[&mailbox_hash].id.clone();
-            let mut update_map: HashMap<String, Value> = HashMap::default();
-            let mut ids: Vec<Id> = Vec::with_capacity(env_hashes.rest.len() + 1);
-            let mut id_map: HashMap<Id, EnvelopeHash> = HashMap::default();
+            let mut update_map: HashMap<Id<EmailObject>, Value> = HashMap::default();
+            let mut ids: Vec<Id<EmailObject>> = Vec::with_capacity(env_hashes.rest.len() + 1);
+            let mut id_map: HashMap<Id<EmailObject>, EnvelopeHash> = HashMap::default();
             let mut update_keywords: HashMap<String, Value> = HashMap::default();
             for (flag, value) in flags.iter() {
                 match flag {
@@ -437,9 +703,8 @@ impl MailBackend for JmapType {
                 }
             }
             {
-                let store_lck = store.read().unwrap();
                 for hash in env_hashes.iter() {
-                    if let Some(id) = store_lck.id_store.get(&hash) {
+                    if let Some(id) = store.id_store.lock().unwrap().get(&hash) {
                         ids.push(id.clone());
                         id_map.insert(id.clone(), hash);
                         update_map.insert(id.clone(), serde_json::json!(update_keywords.clone()));
@@ -450,24 +715,25 @@ impl MailBackend for JmapType {
 
             let email_set_call: EmailSet = EmailSet::new(
                 Set::<EmailObject>::new()
-                    .account_id(conn.mail_account_id().to_string())
+                    .account_id(conn.mail_account_id().clone())
                     .update(Some(update_map)),
             );
 
             let mut req = Request::new(conn.request_no.clone());
-            let prev_seq = req.add_call(&email_set_call);
+            req.add_call(&email_set_call);
             let email_call: EmailGet = EmailGet::new(
                 Get::new()
                     .ids(Some(JmapArgument::Value(ids)))
-                    .account_id(conn.mail_account_id().to_string())
+                    .account_id(conn.mail_account_id().clone())
                     .properties(Some(vec!["keywords".to_string()])),
             );
 
             req.add_call(&email_call);
+            let api_url = conn.session.lock().unwrap().api_url.clone();
             //debug!(serde_json::to_string(&req)?);
             let mut res = conn
                 .client
-                .post_async(&conn.session.api_url, serde_json::to_string(&req)?)
+                .post_async(api_url.as_str(), serde_json::to_string(&req)?)
                 .await?;
 
             let res_text = res.text_async().await?;
@@ -476,7 +742,7 @@ impl MailBackend for JmapType {
              */
             //debug!("res_text = {}", &res_text);
             let mut v: MethodResponse = serde_json::from_str(&res_text).unwrap();
-            *conn.online_status.lock().await = (std::time::Instant::now(), Ok(()));
+            *store.online_status.lock().await = (std::time::Instant::now(), Ok(()));
             let m = SetResponse::<EmailObject>::try_from(v.method_responses.remove(0))?;
             if let Some(ids) = m.not_updated {
                 return Err(MeliError::new(
@@ -487,24 +753,51 @@ impl MailBackend for JmapType {
                 ));
             }
 
-            let mut tag_index_lck = tag_index.write().unwrap();
-            for (flag, value) in flags.iter() {
-                match flag {
-                    Ok(f) => {}
-                    Err(t) => {
-                        if *value {
-                            tag_index_lck.insert(tag_hash!(t), t.clone());
+            {
+                let mut tag_index_lck = store.collection.tag_index.write().unwrap();
+                for (flag, value) in flags.iter() {
+                    match flag {
+                        Ok(_) => {}
+                        Err(t) => {
+                            if *value {
+                                tag_index_lck.insert(tag_hash!(t), t.clone());
+                            }
                         }
                     }
                 }
+                drop(tag_index_lck);
             }
             let e = GetResponse::<EmailObject>::try_from(v.method_responses.pop().unwrap())?;
             let GetResponse::<EmailObject> { list, state, .. } = e;
-            //debug!(&list);
+            {
+                let (is_empty, is_equal) = {
+                    let mailboxes_lck = conn.store.mailboxes.read().unwrap();
+                    mailboxes_lck
+                        .get(&mailbox_hash)
+                        .map(|mbox| {
+                            let current_state_lck = mbox.email_state.lock().unwrap();
+                            (
+                                current_state_lck.is_some(),
+                                current_state_lck.as_ref() != Some(&state),
+                            )
+                        })
+                        .unwrap_or((true, true))
+                };
+                if is_empty {
+                    let mut mailboxes_lck = conn.store.mailboxes.write().unwrap();
+                    debug!("{:?}: inserting state {}", EmailObject::NAME, &state);
+                    mailboxes_lck.entry(mailbox_hash).and_modify(|mbox| {
+                        *mbox.email_state.lock().unwrap() = Some(state);
+                    });
+                } else if !is_equal {
+                    conn.email_changes(mailbox_hash).await?;
+                }
+            }
+            debug!(&list);
             for envobj in list {
                 let env_hash = id_map[&envobj.id];
                 conn.add_refresh_event(RefreshEvent {
-                    account_hash,
+                    account_hash: store.account_hash,
                     mailbox_hash,
                     kind: RefreshEventKind::NewFlags(
                         env_hash,
@@ -515,6 +808,14 @@ impl MailBackend for JmapType {
             Ok(())
         }))
     }
+
+    fn delete_messages(
+        &mut self,
+        _env_hashes: EnvelopeHashBatch,
+        _mailbox_hash: MailboxHash,
+    ) -> ResultFuture<()> {
+        Err(MeliError::new("Unimplemented."))
+    }
 }
 
 impl JmapType {
@@ -523,34 +824,41 @@ impl JmapType {
         is_subscribed: Box<dyn Fn(&str) -> bool + Send + Sync>,
         event_consumer: BackendEventConsumer,
     ) -> Result<Box<dyn MailBackend>> {
-        let online = Arc::new(FutureMutex::new((
+        let online_status = Arc::new(FutureMutex::new((
             std::time::Instant::now(),
             Err(MeliError::new("Account is uninitialised.")),
         )));
         let server_conf = JmapServerConf::new(s)?;
 
         let account_hash = {
-            use std::collections::hash_map::DefaultHasher;
-            use std::hash::Hasher;
             let mut hasher = DefaultHasher::new();
             hasher.write(s.name.as_bytes());
             hasher.finish()
         };
+        let store = Arc::new(Store {
+            account_name: Arc::new(s.name.clone()),
+            account_hash,
+            account_id: Arc::new(Mutex::new(Id::new())),
+            online_status,
+            event_consumer,
+            is_subscribed: Arc::new(IsSubscribedFn(is_subscribed)),
+            collection: Collection::default(),
+
+            byte_cache: Default::default(),
+            id_store: Default::default(),
+            reverse_id_store: Default::default(),
+            blob_id_store: Default::default(),
+            mailboxes: Default::default(),
+            mailboxes_index: Default::default(),
+            mailbox_state: Default::default(),
+        });
 
         Ok(Box::new(JmapType {
             connection: Arc::new(FutureMutex::new(JmapConnection::new(
                 &server_conf,
-                account_hash,
-                event_consumer,
-                online.clone(),
+                store.clone(),
             )?)),
-            store: Arc::new(RwLock::new(Store::default())),
-            tag_index: Arc::new(RwLock::new(Default::default())),
-            mailboxes: Arc::new(RwLock::new(HashMap::default())),
-            account_name: s.name.clone(),
-            account_hash,
-            online,
-            is_subscribed: Arc::new(IsSubscribedFn(is_subscribed)),
+            store,
             server_conf,
         }))
     }

melib/src/backends/jmap/connection.rs

@@ -24,26 +24,18 @@ use isahc::config::Configurable;
 
 #[derive(Debug)]
 pub struct JmapConnection {
-    pub session: JmapSession,
+    pub session: Arc<Mutex<JmapSession>>,
     pub request_no: Arc<Mutex<usize>>,
     pub client: Arc<HttpClient>,
-    pub online_status: Arc<FutureMutex<(Instant, Result<()>)>>,
     pub server_conf: JmapServerConf,
-    pub account_id: Arc<Mutex<String>>,
-    pub account_hash: AccountHash,
-    pub method_call_states: Arc<Mutex<HashMap<&'static str, String>>>,
-    pub event_consumer: BackendEventConsumer,
+    pub store: Arc<Store>,
 }
 
 impl JmapConnection {
-    pub fn new(
-        server_conf: &JmapServerConf,
-        account_hash: AccountHash,
-        event_consumer: BackendEventConsumer,
-        online_status: Arc<FutureMutex<(Instant, Result<()>)>>,
-    ) -> Result<Self> {
+    pub fn new(server_conf: &JmapServerConf, store: Arc<Store>) -> Result<Self> {
         let client = HttpClient::builder()
             .timeout(std::time::Duration::from_secs(10))
+            .redirect_policy(RedirectPolicy::Limit(10))
             .authentication(isahc::auth::Authentication::basic())
             .credentials(isahc::auth::Credentials::new(
                 &server_conf.server_username,
@@ -52,20 +44,16 @@ impl JmapConnection {
             .build()?;
         let server_conf = server_conf.clone();
         Ok(JmapConnection {
-            session: Default::default(),
+            session: Arc::new(Mutex::new(Default::default())),
             request_no: Arc::new(Mutex::new(0)),
             client: Arc::new(client),
-            online_status,
             server_conf,
-            account_id: Arc::new(Mutex::new(String::new())),
-            account_hash,
-            event_consumer,
-            method_call_states: Arc::new(Mutex::new(Default::default())),
+            store,
         })
     }
 
     pub async fn connect(&mut self) -> Result<()> {
-        if self.online_status.lock().await.1.is_ok() {
+        if self.store.online_status.lock().await.1.is_ok() {
             return Ok(());
         }
         let mut jmap_session_resource_url =
@@ -86,7 +74,7 @@ impl JmapConnection {
         let session: JmapSession = match serde_json::from_str(&res_text) {
             Err(err) => {
                 let err = MeliError::new(format!("Could not connect to JMAP server endpoint for {}. Is your server hostname setting correct? (i.e. \"jmap.mailserver.org\") (Note: only session resource discovery via /.well-known/jmap is supported. DNS SRV records are not suppported.)\nReply from server: {}", &self.server_conf.server_hostname, &res_text)).set_source(Some(Arc::new(err)));
-                *self.online_status.lock().await = (Instant::now(), Err(err.clone()));
+                *self.store.online_status.lock().await = (Instant::now(), Err(err.clone()));
                 return Err(err);
             }
             Ok(s) => s,
@@ -96,7 +84,7 @@ impl JmapConnection {
             .contains_key("urn:ietf:params:jmap:core")
         {
             let err = MeliError::new(format!("Server {} did not return JMAP Core capability (urn:ietf:params:jmap:core). Returned capabilities were: {}", &self.server_conf.server_hostname, session.capabilities.keys().map(String::as_str).collect::<Vec<&str>>().join(", ")));
-            *self.online_status.lock().await = (Instant::now(), Err(err.clone()));
+            *self.store.online_status.lock().await = (Instant::now(), Err(err.clone()));
             return Err(err);
         }
         if !session
@@ -104,20 +92,251 @@ impl JmapConnection {
             .contains_key("urn:ietf:params:jmap:mail")
         {
             let err = MeliError::new(format!("Server {} does not support JMAP Mail capability (urn:ietf:params:jmap:mail). Returned capabilities were: {}", &self.server_conf.server_hostname, session.capabilities.keys().map(String::as_str).collect::<Vec<&str>>().join(", ")));
-            *self.online_status.lock().await = (Instant::now(), Err(err.clone()));
+            *self.store.online_status.lock().await = (Instant::now(), Err(err.clone()));
             return Err(err);
         }
 
-        *self.online_status.lock().await = (Instant::now(), Ok(()));
-        self.session = session;
+        *self.store.online_status.lock().await = (Instant::now(), Ok(()));
+        *self.session.lock().unwrap() = session;
         Ok(())
     }
 
-    pub fn mail_account_id(&self) -> &Id {
-        &self.session.primary_accounts["urn:ietf:params:jmap:mail"]
+    pub fn mail_account_id(&self) -> Id<Account> {
+        self.session.lock().unwrap().primary_accounts["urn:ietf:params:jmap:mail"].clone()
     }
 
     pub fn add_refresh_event(&self, event: RefreshEvent) {
-        (self.event_consumer)(self.account_hash, BackendEvent::Refresh(event));
+        (self.store.event_consumer)(self.store.account_hash, BackendEvent::Refresh(event));
+    }
+
+    pub async fn email_changes(&self, mailbox_hash: MailboxHash) -> Result<()> {
+        let mut current_state: State<EmailObject> = if let Some(s) = self
+            .store
+            .mailboxes
+            .read()
+            .unwrap()
+            .get(&mailbox_hash)
+            .and_then(|mbox| mbox.email_state.lock().unwrap().clone())
+        {
+            s
+        } else {
+            return Ok(());
+        };
+        loop {
+            let email_changes_call: EmailChanges = EmailChanges::new(
+                Changes::<EmailObject>::new()
+                    .account_id(self.mail_account_id().clone())
+                    .since_state(current_state.clone()),
+            );
+
+            let mut req = Request::new(self.request_no.clone());
+            let prev_seq = req.add_call(&email_changes_call);
+            let email_get_call: EmailGet = EmailGet::new(
+                Get::new()
+                    .ids(Some(JmapArgument::reference(
+                        prev_seq,
+                        ResultField::<EmailChanges, EmailObject>::new("created"),
+                    )))
+                    .account_id(self.mail_account_id().clone()),
+            );
+
+            req.add_call(&email_get_call);
+            if let Some(mailbox) = self.store.mailboxes.read().unwrap().get(&mailbox_hash) {
+                if let Some(email_query_state) = mailbox.email_query_state.lock().unwrap().clone() {
+                    let email_query_changes_call = EmailQueryChanges::new(
+                        QueryChanges::new(self.mail_account_id().clone(), email_query_state)
+                            .filter(Some(Filter::Condition(
+                                EmailFilterCondition::new()
+                                    .in_mailbox(Some(mailbox.id.clone()))
+                                    .into(),
+                            ))),
+                    );
+                    let seq_no = req.add_call(&email_query_changes_call);
+                    let email_get_call: EmailGet = EmailGet::new(
+                        Get::new()
+                            .ids(Some(JmapArgument::reference(
+                                seq_no,
+                                ResultField::<EmailQueryChanges, EmailObject>::new("removed"),
+                            )))
+                            .account_id(self.mail_account_id().clone())
+                            .properties(Some(vec![
+                                "keywords".to_string(),
+                                "mailboxIds".to_string(),
+                            ])),
+                    );
+                    req.add_call(&email_get_call);
+                } else {
+                    return Ok(());
+                }
+            } else {
+                return Ok(());
+            }
+            let api_url = self.session.lock().unwrap().api_url.clone();
+            let mut res = self
+                .client
+                .post_async(api_url.as_str(), serde_json::to_string(&req)?)
+                .await?;
+
+            let res_text = res.text_async().await?;
+            debug!(&res_text);
+            let mut v: MethodResponse = serde_json::from_str(&res_text).unwrap();
+            let changes_response =
+                ChangesResponse::<EmailObject>::try_from(v.method_responses.remove(0))?;
+            if changes_response.new_state == current_state {
+                return Ok(());
+            }
+            let get_response = GetResponse::<EmailObject>::try_from(v.method_responses.remove(0))?;
+
+            {
+                /* process get response */
+                let GetResponse::<EmailObject> { list, .. } = get_response;
+
+                let mut mailbox_hashes: Vec<SmallVec<[MailboxHash; 8]>> =
+                    Vec::with_capacity(list.len());
+                for envobj in &list {
+                    let v = self
+                        .store
+                        .mailboxes
+                        .read()
+                        .unwrap()
+                        .iter()
+                        .filter(|(_, m)| envobj.mailbox_ids.contains_key(&m.id))
+                        .map(|(k, _)| *k)
+                        .collect::<SmallVec<[MailboxHash; 8]>>();
+                    mailbox_hashes.push(v);
+                }
+                for (env, mailbox_hashes) in list
+                    .into_iter()
+                    .map(|obj| self.store.add_envelope(obj))
+                    .zip(mailbox_hashes)
+                {
+                    for mailbox_hash in mailbox_hashes.iter().skip(1).cloned() {
+                        let mut mailboxes_lck = self.store.mailboxes.write().unwrap();
+                        mailboxes_lck.entry(mailbox_hash).and_modify(|mbox| {
+                            if !env.is_seen() {
+                                mbox.unread_emails.lock().unwrap().insert_new(env.hash());
+                            }
+                            mbox.total_emails.lock().unwrap().insert_new(env.hash());
+                        });
+                        self.add_refresh_event(RefreshEvent {
+                            account_hash: self.store.account_hash,
+                            mailbox_hash,
+                            kind: RefreshEventKind::Create(Box::new(env.clone())),
+                        });
+                    }
+                    if let Some(mailbox_hash) = mailbox_hashes.first().cloned() {
+                        let mut mailboxes_lck = self.store.mailboxes.write().unwrap();
+                        mailboxes_lck.entry(mailbox_hash).and_modify(|mbox| {
+                            if !env.is_seen() {
+                                mbox.unread_emails.lock().unwrap().insert_new(env.hash());
+                            }
+                            mbox.total_emails.lock().unwrap().insert_new(env.hash());
+                        });
+                        self.add_refresh_event(RefreshEvent {
+                            account_hash: self.store.account_hash,
+                            mailbox_hash,
+                            kind: RefreshEventKind::Create(Box::new(env)),
+                        });
+                    }
+                }
+            }
+            let reverse_id_store_lck = self.store.reverse_id_store.lock().unwrap();
+            let response = v.method_responses.remove(0);
+            match EmailQueryChangesResponse::try_from(response) {
+                Ok(EmailQueryChangesResponse {
+                    collapse_threads: _,
+                    query_changes_response:
+                        QueryChangesResponse {
+                            account_id: _,
+                            old_query_state,
+                            new_query_state,
+                            total: _,
+                            removed,
+                            added,
+                        },
+                }) if old_query_state != new_query_state => {
+                    self.store
+                        .mailboxes
+                        .write()
+                        .unwrap()
+                        .entry(mailbox_hash)
+                        .and_modify(|mbox| {
+                            *mbox.email_query_state.lock().unwrap() = Some(new_query_state);
+                        });
+                    /*  If the "filter" or "sort" includes a mutable property, the server
+                    MUST include all Foos in the current results for which this
+                    property may have changed.  The position of these may have moved
+                    in the results, so they must be reinserted by the client to ensure
+                    its query cache is correct.  */
+                    for email_obj_id in removed
+                        .into_iter()
+                        .filter(|id| !added.iter().any(|item| item.id == *id))
+                    {
+                        if let Some(env_hash) = reverse_id_store_lck.get(&email_obj_id) {
+                            let mut mailboxes_lck = self.store.mailboxes.write().unwrap();
+                            mailboxes_lck.entry(mailbox_hash).and_modify(|mbox| {
+                                mbox.unread_emails.lock().unwrap().remove(*env_hash);
+                                mbox.total_emails.lock().unwrap().insert_new(*env_hash);
+                            });
+                            self.add_refresh_event(RefreshEvent {
+                                account_hash: self.store.account_hash,
+                                mailbox_hash,
+                                kind: RefreshEventKind::Remove(*env_hash),
+                            });
+                        }
+                    }
+                    for AddedItem {
+                        id: _email_obj_id,
+                        index: _,
+                    } in added
+                    {
+                        // FIXME
+                    }
+                }
+                Ok(_) => {}
+                Err(err) => {
+                    debug!(mailbox_hash);
+                    debug!(err);
+                }
+            }
+            let GetResponse::<EmailObject> { list, .. } =
+                GetResponse::<EmailObject>::try_from(v.method_responses.remove(0))?;
+            let mut mailboxes_lck = self.store.mailboxes.write().unwrap();
+            for envobj in list {
+                if let Some(env_hash) = reverse_id_store_lck.get(&envobj.id) {
+                    let new_flags =
+                        protocol::keywords_to_flags(envobj.keywords().keys().cloned().collect());
+                    mailboxes_lck.entry(mailbox_hash).and_modify(|mbox| {
+                        if new_flags.0.contains(Flag::SEEN) {
+                            mbox.unread_emails.lock().unwrap().remove(*env_hash);
+                        } else {
+                            mbox.unread_emails.lock().unwrap().insert_new(*env_hash);
+                        }
+                    });
+                    self.add_refresh_event(RefreshEvent {
+                        account_hash: self.store.account_hash,
+                        mailbox_hash,
+                        kind: RefreshEventKind::NewFlags(*env_hash, new_flags),
+                    });
+                }
+            }
+            drop(mailboxes_lck);
+            if changes_response.has_more_changes {
+                current_state = changes_response.new_state;
+            } else {
+                self.store
+                    .mailboxes
+                    .write()
+                    .unwrap()
+                    .entry(mailbox_hash)
+                    .and_modify(|mbox| {
+                        *mbox.email_state.lock().unwrap() = Some(changes_response.new_state);
+                    });
+
+                break;
+            }
+        }
+
+        Ok(())
     }
 }

melib/src/backends/jmap/mailbox.rs

@@ -20,7 +20,7 @@
  */
 
 use super::*;
-use crate::backends::{MailboxPermissions, SpecialUsageMailbox};
+use crate::backends::{LazyCountSet, MailboxPermissions, SpecialUsageMailbox};
 use std::sync::{Arc, Mutex, RwLock};
 
 #[derive(Debug, Clone)]
@@ -28,18 +28,21 @@ pub struct JmapMailbox {
     pub name: String,
     pub path: String,
     pub hash: MailboxHash,
-    pub v: Vec<MailboxHash>,
-    pub id: String,
+    pub children: Vec<MailboxHash>,
+    pub id: Id<MailboxObject>,
     pub is_subscribed: bool,
     pub my_rights: JmapRights,
-    pub parent_id: Option<String>,
+    pub parent_id: Option<Id<MailboxObject>>,
+    pub parent_hash: Option<MailboxHash>,
     pub role: Option<String>,
     pub sort_order: u64,
-    pub total_emails: Arc<Mutex<u64>>,
+    pub total_emails: Arc<Mutex<LazyCountSet>>,
     pub total_threads: u64,
-    pub unread_emails: Arc<Mutex<u64>>,
+    pub unread_emails: Arc<Mutex<LazyCountSet>>,
     pub unread_threads: u64,
     pub usage: Arc<RwLock<SpecialUsageMailbox>>,
+    pub email_state: Arc<Mutex<Option<State<EmailObject>>>>,
+    pub email_query_state: Arc<Mutex<Option<String>>>,
 }
 
 impl BackendMailbox for JmapMailbox {
@@ -62,11 +65,11 @@ impl BackendMailbox for JmapMailbox {
     }
 
     fn children(&self) -> &[MailboxHash] {
-        &self.v
+        &self.children
     }
 
     fn parent(&self) -> Option<MailboxHash> {
-        None
+        self.parent_hash
     }
 
     fn permissions(&self) -> MailboxPermissions {
@@ -108,8 +111,8 @@ impl BackendMailbox for JmapMailbox {
 
     fn count(&self) -> Result<(usize, usize)> {
         Ok((
-            *self.unread_emails.lock()? as usize,
-            *self.total_emails.lock()? as usize,
+            self.unread_emails.lock()?.len(),
+            self.total_emails.lock()?.len(),
         ))
     }
 }

melib/src/backends/jmap/objects/email.rs

@@ -24,11 +24,30 @@ use crate::backends::jmap::rfc8620::bool_false;
 use crate::email::address::{Address, MailboxAddress};
 use core::marker::PhantomData;
 use serde::de::{Deserialize, Deserializer};
+use serde_json::value::RawValue;
 use serde_json::Value;
 use std::collections::hash_map::DefaultHasher;
 use std::collections::HashMap;
 use std::hash::Hasher;
 
+mod import;
+pub use import::*;
+
+#[derive(Debug)]
+pub struct ThreadObject;
+
+impl Object for ThreadObject {
+    const NAME: &'static str = "Thread";
+}
+
+impl Id<EmailObject> {
+    pub fn into_hash(&self) -> EnvelopeHash {
+        let mut h = DefaultHasher::new();
+        h.write(self.inner.as_bytes());
+        h.finish()
+    }
+}
+
 // 4.1.1.
 // Metadata
 // These properties represent metadata about the message in the mail
@@ -130,58 +149,58 @@ use std::hash::Hasher;
 #[serde(rename_all = "camelCase")]
 pub struct EmailObject {
     #[serde(default)]
-    pub id: Id,
+    pub id: Id<EmailObject>,
     #[serde(default)]
-    pub blob_id: String,
+    pub blob_id: Id<BlobObject>,
     #[serde(default)]
-    mailbox_ids: HashMap<Id, bool>,
+    pub mailbox_ids: HashMap<Id<MailboxObject>, bool>,
     #[serde(default)]
-    size: u64,
+    pub size: u64,
     #[serde(default)]
-    received_at: String,
+    pub received_at: String,
     #[serde(default)]
-    message_id: Vec<String>,
+    pub message_id: Vec<String>,
     #[serde(default)]
-    to: SmallVec<[EmailAddress; 1]>,
+    pub to: Option<SmallVec<[EmailAddress; 1]>>,
     #[serde(default)]
-    bcc: Option<Vec<EmailAddress>>,
+    pub bcc: Option<Vec<EmailAddress>>,
     #[serde(default)]
-    reply_to: Option<Vec<EmailAddress>>,
+    pub reply_to: Option<Vec<EmailAddress>>,
     #[serde(default)]
-    cc: Option<SmallVec<[EmailAddress; 1]>>,
+    pub cc: Option<SmallVec<[EmailAddress; 1]>>,
     #[serde(default)]
-    sender: Option<Vec<EmailAddress>>,
+    pub sender: Option<Vec<EmailAddress>>,
     #[serde(default)]
-    from: SmallVec<[EmailAddress; 1]>,
+    pub from: Option<SmallVec<[EmailAddress; 1]>>,
     #[serde(default)]
-    in_reply_to: Option<Vec<String>>,
+    pub in_reply_to: Option<Vec<String>>,
     #[serde(default)]
-    references: Option<Vec<String>>,
+    pub references: Option<Vec<String>>,
     #[serde(default)]
-    keywords: HashMap<String, bool>,
+    pub keywords: HashMap<String, bool>,
     #[serde(default)]
-    attached_emails: Option<Id>,
+    pub attached_emails: Option<Id<BlobObject>>,
     #[serde(default)]
-    attachments: Vec<Value>,
+    pub attachments: Vec<Value>,
     #[serde(default)]
-    has_attachment: bool,
+    pub has_attachment: bool,
     #[serde(default)]
     #[serde(deserialize_with = "deserialize_header")]
-    headers: HashMap<String, String>,
+    pub headers: HashMap<String, String>,
     #[serde(default)]
-    html_body: Vec<HtmlBody>,
+    pub html_body: Vec<HtmlBody>,
     #[serde(default)]
-    preview: Option<String>,
+    pub preview: Option<String>,
     #[serde(default)]
-    sent_at: Option<String>,
+    pub sent_at: Option<String>,
     #[serde(default)]
-    subject: Option<String>,
+    pub subject: Option<String>,
     #[serde(default)]
-    text_body: Vec<TextBody>,
+    pub text_body: Vec<TextBody>,
     #[serde(default)]
-    thread_id: Id,
+    pub thread_id: Id<ThreadObject>,
     #[serde(flatten)]
-    extra: HashMap<String, Value>,
+    pub extra: HashMap<String, Value>,
 }
 
 impl EmailObject {
@@ -190,9 +209,9 @@ impl EmailObject {
 
 #[derive(Deserialize, Serialize, Debug, Default)]
 #[serde(rename_all = "camelCase")]
-struct Header {
-    name: String,
-    value: String,
+pub struct Header {
+    pub name: String,
+    pub value: String,
 }
 
 fn deserialize_header<'de, D>(
@@ -207,9 +226,9 @@ where
 
 #[derive(Deserialize, Serialize, Debug, Default)]
 #[serde(rename_all = "camelCase")]
-struct EmailAddress {
-    email: String,
-    name: Option<String>,
+pub struct EmailAddress {
+    pub email: String,
+    pub name: Option<String>,
 }
 
 impl Into<crate::email::Address> for EmailAddress {
@@ -247,15 +266,11 @@ impl std::convert::From<EmailObject> for crate::Envelope {
         }
         if let Some(ref in_reply_to) = t.in_reply_to {
             env.set_in_reply_to(in_reply_to[0].as_bytes());
-            env.push_references(env.in_reply_to().unwrap().clone());
+            if let Some(in_reply_to) = env.in_reply_to().cloned() {
+                env.push_references(in_reply_to);
+            }
         }
         if let Some(v) = t.headers.get("References") {
-            let parse_result = crate::email::parser::address::msg_id_list(v.as_bytes());
-            if let Ok((_, v)) = parse_result {
-                for v in v {
-                    env.push_references(v);
-                }
-            }
             env.set_references(v.as_bytes());
         }
         if let Some(v) = t.headers.get("Date") {
@@ -263,24 +278,30 @@ impl std::convert::From<EmailObject> for crate::Envelope {
             if let Ok(d) = crate::email::parser::dates::rfc5322_date(v.as_bytes()) {
                 env.set_datetime(d);
             }
+        } else if let Ok(d) = crate::email::parser::dates::rfc5322_date(t.received_at.as_bytes()) {
+            env.set_datetime(d);
         }
         env.set_has_attachments(t.has_attachment);
         if let Some(ref mut subject) = t.subject {
             env.set_subject(std::mem::replace(subject, String::new()).into_bytes());
         }
 
-        env.set_from(
-            std::mem::replace(&mut t.from, SmallVec::new())
-                .into_iter()
-                .map(|addr| addr.into())
-                .collect::<SmallVec<[crate::email::Address; 1]>>(),
-        );
-        env.set_to(
-            std::mem::replace(&mut t.to, SmallVec::new())
-                .into_iter()
-                .map(|addr| addr.into())
-                .collect::<SmallVec<[crate::email::Address; 1]>>(),
-        );
+        if let Some(ref mut from) = t.from {
+            env.set_from(
+                std::mem::replace(from, SmallVec::new())
+                    .into_iter()
+                    .map(|addr| addr.into())
+                    .collect::<SmallVec<[crate::email::Address; 1]>>(),
+            );
+        }
+        if let Some(ref mut to) = t.to {
+            env.set_to(
+                std::mem::replace(to, SmallVec::new())
+                    .into_iter()
+                    .map(|addr| addr.into())
+                    .collect::<SmallVec<[crate::email::Address; 1]>>(),
+            );
+        }
 
         if let Some(ref mut cc) = t.cc {
             env.set_cc(
@@ -300,99 +321,75 @@ impl std::convert::From<EmailObject> for crate::Envelope {
             );
         }
 
-        if env.references.is_some() {
-            if let Some(pos) = env
-                .references
-                .as_ref()
-                .map(|r| &r.refs)
-                .unwrap()
-                .iter()
-                .position(|r| r == env.message_id())
-            {
+        if let Some(ref r) = env.references {
+            if let Some(pos) = r.refs.iter().position(|r| r == env.message_id()) {
                 env.references.as_mut().unwrap().refs.remove(pos);
             }
         }
 
-        let mut h = DefaultHasher::new();
-        h.write(t.id.as_bytes());
-        env.set_hash(h.finish());
+        env.set_hash(t.id.into_hash());
         env
     }
 }
 
 #[derive(Deserialize, Serialize, Debug)]
 #[serde(rename_all = "camelCase")]
-struct HtmlBody {
-    blob_id: Id,
+pub struct HtmlBody {
+    pub blob_id: Id<BlobObject>,
     #[serde(default)]
-    charset: String,
+    pub charset: String,
     #[serde(default)]
-    cid: Option<String>,
+    pub cid: Option<String>,
     #[serde(default)]
-    disposition: Option<String>,
+    pub disposition: Option<String>,
     #[serde(default)]
-    headers: Value,
+    pub headers: Value,
     #[serde(default)]
-    language: Option<Vec<String>>,
+    pub language: Option<Vec<String>>,
     #[serde(default)]
-    location: Option<String>,
+    pub location: Option<String>,
     #[serde(default)]
-    name: Option<String>,
+    pub name: Option<String>,
     #[serde(default)]
-    part_id: Option<String>,
-    size: u64,
+    pub part_id: Option<String>,
+    pub size: u64,
     #[serde(alias = "type")]
-    content_type: String,
+    pub content_type: String,
     #[serde(default)]
-    sub_parts: Vec<Value>,
+    pub sub_parts: Vec<Value>,
 }
 
 #[derive(Deserialize, Serialize, Debug)]
 #[serde(rename_all = "camelCase")]
-struct TextBody {
-    blob_id: Id,
+pub struct TextBody {
+    pub blob_id: Id<BlobObject>,
     #[serde(default)]
-    charset: String,
+    pub charset: String,
     #[serde(default)]
-    cid: Option<String>,
+    pub cid: Option<String>,
     #[serde(default)]
-    disposition: Option<String>,
+    pub disposition: Option<String>,
     #[serde(default)]
-    headers: Value,
+    pub headers: Value,
     #[serde(default)]
-    language: Option<Vec<String>>,
+    pub language: Option<Vec<String>>,
     #[serde(default)]
-    location: Option<String>,
+    pub location: Option<String>,
     #[serde(default)]
-    name: Option<String>,
+    pub name: Option<String>,
     #[serde(default)]
-    part_id: Option<String>,
-    size: u64,
+    pub part_id: Option<String>,
+    pub size: u64,
     #[serde(alias = "type")]
-    content_type: String,
+    pub content_type: String,
     #[serde(default)]
-    sub_parts: Vec<Value>,
+    pub sub_parts: Vec<Value>,
 }
 
 impl Object for EmailObject {
     const NAME: &'static str = "Email";
 }
 
-#[derive(Deserialize, Serialize, Debug)]
-#[serde(rename_all = "camelCase")]
-pub struct EmailQueryResponse {
-    pub account_id: Id,
-    pub can_calculate_changes: bool,
-    pub collapse_threads: bool,
-    // FIXME
-    pub filter: String,
-    pub ids: Vec<Id>,
-    pub position: u64,
-    pub query_state: String,
-    pub sort: Option<String>,
-    pub total: usize,
-}
-
 #[derive(Serialize, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct EmailQuery {
@@ -469,9 +466,9 @@ impl EmailGet {
 #[serde(rename_all = "camelCase")]
 pub struct EmailFilterCondition {
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub in_mailbox: Option<Id>,
+    pub in_mailbox: Option<Id<MailboxObject>>,
     #[serde(skip_serializing_if = "Vec::is_empty")]
-    pub in_mailbox_other_than: Vec<Id>,
+    pub in_mailbox_other_than: Vec<Id<MailboxObject>>,
     #[serde(skip_serializing_if = "String::is_empty")]
     pub before: UtcDate,
     #[serde(skip_serializing_if = "String::is_empty")]
@@ -517,8 +514,8 @@ impl EmailFilterCondition {
         Self::default()
     }
 
-    _impl!(in_mailbox: Option<Id>);
-    _impl!(in_mailbox_other_than: Vec<Id>);
+    _impl!(in_mailbox: Option<Id<MailboxObject>>);
+    _impl!(in_mailbox_other_than: Vec<Id<MailboxObject>>);
     _impl!(before: UtcDate);
     _impl!(after: UtcDate);
     _impl!(min_size: Option<u64>);
@@ -582,6 +579,7 @@ impl From<crate::search::Query> for Filter<EmailFilterCondition, EmailObject> {
     fn from(val: crate::search::Query) -> Self {
         let mut ret = Filter::Condition(EmailFilterCondition::new().into());
         fn rec(q: &crate::search::Query, f: &mut Filter<EmailFilterCondition, EmailObject>) {
+            use crate::datetime::{timestamp_to_string, RFC3339_FMT};
             use crate::search::Query::*;
             match q {
                 Subject(t) => {
@@ -605,23 +603,48 @@ impl From<crate::search::Query> for Filter<EmailFilterCondition, EmailObject> {
                 Body(t) => {
                     *f = Filter::Condition(EmailFilterCondition::new().body(t.clone()).into());
                 }
-                Before(_) => {
-                    //TODO, convert UNIX timestamp into UtcDate
+                Before(t) => {
+                    *f = Filter::Condition(
+                        EmailFilterCondition::new()
+                            .before(timestamp_to_string(*t, Some(RFC3339_FMT), true))
+                            .into(),
+                    );
                 }
-                After(_) => {
-                    //TODO
+                After(t) => {
+                    *f = Filter::Condition(
+                        EmailFilterCondition::new()
+                            .after(timestamp_to_string(*t, Some(RFC3339_FMT), true))
+                            .into(),
+                    );
                 }
-                Between(_, _) => {
-                    //TODO
+                Between(a, b) => {
+                    *f = Filter::Condition(
+                        EmailFilterCondition::new()
+                            .after(timestamp_to_string(*a, Some(RFC3339_FMT), true))
+                            .into(),
+                    );
+                    *f &= Filter::Condition(
+                        EmailFilterCondition::new()
+                            .before(timestamp_to_string(*b, Some(RFC3339_FMT), true))
+                            .into(),
+                    );
                 }
-                On(_) => {
-                    //TODO
+                On(t) => {
+                    rec(&Between(*t, *t), f);
                 }
-                InReplyTo(_) => {
-                    //TODO, look inside Headers
+                InReplyTo(ref s) => {
+                    *f = Filter::Condition(
+                        EmailFilterCondition::new()
+                            .header(vec!["In-Reply-To".to_string().into(), s.to_string().into()])
+                            .into(),
+                    );
                 }
-                References(_) => {
-                    //TODO
+                References(ref s) => {
+                    *f = Filter::Condition(
+                        EmailFilterCondition::new()
+                            .header(vec!["References".to_string().into(), s.to_string().into()])
+                            .into(),
+                    );
                 }
                 AllAddresses(_) => {
                     //TODO
@@ -728,7 +751,7 @@ fn test_jmap_query() {
 
         let mut r = Filter::Condition(
             EmailFilterCondition::new()
-                .in_mailbox(Some(mailbox_id))
+                .in_mailbox(Some(mailbox_id.into()))
                 .into(),
         );
         r &= f;
@@ -737,7 +760,7 @@ fn test_jmap_query() {
 
     let email_call: EmailQuery = EmailQuery::new(
         Query::new()
-            .account_id("account_id".to_string())
+            .account_id("account_id".to_string().into())
             .filter(Some(filter))
             .position(0),
     )
@@ -770,3 +793,57 @@ impl EmailSet {
         EmailSet { set_call }
     }
 }
+
+#[derive(Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct EmailChanges {
+    #[serde(flatten)]
+    pub changes_call: Changes<EmailObject>,
+}
+
+impl Method<EmailObject> for EmailChanges {
+    const NAME: &'static str = "Email/changes";
+}
+
+impl EmailChanges {
+    pub fn new(changes_call: Changes<EmailObject>) -> Self {
+        EmailChanges { changes_call }
+    }
+}
+
+#[derive(Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct EmailQueryChanges {
+    #[serde(flatten)]
+    pub query_changes_call: QueryChanges<Filter<EmailFilterCondition, EmailObject>, EmailObject>,
+}
+
+impl Method<EmailObject> for EmailQueryChanges {
+    const NAME: &'static str = "Email/queryChanges";
+}
+
+impl EmailQueryChanges {
+    pub fn new(
+        query_changes_call: QueryChanges<Filter<EmailFilterCondition, EmailObject>, EmailObject>,
+    ) -> Self {
+        EmailQueryChanges { query_changes_call }
+    }
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct EmailQueryChangesResponse {
+    ///o  The "collapseThreads" argument that was used with "Email/query".
+    #[serde(default = "bool_false")]
+    pub collapse_threads: bool,
+    #[serde(flatten)]
+    pub query_changes_response: QueryChangesResponse<EmailObject>,
+}
+
+impl std::convert::TryFrom<&RawValue> for EmailQueryChangesResponse {
+    type Error = crate::error::MeliError;
+    fn try_from(t: &RawValue) -> Result<EmailQueryChangesResponse> {
+        let res: (String, EmailQueryChangesResponse, String) = serde_json::from_str(t.get())?;
+        assert_eq!(&res.0, "Email/queryChanges");
+        Ok(res.1)
+    }
+}

melib/src/backends/jmap/objects/email/import.rs

@@ -0,0 +1,200 @@
+/*
+ * meli -
+ *
+ * Copyright  Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+use serde_json::value::RawValue;
+
+/// #`import`
+///
+///    Objects of type `Foo` are imported via a call to `Foo/import`.
+///
+///    It takes the following arguments:
+///
+///    - `account_id`: "Id"
+///
+///       The id of the account to use.
+///
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct ImportCall {
+    ///accountId: "Id"
+    ///The id of the account to use.
+    pub account_id: Id<Account>,
+    ///ifInState: "String|null"
+    ///This is a state string as returned by the "Email/get" method.  If
+    ///supplied, the string must match the current state of the account
+    ///referenced by the accountId; otherwise, the method will be aborted
+    ///and a "stateMismatch" error returned.  If null, any changes will
+    ///be applied to the current state.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub if_in_state: Option<State<EmailObject>>,
+    ///o  emails: "Id[EmailImport]"
+    ///A map of creation id (client specified) to EmailImport objects.
+    pub emails: HashMap<Id<EmailObject>, EmailImport>,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct EmailImport {
+    ///o  blobId: "Id"
+    ///The id of the blob containing the raw message [RFC5322].
+    pub blob_id: Id<BlobObject>,
+    ///o  mailboxIds: "Id[Boolean]"
+    ///The ids of the Mailboxes to assign this Email to.  At least one
+    ///Mailbox MUST be given.
+    pub mailbox_ids: HashMap<Id<MailboxObject>, bool>,
+    ///o  keywords: "String[Boolean]" (default: {})
+    ///The keywords to apply to the Email.
+    pub keywords: HashMap<String, bool>,
+
+    ///o  receivedAt: "UTCDate" (default: time of most recent Received
+    ///header, or time of import on server if none)
+    ///The "receivedAt" date to set on the Email.
+    pub received_at: Option<String>,
+}
+
+impl ImportCall {
+    pub fn new() -> Self {
+        Self {
+            account_id: Id::new(),
+            if_in_state: None,
+            emails: HashMap::default(),
+        }
+    }
+
+    _impl!(
+        ///   -  accountId: "Id"
+        ///
+        ///      The id of the account to use.
+        ///
+        account_id: Id<Account>
+    );
+    _impl!(if_in_state: Option<State<EmailObject>>);
+    _impl!(emails: HashMap<Id<EmailObject>, EmailImport>);
+}
+
+impl Method<EmailObject> for ImportCall {
+    const NAME: &'static str = "Email/import";
+}
+
+impl EmailImport {
+    pub fn new() -> Self {
+        Self {
+            blob_id: Id::new(),
+            mailbox_ids: HashMap::default(),
+            keywords: HashMap::default(),
+            received_at: None,
+        }
+    }
+
+    _impl!(blob_id: Id<BlobObject>);
+    _impl!(mailbox_ids: HashMap<Id<MailboxObject>, bool>);
+    _impl!(keywords: HashMap<String, bool>);
+    _impl!(received_at: Option<String>);
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+#[serde(tag = "type")]
+pub enum ImportError {
+    ///The server MAY forbid two Email objects with the same exact content
+    ///   [RFC5322], or even just with the same Message-ID [RFC5322], to
+    ///   coexist within an account.  In this case, it MUST reject attempts to
+    ///   import an Email considered to be a duplicate with an "alreadyExists"
+    ///   SetError.
+    AlreadyExists {
+        description: Option<String>,
+        /// An "existingId" property of type "Id" MUST be included on
+        ///the SetError object with the id of the existing Email.  If duplicates
+        ///are allowed, the newly created Email object MUST have a separate id
+        ///and independent mutable properties to the existing object.
+        existing_id: Id<EmailObject>,
+    },
+    ///If the "blobId", "mailboxIds", or "keywords" properties are invalid
+    ///(e.g., missing, wrong type, id not found), the server MUST reject the
+    ///import with an "invalidProperties" SetError.
+    InvalidProperties {
+        description: Option<String>,
+        properties: Vec<String>,
+    },
+    ///If the Email cannot be imported because it would take the account
+    ///over quota, the import should be rejected with an "overQuota"
+    ///SetError.
+    OverQuota { description: Option<String> },
+    ///If the blob referenced is not a valid message [RFC5322], the server
+    ///MAY modify the message to fix errors (such as removing NUL octets or
+    ///fixing invalid headers).  If it does this, the "blobId" on the
+    ///response MUST represent the new representation and therefore be
+    ///different to the "blobId" on the EmailImport object.  Alternatively,
+    ///the server MAY reject the import with an "invalidEmail" SetError.
+    InvalidEmail { description: Option<String> },
+    ///An "ifInState" argument was supplied, and it does not match the current state.
+    StateMismatch,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct ImportResponse {
+    ///o  accountId: "Id"
+    ///The id of the account used for this call.
+    pub account_id: Id<Account>,
+
+    ///o  oldState: "String|null"
+    ///The state string that would have been returned by "Email/get" on
+    ///this account before making the requested changes, or null if the
+    ///server doesn't know what the previous state string was.
+    pub old_state: Option<State<EmailObject>>,
+
+    ///o  newState: "String"
+    ///The state string that will now be returned by "Email/get" on this
+    ///account.
+    pub new_state: Option<State<EmailObject>>,
+
+    ///o  created: "Id[Email]|null"
+    ///A map of the creation id to an object containing the "id",
+    ///"blobId", "threadId", and "size" properties for each successfully
+    ///imported Email, or null if none.
+    pub created: HashMap<Id<EmailObject>, ImportEmailResult>,
+
+    ///o  notCreated: "Id[SetError]|null"
+    ///A map of the creation id to a SetError object for each Email that
+    ///failed to be created, or null if all successful.  The possible
+    ///errors are defined above.
+    pub not_created: HashMap<Id<EmailObject>, ImportError>,
+}
+
+impl std::convert::TryFrom<&RawValue> for ImportResponse {
+    type Error = crate::error::MeliError;
+    fn try_from(t: &RawValue) -> Result<ImportResponse> {
+        let res: (String, ImportResponse, String) = serde_json::from_str(t.get())?;
+        assert_eq!(&res.0, &ImportCall::NAME);
+        Ok(res.1)
+    }
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct ImportEmailResult {
+    pub id: Id<EmailObject>,
+    pub blob_id: Id<BlobObject>,
+    pub thread_id: Id<ThreadObject>,
+    pub size: usize,
+}

melib/src/backends/jmap/objects/mailbox.rs

@@ -21,14 +21,22 @@
 
 use super::*;
 
+impl Id<MailboxObject> {
+    pub fn into_hash(&self) -> MailboxHash {
+        let mut h = DefaultHasher::new();
+        h.write(self.inner.as_bytes());
+        h.finish()
+    }
+}
+
 #[derive(Deserialize, Serialize, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct MailboxObject {
-    pub id: String,
+    pub id: Id<MailboxObject>,
     pub is_subscribed: bool,
     pub my_rights: JmapRights,
     pub name: String,
-    pub parent_id: Option<String>,
+    pub parent_id: Option<Id<MailboxObject>>,
     pub role: Option<String>,
     pub sort_order: u64,
     pub total_emails: u64,

melib/src/backends/jmap/operations.rs

@@ -20,21 +20,21 @@
  */
 
 use super::*;
-use std::sync::{Arc, RwLock};
+use std::sync::Arc;
 
 /// `BackendOp` implementor for Imap
 #[derive(Debug, Clone)]
 pub struct JmapOp {
     hash: EnvelopeHash,
     connection: Arc<FutureMutex<JmapConnection>>,
-    store: Arc<RwLock<Store>>,
+    store: Arc<Store>,
 }
 
 impl JmapOp {
     pub fn new(
         hash: EnvelopeHash,
         connection: Arc<FutureMutex<JmapConnection>>,
-        store: Arc<RwLock<Store>>,
+        store: Arc<Store>,
     ) -> Self {
         JmapOp {
             hash,
@@ -47,11 +47,9 @@ impl JmapOp {
 impl BackendOp for JmapOp {
     fn as_bytes(&mut self) -> ResultFuture<Vec<u8>> {
         {
-            let store_lck = self.store.read().unwrap();
-            if store_lck.byte_cache.contains_key(&self.hash)
-                && store_lck.byte_cache[&self.hash].bytes.is_some()
-            {
-                let ret = store_lck.byte_cache[&self.hash].bytes.clone().unwrap();
+            let byte_lck = self.store.byte_cache.lock().unwrap();
+            if byte_lck.contains_key(&self.hash) && byte_lck[&self.hash].bytes.is_some() {
+                let ret = byte_lck[&self.hash].bytes.clone().unwrap();
                 return Ok(Box::pin(async move { Ok(ret.into_bytes()) }));
             }
         }
@@ -59,14 +57,15 @@ impl BackendOp for JmapOp {
         let hash = self.hash;
         let connection = self.connection.clone();
         Ok(Box::pin(async move {
-            let blob_id = store.read().unwrap().blob_id_store[&hash].clone();
+            let blob_id = store.blob_id_store.lock().unwrap()[&hash].clone();
             let mut conn = connection.lock().await;
             conn.connect().await?;
+            let download_url = conn.session.lock().unwrap().download_url.clone();
             let mut res = conn
                 .client
                 .get_async(&download_request_format(
-                    &conn.session,
-                    conn.mail_account_id(),
+                    download_url.as_str(),
+                    &conn.mail_account_id(),
                     &blob_id,
                     None,
                 ))
@@ -75,9 +74,9 @@ impl BackendOp for JmapOp {
             let res_text = res.text_async().await?;
 
             store
-                .write()
-                .unwrap()
                 .byte_cache
+                .lock()
+                .unwrap()
                 .entry(hash)
                 .or_default()
                 .bytes = Some(res_text.clone());

melib/src/backends/jmap/protocol.rs

@@ -23,13 +23,8 @@ use super::mailbox::JmapMailbox;
 use super::*;
 use serde::Serialize;
 use serde_json::{json, Value};
-use smallvec::SmallVec;
-use std::collections::hash_map::DefaultHasher;
-use std::collections::HashMap;
-use std::convert::TryFrom;
-use std::hash::{Hash, Hasher};
+use std::convert::{TryFrom, TryInto};
 
-pub type Id = String;
 pub type UtcDate = String;
 
 use super::rfc8620::Object;
@@ -43,19 +38,6 @@ macro_rules! get_request_no {
     }};
 }
 
-macro_rules! tag_hash {
-    ($t:ident) => {{
-        let mut hasher = DefaultHasher::default();
-        $t.hash(&mut hasher);
-        hasher.finish()
-    }};
-    ($t:literal) => {{
-        let mut hasher = DefaultHasher::default();
-        $t.hash(&mut hasher);
-        hasher.finish()
-    }};
-}
-
 pub trait Response<OBJ: Object> {
     const NAME: &'static str;
 }
@@ -104,10 +86,11 @@ pub struct JsonResponse<'a> {
 
 pub async fn get_mailboxes(conn: &JmapConnection) -> Result<HashMap<MailboxHash, JmapMailbox>> {
     let seq = get_request_no!(conn.request_no);
+    let api_url = conn.session.lock().unwrap().api_url.clone();
     let mut res = conn
         .client
         .post_async(
-            &conn.session.api_url,
+            api_url.as_str(),
             serde_json::to_string(&json!({
                 "using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"],
                 "methodCalls": [["Mailbox/get", {
@@ -120,13 +103,13 @@ pub async fn get_mailboxes(conn: &JmapConnection) -> Result<HashMap<MailboxHash,
 
     let res_text = res.text_async().await?;
     let mut v: MethodResponse = serde_json::from_str(&res_text).unwrap();
-    *conn.online_status.lock().await = (std::time::Instant::now(), Ok(()));
+    *conn.store.online_status.lock().await = (std::time::Instant::now(), Ok(()));
     let m = GetResponse::<MailboxObject>::try_from(v.method_responses.remove(0))?;
     let GetResponse::<MailboxObject> {
         list, account_id, ..
     } = m;
-    *conn.account_id.lock().unwrap() = account_id;
-    Ok(list
+    *conn.store.account_id.lock().unwrap() = account_id;
+    let mut ret: HashMap<MailboxHash, JmapMailbox> = list
         .into_iter()
         .map(|r| {
             let MailboxObject {
@@ -142,18 +125,26 @@ pub async fn get_mailboxes(conn: &JmapConnection) -> Result<HashMap<MailboxHash,
                 unread_emails,
                 unread_threads,
             } = r;
-            let hash = crate::get_path_hash!(&name);
+            let mut total_emails_set = LazyCountSet::default();
+            total_emails_set.set_not_yet_seen(total_emails.try_into().unwrap_or(0));
+            let total_emails = total_emails_set;
+            let mut unread_emails_set = LazyCountSet::default();
+            unread_emails_set.set_not_yet_seen(unread_emails.try_into().unwrap_or(0));
+            let unread_emails = unread_emails_set;
+            let hash = id.into_hash();
+            let parent_hash = parent_id.clone().map(|id| id.into_hash());
             (
                 hash,
                 JmapMailbox {
                     name: name.clone(),
                     hash,
                     path: name,
-                    v: Vec::new(),
+                    children: Vec::new(),
                     id,
                     is_subscribed,
                     my_rights,
                     parent_id,
+                    parent_hash,
                     role,
                     usage: Default::default(),
                     sort_order,
@@ -161,16 +152,27 @@ pub async fn get_mailboxes(conn: &JmapConnection) -> Result<HashMap<MailboxHash,
                     total_threads,
                     unread_emails: Arc::new(Mutex::new(unread_emails)),
                     unread_threads,
+                    email_state: Arc::new(Mutex::new(None)),
+                    email_query_state: Arc::new(Mutex::new(None)),
                 },
             )
         })
-        .collect())
+        .collect();
+    for key in ret.keys().cloned().collect::<SmallVec<[MailboxHash; 24]>>() {
+        if let Some(parent_hash) = ret[&key].parent_hash.clone() {
+            ret.entry(parent_hash).and_modify(|e| e.children.push(key));
+        }
+    }
+    Ok(ret)
 }
 
-pub async fn get_message_list(conn: &JmapConnection, mailbox: &JmapMailbox) -> Result<Vec<String>> {
+pub async fn get_message_list(
+    conn: &JmapConnection,
+    mailbox: &JmapMailbox,
+) -> Result<Vec<Id<EmailObject>>> {
     let email_call: EmailQuery = EmailQuery::new(
         Query::new()
-            .account_id(conn.mail_account_id().to_string())
+            .account_id(conn.mail_account_id().clone())
             .filter(Some(Filter::Condition(
                 EmailFilterCondition::new()
                     .in_mailbox(Some(mailbox.id.clone()))
@@ -183,19 +185,21 @@ pub async fn get_message_list(conn: &JmapConnection, mailbox: &JmapMailbox) -> R
     let mut req = Request::new(conn.request_no.clone());
     req.add_call(&email_call);
 
+    let api_url = conn.session.lock().unwrap().api_url.clone();
     let mut res = conn
         .client
-        .post_async(&conn.session.api_url, serde_json::to_string(&req)?)
+        .post_async(api_url.as_str(), serde_json::to_string(&req)?)
         .await?;
 
     let res_text = res.text_async().await?;
     let mut v: MethodResponse = serde_json::from_str(&res_text).unwrap();
-    *conn.online_status.lock().await = (std::time::Instant::now(), Ok(()));
+    *conn.store.online_status.lock().await = (std::time::Instant::now(), Ok(()));
     let m = QueryResponse::<EmailObject>::try_from(v.method_responses.remove(0))?;
     let QueryResponse::<EmailObject> { ids, .. } = m;
     Ok(ids)
 }
 
+/*
 pub async fn get_message(conn: &JmapConnection, ids: &[String]) -> Result<Vec<Envelope>> {
     let email_call: EmailGet = EmailGet::new(
         Get::new()
@@ -219,18 +223,17 @@ pub async fn get_message(conn: &JmapConnection, ids: &[String]) -> Result<Vec<En
         .map(std::convert::Into::into)
         .collect::<Vec<Envelope>>())
 }
+*/
 
 pub async fn fetch(
     conn: &JmapConnection,
-    store: &Arc<RwLock<Store>>,
-    tag_index: &Arc<RwLock<BTreeMap<u64, String>>>,
-    mailboxes: &Arc<RwLock<HashMap<MailboxHash, JmapMailbox>>>,
+    store: &Store,
     mailbox_hash: MailboxHash,
 ) -> Result<Vec<Envelope>> {
-    let mailbox_id = mailboxes.read().unwrap()[&mailbox_hash].id.clone();
+    let mailbox_id = store.mailboxes.read().unwrap()[&mailbox_hash].id.clone();
     let email_query_call: EmailQuery = EmailQuery::new(
         Query::new()
-            .account_id(conn.mail_account_id().to_string())
+            .account_id(conn.mail_account_id().clone())
             .filter(Some(Filter::Condition(
                 EmailFilterCondition::new()
                     .in_mailbox(Some(mailbox_id))
@@ -249,93 +252,74 @@ pub async fn fetch(
                 prev_seq,
                 EmailQuery::RESULT_FIELD_IDS,
             )))
-            .account_id(conn.mail_account_id().to_string()),
+            .account_id(conn.mail_account_id().clone()),
     );
 
     req.add_call(&email_call);
 
+    let api_url = conn.session.lock().unwrap().api_url.clone();
     let mut res = conn
         .client
-        .post_async(&conn.session.api_url, serde_json::to_string(&req)?)
+        .post_async(api_url.as_str(), serde_json::to_string(&req)?)
         .await?;
 
     let res_text = res.text_async().await?;
 
     let mut v: MethodResponse = serde_json::from_str(&res_text).unwrap();
     let e = GetResponse::<EmailObject>::try_from(v.method_responses.pop().unwrap())?;
+    let query_response = QueryResponse::<EmailObject>::try_from(v.method_responses.pop().unwrap())?;
+    store
+        .mailboxes
+        .write()
+        .unwrap()
+        .entry(mailbox_hash)
+        .and_modify(|mbox| {
+            *mbox.email_query_state.lock().unwrap() = Some(query_response.query_state);
+        });
     let GetResponse::<EmailObject> { list, state, .. } = e;
     {
-        let mut states_lck = conn.method_call_states.lock().unwrap();
-
-        if let Some(prev_state) = states_lck.get_mut(&EmailGet::NAME) {
-            debug!("{:?}: prev_state was {}", EmailGet::NAME, prev_state);
-
-            if *prev_state != state { /* FIXME Query Changes. */ }
-
-            *prev_state = state;
-            debug!("{:?}: curr state is {}", EmailGet::NAME, prev_state);
-        } else {
-            debug!("{:?}: inserting state {}", EmailGet::NAME, &state);
-            states_lck.insert(EmailGet::NAME, state);
-        }
-    }
-    let mut tag_lck = tag_index.write().unwrap();
-    let ids = list
-        .iter()
-        .map(|obj| {
-            let tags = obj
-                .keywords()
-                .keys()
-                .map(|tag| {
-                    let tag_hash = {
-                        let mut hasher = DefaultHasher::default();
-                        tag.hash(&mut hasher);
-                        hasher.finish()
-                    };
-                    if !tag_lck.contains_key(&tag_hash) {
-                        tag_lck.insert(tag_hash, tag.to_string());
-                    }
-                    tag_hash
+        let (is_empty, is_equal) = {
+            let mailboxes_lck = conn.store.mailboxes.read().unwrap();
+            mailboxes_lck
+                .get(&mailbox_hash)
+                .map(|mbox| {
+                    let current_state_lck = mbox.email_state.lock().unwrap();
+                    (
+                        current_state_lck.is_none(),
+                        current_state_lck.as_ref() != Some(&state),
+                    )
                 })
-                .collect::<SmallVec<[u64; 1024]>>();
-            (tags, obj.id.clone(), obj.blob_id.clone())
-        })
-        .collect::<Vec<(SmallVec<[u64; 1024]>, Id, Id)>>();
-    drop(tag_lck);
-    let mut ret = list
-        .into_iter()
-        .map(std::convert::Into::into)
-        .collect::<Vec<Envelope>>();
-
-    let mut store_lck = store.write().unwrap();
-    debug_assert_eq!(tag_hash!("$draft"), 6613915297903591176);
-    debug_assert_eq!(tag_hash!("$seen"), 1683863812294339685);
-    debug_assert_eq!(tag_hash!("$flagged"), 2714010747478170100);
-    debug_assert_eq!(tag_hash!("$answered"), 8940855303929342213);
-    debug_assert_eq!(tag_hash!("$junk"), 2656839745430720464);
-    debug_assert_eq!(tag_hash!("$notjunk"), 4091323799684325059);
-    for (env, (tags, id, blob_id)) in ret.iter_mut().zip(ids.into_iter()) {
-        store_lck.id_store.insert(env.hash(), id);
-        store_lck.blob_id_store.insert(env.hash(), blob_id);
-        for t in tags {
-            match t {
-                6613915297903591176 => {
-                    env.set_flags(env.flags() | Flag::DRAFT);
-                }
-                1683863812294339685 => {
-                    env.set_flags(env.flags() | Flag::SEEN);
-                }
-                2714010747478170100 => {
-                    env.set_flags(env.flags() | Flag::FLAGGED);
-                }
-                8940855303929342213 => {
-                    env.set_flags(env.flags() | Flag::REPLIED);
-                }
-                2656839745430720464 | 4091323799684325059 => { /* ignore */ }
-                _ => env.labels_mut().push(t),
-            }
+                .unwrap_or((true, true))
+        };
+        if is_empty {
+            let mut mailboxes_lck = conn.store.mailboxes.write().unwrap();
+            debug!("{:?}: inserting state {}", EmailObject::NAME, &state);
+            mailboxes_lck.entry(mailbox_hash).and_modify(|mbox| {
+                *mbox.email_state.lock().unwrap() = Some(state);
+            });
+        } else if !is_equal {
+            conn.email_changes(mailbox_hash).await?;
         }
     }
+    let mut total = BTreeSet::default();
+    let mut unread = BTreeSet::default();
+    let mut ret = Vec::with_capacity(list.len());
+    for obj in list {
+        let env = store.add_envelope(obj);
+        total.insert(env.hash());
+        if !env.is_seen() {
+            unread.insert(env.hash());
+        }
+        ret.push(env);
+    }
+    let mut mailboxes_lck = store.mailboxes.write().unwrap();
+    mailboxes_lck.entry(mailbox_hash).and_modify(|mbox| {
+        mbox.total_emails.lock().unwrap().insert_existing_set(total);
+        mbox.unread_emails
+            .lock()
+            .unwrap()
+            .insert_existing_set(unread);
+    });
     Ok(ret)
 }
 

melib/src/backends/jmap/rfc8620.rs

@@ -19,12 +19,13 @@
  * along with meli. If not, see <http://www.gnu.org/licenses/>.
  */
 
-use super::Id;
 use crate::email::parser::BytesExt;
 use core::marker::PhantomData;
 use serde::de::DeserializeOwned;
 use serde::ser::{Serialize, SerializeStruct, Serializer};
 use serde_json::{value::RawValue, Value};
+use std::hash::{Hash, Hasher};
+use std::sync::Arc;
 
 mod filters;
 pub use filters::*;
@@ -39,23 +40,189 @@ pub trait Object {
     const NAME: &'static str;
 }
 
+#[derive(Deserialize, Serialize)]
+#[serde(transparent)]
+pub struct Id<OBJ> {
+    pub inner: String,
+    #[serde(skip)]
+    pub _ph: PhantomData<fn() -> OBJ>,
+}
+
+impl<OBJ: Object> core::fmt::Debug for Id<OBJ> {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        f.debug_tuple(&format!("Id<{}>", OBJ::NAME))
+            .field(&self.inner)
+            .finish()
+    }
+}
+
+impl core::fmt::Debug for Id<String> {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        f.debug_tuple("Id<Any>").field(&self.inner).finish()
+    }
+}
+
+//, Hash, Eq, PartialEq, Default)]
+impl<OBJ> Clone for Id<OBJ> {
+    fn clone(&self) -> Self {
+        Id {
+            inner: self.inner.clone(),
+            _ph: PhantomData,
+        }
+    }
+}
+
+impl<OBJ> std::cmp::Eq for Id<OBJ> {}
+
+impl<OBJ> std::cmp::PartialEq for Id<OBJ> {
+    fn eq(&self, other: &Self) -> bool {
+        self.inner == other.inner
+    }
+}
+
+impl<OBJ> Hash for Id<OBJ> {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.inner.hash(state);
+    }
+}
+
+impl<OBJ> Default for Id<OBJ> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<OBJ> From<String> for Id<OBJ> {
+    fn from(inner: String) -> Self {
+        Id {
+            inner,
+            _ph: PhantomData,
+        }
+    }
+}
+
+impl<OBJ> core::fmt::Display for Id<OBJ> {
+    fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
+        core::fmt::Display::fmt(&self.inner, fmt)
+    }
+}
+
+impl<OBJ> Id<OBJ> {
+    pub fn new() -> Self {
+        Self {
+            inner: String::new(),
+            _ph: PhantomData,
+        }
+    }
+
+    pub fn as_str(&self) -> &str {
+        self.inner.as_str()
+    }
+
+    pub fn len(&self) -> usize {
+        self.inner.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.inner.is_empty()
+    }
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(transparent)]
+pub struct State<OBJ> {
+    pub inner: String,
+    #[serde(skip)]
+    pub _ph: PhantomData<fn() -> OBJ>,
+}
+
+//, Hash, Eq, PartialEq, Default)]
+impl<OBJ> Clone for State<OBJ> {
+    fn clone(&self) -> Self {
+        State {
+            inner: self.inner.clone(),
+            _ph: PhantomData,
+        }
+    }
+}
+
+impl<OBJ> std::cmp::Eq for State<OBJ> {}
+
+impl<OBJ> std::cmp::PartialEq for State<OBJ> {
+    fn eq(&self, other: &Self) -> bool {
+        self.inner == other.inner
+    }
+}
+
+impl<OBJ> Hash for State<OBJ> {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.inner.hash(state);
+    }
+}
+
+impl<OBJ> Default for State<OBJ> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<OBJ> From<String> for State<OBJ> {
+    fn from(inner: String) -> Self {
+        State {
+            inner,
+            _ph: PhantomData,
+        }
+    }
+}
+
+impl<OBJ> core::fmt::Display for State<OBJ> {
+    fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
+        core::fmt::Display::fmt(&self.inner, fmt)
+    }
+}
+
+impl<OBJ> State<OBJ> {
+    pub fn new() -> Self {
+        Self {
+            inner: String::new(),
+            _ph: PhantomData,
+        }
+    }
+
+    pub fn as_str(&self) -> &str {
+        self.inner.as_str()
+    }
+
+    pub fn len(&self) -> usize {
+        self.inner.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.inner.is_empty()
+    }
+}
+
 #[derive(Deserialize, Serialize, Debug, Default)]
 #[serde(rename_all = "camelCase")]
 pub struct JmapSession {
     pub capabilities: HashMap<String, CapabilitiesObject>,
-    pub accounts: HashMap<Id, Account>,
-    pub primary_accounts: HashMap<String, Id>,
+    pub accounts: HashMap<Id<Account>, Account>,
+    pub primary_accounts: HashMap<String, Id<Account>>,
     pub username: String,
-    pub api_url: String,
-    pub download_url: String,
+    pub api_url: Arc<String>,
+    pub download_url: Arc<String>,
 
-    pub upload_url: String,
-    pub event_source_url: String,
-    pub state: String,
+    pub upload_url: Arc<String>,
+    pub event_source_url: Arc<String>,
+    pub state: State<JmapSession>,
     #[serde(flatten)]
     pub extra_properties: HashMap<String, Value>,
 }
 
+impl Object for JmapSession {
+    const NAME: &'static str = "Session";
+}
+
 #[derive(Deserialize, Serialize, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct CapabilitiesObject {
@@ -88,6 +255,17 @@ pub struct Account {
     extra_properties: HashMap<String, Value>,
 }
 
+impl Object for Account {
+    const NAME: &'static str = "Account";
+}
+
+#[derive(Debug)]
+pub struct BlobObject;
+
+impl Object for BlobObject {
+    const NAME: &'static str = "Blob";
+}
+
 /// #`get`
 ///
 ///    Objects of type `Foo` are fetched via a call to `Foo/get`.
@@ -104,11 +282,10 @@ pub struct Get<OBJ: Object>
 where
     OBJ: std::fmt::Debug + Serialize,
 {
-    #[serde(skip_serializing_if = "String::is_empty")]
-    pub account_id: String,
+    pub account_id: Id<Account>,
     #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(flatten)]
-    pub ids: Option<JmapArgument<Vec<String>>>,
+    pub ids: Option<JmapArgument<Vec<Id<OBJ>>>>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub properties: Option<Vec<String>>,
     #[serde(skip)]
@@ -121,7 +298,7 @@ where
 {
     pub fn new() -> Self {
         Self {
-            account_id: String::new(),
+            account_id: Id::new(),
             ids: None,
             properties: None,
             _ph: PhantomData,
@@ -132,7 +309,7 @@ where
         ///
         ///      The id of the account to use.
         ///
-        account_id: String
+        account_id: Id<Account>
     );
     _impl!(
         ///   -  ids: `Option<JmapArgument<Vec<String>>>`
@@ -142,7 +319,7 @@ where
         ///      type and the number of records does not exceed the
         ///      "max_objects_in_get" limit.
         ///
-        ids: Option<JmapArgument<Vec<String>>>
+        ids: Option<JmapArgument<Vec<Id<OBJ>>>>
     );
     _impl!(
         ///   -  properties: Option<Vec<String>>
@@ -218,20 +395,19 @@ pub struct MethodResponse<'a> {
     #[serde(borrow)]
     pub method_responses: Vec<&'a RawValue>,
     #[serde(default)]
-    pub created_ids: HashMap<Id, Id>,
+    pub created_ids: HashMap<Id<String>, Id<String>>,
     #[serde(default)]
-    pub session_state: String,
+    pub session_state: State<JmapSession>,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct GetResponse<OBJ: Object> {
-    #[serde(skip_serializing_if = "String::is_empty")]
-    pub account_id: String,
-    #[serde(default)]
-    pub state: String,
+    pub account_id: Id<Account>,
+    #[serde(default = "State::default")]
+    pub state: State<OBJ>,
     pub list: Vec<OBJ>,
-    pub not_found: Vec<String>,
+    pub not_found: Vec<Id<OBJ>>,
 }
 
 impl<OBJ: Object + DeserializeOwned> std::convert::TryFrom<&RawValue> for GetResponse<OBJ> {
@@ -244,10 +420,10 @@ impl<OBJ: Object + DeserializeOwned> std::convert::TryFrom<&RawValue> for GetRes
 }
 
 impl<OBJ: Object> GetResponse<OBJ> {
-    _impl!(get_mut  account_id_mut, account_id: String);
-    _impl!(get_mut  state_mut, state: String);
+    _impl!(get_mut  account_id_mut, account_id: Id<Account>);
+    _impl!(get_mut  state_mut, state: State<OBJ>);
     _impl!(get_mut  list_mut, list: Vec<OBJ>);
-    _impl!(get_mut  not_found_mut, not_found: Vec<String>);
+    _impl!(get_mut  not_found_mut, not_found: Vec<Id<OBJ>>);
 }
 
 #[derive(Deserialize, Debug)]
@@ -264,7 +440,7 @@ pub struct Query<F: FilterTrait<OBJ>, OBJ: Object>
 where
     OBJ: std::fmt::Debug + Serialize,
 {
-    account_id: String,
+    account_id: Id<Account>,
     filter: Option<F>,
     sort: Option<Comparator<OBJ>>,
     #[serde(default)]
@@ -288,7 +464,7 @@ where
 {
     pub fn new() -> Self {
         Self {
-            account_id: String::new(),
+            account_id: Id::new(),
             filter: None,
             sort: None,
             position: 0,
@@ -300,7 +476,7 @@ where
         }
     }
 
-    _impl!(account_id: String);
+    _impl!(account_id: Id<Account>);
     _impl!(filter: Option<F>);
     _impl!(sort: Option<Comparator<OBJ>>);
     _impl!(position: u64);
@@ -325,12 +501,11 @@ pub fn bool_true() -> bool {
 #[derive(Serialize, Deserialize, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct QueryResponse<OBJ: Object> {
-    #[serde(skip_serializing_if = "String::is_empty", default)]
-    pub account_id: String,
+    pub account_id: Id<Account>,
     pub query_state: String,
     pub can_calculate_changes: bool,
     pub position: u64,
-    pub ids: Vec<Id>,
+    pub ids: Vec<Id<OBJ>>,
     #[serde(default)]
     pub total: u64,
     #[serde(default)]
@@ -349,7 +524,7 @@ impl<OBJ: Object + DeserializeOwned> std::convert::TryFrom<&RawValue> for QueryR
 }
 
 impl<OBJ: Object> QueryResponse<OBJ> {
-    _impl!(get_mut  ids_mut, ids: Vec<Id>);
+    _impl!(get_mut  ids_mut, ids: Vec<Id<OBJ>>);
 }
 
 pub struct ResultField<M: Method<OBJ>, OBJ: Object> {
@@ -410,9 +585,8 @@ pub struct Changes<OBJ: Object>
 where
     OBJ: std::fmt::Debug + Serialize,
 {
-    #[serde(skip_serializing_if = "String::is_empty")]
-    pub account_id: String,
-    pub since_state: String,
+    pub account_id: Id<Account>,
+    pub since_state: State<OBJ>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub max_changes: Option<u64>,
     #[serde(skip)]
@@ -425,8 +599,8 @@ where
 {
     pub fn new() -> Self {
         Self {
-            account_id: String::new(),
-            since_state: String::new(),
+            account_id: Id::new(),
+            since_state: State::new(),
             max_changes: None,
             _ph: PhantomData,
         }
@@ -436,7 +610,7 @@ where
         ///
         ///      The id of the account to use.
         ///
-        account_id: String
+        account_id: Id<Account>
     );
     _impl!(
         ///    - since_state: "String"
@@ -446,7 +620,7 @@ where
         ///     state.
         ///
         ///
-        since_state: String
+        since_state: State<OBJ>
     );
     _impl!(
         ///    - max_changes: "UnsignedInt|null"
@@ -463,16 +637,15 @@ where
 #[derive(Serialize, Deserialize, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct ChangesResponse<OBJ: Object> {
-    #[serde(skip_serializing_if = "String::is_empty")]
-    pub account_id: String,
-    pub old_state: String,
-    pub new_state: String,
+    pub account_id: Id<Account>,
+    pub old_state: State<OBJ>,
+    pub new_state: State<OBJ>,
     pub has_more_changes: bool,
-    pub created: Vec<Id>,
-    pub updated: Vec<Id>,
-    pub destroyed: Vec<Id>,
+    pub created: Vec<Id<OBJ>>,
+    pub updated: Vec<Id<OBJ>>,
+    pub destroyed: Vec<Id<OBJ>>,
     #[serde(skip)]
-    _ph: PhantomData<fn() -> OBJ>,
+    pub _ph: PhantomData<fn() -> OBJ>,
 }
 
 impl<OBJ: Object + DeserializeOwned> std::convert::TryFrom<&RawValue> for ChangesResponse<OBJ> {
@@ -485,13 +658,13 @@ impl<OBJ: Object + DeserializeOwned> std::convert::TryFrom<&RawValue> for Change
 }
 
 impl<OBJ: Object> ChangesResponse<OBJ> {
-    _impl!(get_mut  account_id_mut, account_id: String);
-    _impl!(get_mut  old_state_mut, old_state: String);
-    _impl!(get_mut  new_state_mut, new_state: String);
+    _impl!(get_mut  account_id_mut, account_id: Id<Account>);
+    _impl!(get_mut  old_state_mut, old_state: State<OBJ>);
+    _impl!(get_mut  new_state_mut, new_state: State<OBJ>);
     _impl!(get has_more_changes, has_more_changes: bool);
-    _impl!(get_mut  created_mut, created: Vec<String>);
-    _impl!(get_mut  updated_mut, updated: Vec<String>);
-    _impl!(get_mut  destroyed_mut, destroyed: Vec<String>);
+    _impl!(get_mut  created_mut, created: Vec<Id<OBJ>>);
+    _impl!(get_mut  updated_mut, updated: Vec<Id<OBJ>>);
+    _impl!(get_mut  destroyed_mut, destroyed: Vec<Id<OBJ>>);
 }
 
 ///#`set`
@@ -508,11 +681,10 @@ pub struct Set<OBJ: Object>
 where
     OBJ: std::fmt::Debug + Serialize,
 {
-    #[serde(skip_serializing_if = "String::is_empty")]
     ///o  accountId: "Id"
     ///
     ///   The id of the account to use.
-    pub account_id: String,
+    pub account_id: Id<Account>,
     ///o  ifInState: "String|null"
     ///
     ///   This is a state string as returned by the "Foo/get" method
@@ -521,7 +693,7 @@ where
     ///   otherwise, the method will be aborted and a "stateMismatch" error
     ///   returned.  If null, any changes will be applied to the current
     ///   state.
-    pub if_in_state: Option<String>,
+    pub if_in_state: Option<State<OBJ>>,
     ///o  create: "Id[Foo]|null"
     ///
     ///   A map of a *creation id* (a temporary id set by the client) to Foo
@@ -533,7 +705,7 @@ where
     ///   The client MUST omit any properties that may only be set by the
     ///   server (for example, the "id" property on most object types).
     ///
-    pub create: Option<HashMap<Id, OBJ>>,
+    pub create: Option<HashMap<Id<OBJ>, OBJ>>,
     ///o  update: "Id[PatchObject]|null"
     ///
     ///   A map of an id to a Patch object to apply to the current Foo
@@ -577,12 +749,12 @@ where
     ///   is also a valid PatchObject.  The client may choose to optimise
     ///   network usage by just sending the diff or may send the whole
     ///   object; the server processes it the same either way.
-    pub update: Option<HashMap<Id, Value>>,
+    pub update: Option<HashMap<Id<OBJ>, Value>>,
     ///o  destroy: "Id[]|null"
     ///
     ///   A list of ids for Foo objects to permanently delete, or null if no
     ///   objects are to be destroyed.
-    pub destroy: Option<Vec<Id>>,
+    pub destroy: Option<Vec<Id<OBJ>>>,
 }
 
 impl<OBJ: Object> Set<OBJ>
@@ -591,14 +763,14 @@ where
 {
     pub fn new() -> Self {
         Self {
-            account_id: String::new(),
+            account_id: Id::new(),
             if_in_state: None,
             create: None,
             update: None,
             destroy: None,
         }
     }
-    _impl!(account_id: String);
+    _impl!(account_id: Id<Account>);
     _impl!(
         ///o  ifInState: "String|null"
         ///
@@ -608,9 +780,9 @@ where
         ///   otherwise, the method will be aborted and a "stateMismatch" error
         ///   returned.  If null, any changes will be applied to the current
         ///   state.
-        if_in_state: Option<String>
+        if_in_state: Option<State<OBJ>>
     );
-    _impl!(update: Option<HashMap<Id, Value>>);
+    _impl!(update: Option<HashMap<Id<OBJ>, Value>>);
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -619,17 +791,17 @@ pub struct SetResponse<OBJ: Object> {
     ///o  accountId: "Id"
     ///
     ///   The id of the account used for the call.
-    pub account_id: String,
+    pub account_id: Id<Account>,
     ///o  oldState: "String|null"
     ///
     ///   The state string that would have been returned by "Foo/get" before
     ///   making the requested changes, or null if the server doesn't know
     ///   what the previous state string was.
-    pub old_state: String,
+    pub old_state: State<OBJ>,
     ///o  newState: "String"
     ///
     ///   The state string that will now be returned by "Foo/get".
-    pub new_state: String,
+    pub new_state: State<OBJ>,
     ///o  created: "Id[Foo]|null"
     ///
     ///   A map of the creation id to an object containing any properties of
@@ -639,7 +811,7 @@ pub struct SetResponse<OBJ: Object> {
     ///   and thus set to a default by the server.
     ///
     ///   This argument is null if no Foo objects were successfully created.
-    pub created: Option<HashMap<Id, OBJ>>,
+    pub created: Option<HashMap<Id<OBJ>, OBJ>>,
     ///o  updated: "Id[Foo|null]|null"
     ///
     ///   The keys in this map are the ids of all Foos that were
@@ -651,12 +823,12 @@ pub struct SetResponse<OBJ: Object> {
     ///   any changes to server-set or computed properties.
     ///
     ///   This argument is null if no Foo objects were successfully updated.
-    pub updated: Option<HashMap<Id, Option<OBJ>>>,
+    pub updated: Option<HashMap<Id<OBJ>, Option<OBJ>>>,
     ///o  destroyed: "Id[]|null"
     ///
     ///   A list of Foo ids for records that were successfully destroyed, or
     ///   null if none.
-    pub destroyed: Option<Vec<Id>>,
+    pub destroyed: Option<Vec<Id<OBJ>>>,
     ///o  notCreated: "Id[SetError]|null"
     ///
     ///   A map of the creation id to a SetError object for each record that
@@ -759,50 +931,50 @@ impl core::fmt::Display for SetError {
 }
 
 pub fn download_request_format(
-    session: &JmapSession,
-    account_id: &Id,
-    blob_id: &Id,
+    download_url: &str,
+    account_id: &Id<Account>,
+    blob_id: &Id<BlobObject>,
     name: Option<String>,
 ) -> String {
     // https://jmap.fastmail.com/download/{accountId}/{blobId}/{name}
     let mut ret = String::with_capacity(
-        session.download_url.len()
+        download_url.len()
             + blob_id.len()
             + name.as_ref().map(|n| n.len()).unwrap_or(0)
             + account_id.len(),
     );
     let mut prev_pos = 0;
 
-    while let Some(pos) = session.download_url.as_bytes()[prev_pos..].find(b"{") {
-        ret.push_str(&session.download_url[prev_pos..prev_pos + pos]);
+    while let Some(pos) = download_url.as_bytes()[prev_pos..].find(b"{") {
+        ret.push_str(&download_url[prev_pos..prev_pos + pos]);
         prev_pos += pos;
-        if session.download_url[prev_pos..].starts_with("{accountId}") {
-            ret.push_str(account_id);
+        if download_url[prev_pos..].starts_with("{accountId}") {
+            ret.push_str(account_id.as_str());
             prev_pos += "{accountId}".len();
-        } else if session.download_url[prev_pos..].starts_with("{blobId}") {
-            ret.push_str(blob_id);
+        } else if download_url[prev_pos..].starts_with("{blobId}") {
+            ret.push_str(blob_id.as_str());
             prev_pos += "{blobId}".len();
-        } else if session.download_url[prev_pos..].starts_with("{name}") {
+        } else if download_url[prev_pos..].starts_with("{name}") {
             ret.push_str(name.as_ref().map(String::as_str).unwrap_or(""));
             prev_pos += "{name}".len();
         }
     }
-    if prev_pos != session.download_url.len() {
-        ret.push_str(&session.download_url[prev_pos..]);
+    if prev_pos != download_url.len() {
+        ret.push_str(&download_url[prev_pos..]);
     }
     ret
 }
 
-pub fn upload_request_format(session: &JmapSession, account_id: &Id) -> String {
+pub fn upload_request_format(upload_url: &str, account_id: &Id<Account>) -> String {
     //"uploadUrl": "https://jmap.fastmail.com/upload/{accountId}/",
-    let mut ret = String::with_capacity(session.upload_url.len() + account_id.len());
+    let mut ret = String::with_capacity(upload_url.len() + account_id.len());
     let mut prev_pos = 0;
 
-    while let Some(pos) = session.upload_url.as_bytes()[prev_pos..].find(b"{") {
-        ret.push_str(&session.upload_url[prev_pos..prev_pos + pos]);
+    while let Some(pos) = upload_url.as_bytes()[prev_pos..].find(b"{") {
+        ret.push_str(&upload_url[prev_pos..prev_pos + pos]);
         prev_pos += pos;
-        if session.upload_url[prev_pos..].starts_with("{accountId}") {
-            ret.push_str(account_id);
+        if upload_url[prev_pos..].starts_with("{accountId}") {
+            ret.push_str(account_id.as_str());
             prev_pos += "{accountId}".len();
             break;
         } else {
@@ -810,8 +982,193 @@ pub fn upload_request_format(session: &JmapSession, account_id: &Id) -> String {
             prev_pos += 1;
         }
     }
-    if prev_pos != session.upload_url.len() {
-        ret.push_str(&session.upload_url[prev_pos..]);
+    if prev_pos != upload_url.len() {
+        ret.push_str(&upload_url[prev_pos..]);
     }
     ret
 }
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct UploadResponse {
+    ///o  accountId: "Id"
+    ///
+    ///   The id of the account used for the call.
+    pub account_id: Id<Account>,
+    ///o  blobId: "Id"
+    ///
+    ///The id representing the binary data uploaded.  The data for this id is immutable.
+    ///The id *only* refers to the binary data, not any metadata.
+    pub blob_id: Id<BlobObject>,
+    ///o  type: "String"
+    ///
+    ///The media type of the file (as specified in [RFC6838],
+    ///Section 4.2) as set in the Content-Type header of the upload HTTP
+    ///request.
+
+    #[serde(rename = "type")]
+    pub _type: String,
+
+    ///o  size: "UnsignedInt"
+    ///
+    /// The size of the file in octets.
+    pub size: usize,
+}
+
+/// #`queryChanges`
+///
+///   The "Foo/queryChanges" method allows a client to efficiently update
+///   the state of a cached query to match the new state on the server.  It
+///   takes the following arguments:
+#[derive(Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct QueryChanges<F: FilterTrait<OBJ>, OBJ: Object>
+where
+    OBJ: std::fmt::Debug + Serialize,
+{
+    pub account_id: Id<Account>,
+    pub filter: Option<F>,
+    pub sort: Option<Comparator<OBJ>>,
+    ///sinceQueryState: "String"
+    ///
+    ///The current state of the query in the client.  This is the string
+    ///that was returned as the "queryState" argument in the "Foo/query"
+    ///response with the same sort/filter.  The server will return the
+    ///changes made to the query since this state.
+    pub since_query_state: String,
+    ///o  maxChanges: "UnsignedInt|null"
+    ///
+    ///The maximum number of changes to return in the response.  See
+    ///error descriptions below for more details.
+    pub max_changes: Option<usize>,
+    ///o  upToId: "Id|null"
+    ///
+    ///The last (highest-index) id the client currently has cached from
+    ///the query results.  When there are a large number of results, in a
+    ///common case, the client may have only downloaded and cached a
+    ///small subset from the beginning of the results.  If the sort and
+    ///filter are both only on immutable properties, this allows the
+    ///server to omit changes after this point in the results, which can
+    ///significantly increase efficiency.  If they are not immutable,
+    ///this argument is ignored.
+    pub up_to_id: Option<Id<OBJ>>,
+
+    ///o  calculateTotal: "Boolean" (default: false)
+    ///
+    ///Does the client wish to know the total number of results now in
+    ///the query?  This may be slow and expensive for servers to
+    ///calculate, particularly with complex filters, so clients should
+    ///take care to only request the total when needed.
+    #[serde(default = "bool_false")]
+    pub calculate_total: bool,
+
+    #[serde(skip)]
+    _ph: PhantomData<fn() -> OBJ>,
+}
+
+impl<F: FilterTrait<OBJ>, OBJ: Object> QueryChanges<F, OBJ>
+where
+    OBJ: std::fmt::Debug + Serialize,
+{
+    pub fn new(account_id: Id<Account>, since_query_state: String) -> Self {
+        Self {
+            account_id,
+            filter: None,
+            sort: None,
+            since_query_state,
+            max_changes: None,
+            up_to_id: None,
+            calculate_total: false,
+            _ph: PhantomData,
+        }
+    }
+    _impl!(filter: Option<F>);
+    _impl!(sort: Option<Comparator<OBJ>>);
+    _impl!(max_changes: Option<usize>);
+    _impl!(up_to_id: Option<Id<OBJ>>);
+    _impl!(calculate_total: bool);
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct QueryChangesResponse<OBJ: Object> {
+    /// The id of the account used for the call.
+    pub account_id: Id<Account>,
+    /// This is the "sinceQueryState" argument echoed back; that is, the state from which the server is returning changes.
+    pub old_query_state: String,
+    ///This is the state the query will be in after applying the set of changes to the old state.
+    pub new_query_state: String,
+    /// The total number of Foos in the results (given the "filter").  This argument MUST be omitted if the "calculateTotal" request argument is not true.
+    #[serde(default)]
+    pub total: Option<usize>,
+    ///The "id" for every Foo that was in the query results in the old
+    ///state and that is not in the results in the new state.
+
+    ///If the server cannot calculate this exactly, the server MAY return
+    ///the ids of extra Foos in addition that may have been in the old
+    ///results but are not in the new results.
+
+    ///If the sort and filter are both only on immutable properties and
+    ///an "upToId" is supplied and exists in the results, any ids that
+    ///were removed but have a higher index than "upToId" SHOULD be
+    ///omitted.
+
+    ///If the "filter" or "sort" includes a mutable property, the server
+    ///MUST include all Foos in the current results for which this
+    ///property may have changed.  The position of these may have moved
+    ///in the results, so they must be reinserted by the client to ensure
+    ///its query cache is correct.
+    pub removed: Vec<Id<OBJ>>,
+    ///The id and index in the query results (in the new state) for every
+    ///Foo that has been added to the results since the old state AND
+    ///every Foo in the current results that was included in the
+    ///"removed" array (due to a filter or sort based upon a mutable
+    ///property).
+
+    ///If the sort and filter are both only on immutable properties and
+    ///an "upToId" is supplied and exists in the results, any ids that
+    ///were added but have a higher index than "upToId" SHOULD be
+    ///omitted.
+
+    ///The array MUST be sorted in order of index, with the lowest index
+    ///first.
+
+    ///An *AddedItem* object has the following properties:
+
+    ///*  id: "Id"
+
+    ///*  index: "UnsignedInt"
+
+    ///The result of this is that if the client has a cached sparse array of
+    ///Foo ids corresponding to the results in the old state, then:
+
+    ///fooIds = [ "id1", "id2", null, null, "id3", "id4", null, null, null ]
+
+    ///If it *splices out* all ids in the removed array that it has in its
+    ///cached results, then:
+
+    ///   removed = [ "id2", "id31", ... ];
+    ///   fooIds => [ "id1", null, null, "id3", "id4", null, null, null ]
+
+    ///and *splices in* (one by one in order, starting with the lowest
+    ///index) all of the ids in the added array:
+
+    ///added = [{ id: "id5", index: 0, ... }];
+    ///fooIds => [ "id5", "id1", null, null, "id3", "id4", null, null, null ]
+
+    ///and *truncates* or *extends* to the new total length, then the
+    ///results will now be in the new state.
+
+    ///Note: splicing in adds the item at the given index, incrementing the
+    ///index of all items previously at that or a higher index.  Splicing
+    ///out is the inverse, removing the item and decrementing the index of
+    ///every item after it in the array.
+    pub added: Vec<AddedItem<OBJ>>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct AddedItem<OBJ: Object> {
+    pub id: Id<OBJ>,
+    pub index: usize,
+}

melib/src/backends/maildir.rs

@@ -30,12 +30,11 @@ use crate::backends::*;
 use crate::email::Flag;
 use crate::error::{MeliError, Result};
 use crate::shellexpand::ShellExpandTrait;
-pub use futures::stream::Stream;
-
-use memmap::{Mmap, Protection};
+use futures::stream::Stream;
 use std::collections::hash_map::DefaultHasher;
 use std::fs;
 use std::hash::{Hash, Hasher};
+use std::io::{BufReader, Read};
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex};
 
@@ -45,7 +44,7 @@ pub struct MaildirOp {
     hash_index: HashIndexes,
     mailbox_hash: MailboxHash,
     hash: EnvelopeHash,
-    slice: Option<Mmap>,
+    slice: Option<Vec<u8>>,
 }
 
 impl Clone for MaildirOp {
@@ -68,7 +67,7 @@ impl MaildirOp {
             slice: None,
         }
     }
-    fn path(&self) -> PathBuf {
+    fn path(&self) -> Result<PathBuf> {
         let map = self.hash_index.lock().unwrap();
         let map = &map[&self.mailbox_hash];
         debug!("looking for {} in {} map", self.hash, self.mailbox_hash);
@@ -77,30 +76,38 @@ impl MaildirOp {
             for e in map.iter() {
                 debug!("{:#?}", e);
             }
+            return Err(MeliError::new("File not found"));
         }
-        if let Some(modif) = &map[&self.hash].modified {
+
+        Ok(if let Some(modif) = &map[&self.hash].modified {
             match modif {
                 PathMod::Path(ref path) => path.clone(),
                 PathMod::Hash(hash) => map[&hash].to_path_buf(),
             }
         } else {
             map.get(&self.hash).unwrap().to_path_buf()
-        }
+        })
     }
 }
 
 impl<'a> BackendOp for MaildirOp {
     fn as_bytes(&mut self) -> ResultFuture<Vec<u8>> {
         if self.slice.is_none() {
-            self.slice = Some(Mmap::open_path(self.path(), Protection::Read)?);
+            let file = std::fs::OpenOptions::new()
+                .read(true)
+                .write(false)
+                .open(&self.path()?)?;
+            let mut buf_reader = BufReader::new(file);
+            let mut contents = Vec::new();
+            buf_reader.read_to_end(&mut contents)?;
+            self.slice = Some(contents);
         }
-        /* Unwrap is safe since we use ? above. */
-        let ret = Ok((unsafe { self.slice.as_ref().unwrap().as_slice() }).to_vec());
+        let ret = Ok(self.slice.as_ref().unwrap().as_slice().to_vec());
         Ok(Box::pin(async move { ret }))
     }
 
     fn fetch_flags(&self) -> ResultFuture<Flag> {
-        let path = self.path();
+        let path = self.path()?;
         let ret = Ok(path.flags());
         Ok(Box::pin(async move { ret }))
     }

melib/src/backends/maildir/backend.rs

@@ -25,9 +25,9 @@ use crate::conf::AccountSettings;
 use crate::email::{Envelope, EnvelopeHash, Flag};
 use crate::error::{ErrorKind, MeliError, Result};
 use crate::shellexpand::ShellExpandTrait;
+use crate::Collection;
 use futures::prelude::Stream;
 
-use memmap::{Mmap, Protection};
 extern crate notify;
 use self::notify::{watcher, DebouncedEvent, RecursiveMode, Watcher};
 use std::time::Duration;
@@ -110,6 +110,7 @@ pub struct MaildirType {
     mailbox_index: Arc<Mutex<HashMap<EnvelopeHash, MailboxHash>>>,
     hash_indexes: HashIndexes,
     event_consumer: BackendEventConsumer,
+    collection: Collection,
     path: PathBuf,
 }
 
@@ -142,15 +143,6 @@ macro_rules! get_path_hash {
 }
 
 pub(super) fn get_file_hash(file: &Path) -> EnvelopeHash {
-    /*
-    let mut buf = Vec::with_capacity(2048);
-    let mut f = fs::File::open(&file).unwrap_or_else(|_| panic!("Can't open {}", file.display()));
-    f.read_to_end(&mut buf)
-        .unwrap_or_else(|_| panic!("Can't read {}", file.display()));
-    let mut hasher = DefaultHasher::default();
-    hasher.write(&buf);
-    hasher.finish()
-        */
     let mut hasher = DefaultHasher::default();
     file.hash(&mut hasher);
     hasher.finish()
@@ -241,6 +233,7 @@ impl MailBackend for MaildirType {
         Ok(Box::pin(async move {
             let thunk = move |sender: &BackendEventConsumer| {
                 debug!("refreshing");
+                let mut buf = Vec::with_capacity(4096);
                 let mut path = path.clone();
                 path.push("new");
                 for d in path.read_dir()? {
@@ -278,10 +271,10 @@ impl MailBackend for MaildirType {
                         }
                         (*map).insert(hash, PathBuf::from(&file).into());
                     }
-                    if let Ok(mut env) = Envelope::from_bytes(
-                        unsafe { &Mmap::open_path(&file, Protection::Read)?.as_slice() },
-                        Some(file.flags()),
-                    ) {
+                    let mut reader = io::BufReader::new(fs::File::open(&file)?);
+                    buf.clear();
+                    reader.read_to_end(&mut buf)?;
+                    if let Ok(mut env) = Envelope::from_bytes(buf.as_slice(), Some(file.flags())) {
                         env.set_hash(hash);
                         mailbox_index
                             .lock()
@@ -298,7 +291,11 @@ impl MailBackend for MaildirType {
                             f.set_permissions(permissions)?;
 
                             let writer = io::BufWriter::new(f);
-                            bincode::serialize_into(writer, &env)?;
+                            bincode::Options::serialize_into(
+                                bincode::config::DefaultOptions::new(),
+                                writer,
+                                &env,
+                            )?;
                         }
                         (sender)(
                             account_hash,
@@ -371,6 +368,7 @@ impl MailBackend for MaildirType {
         Ok(Box::pin(async move {
             // Move `watcher` in the closure's scope so that it doesn't get dropped.
             let _watcher = watcher;
+            let mut buf = Vec::with_capacity(4096);
             loop {
                 match rx.recv() {
                     /*
@@ -410,6 +408,7 @@ impl MailBackend for MaildirType {
                                 pathbuf.as_path(),
                                 &cache_dir,
                                 file_name,
+                                &mut buf,
                             ) {
                                 mailbox_index
                                     .lock()
@@ -464,6 +463,7 @@ impl MailBackend for MaildirType {
                                         pathbuf.as_path(),
                                         &cache_dir,
                                         file_name,
+                                        &mut buf,
                                     ) {
                                         mailbox_index
                                             .lock()
@@ -482,14 +482,14 @@ impl MailBackend for MaildirType {
                                 }
                             };
                             let new_hash: EnvelopeHash = get_file_hash(pathbuf.as_path());
+                            let mut reader = io::BufReader::new(fs::File::open(&pathbuf)?);
+                            buf.clear();
+                            reader.read_to_end(&mut buf)?;
                             if index_lock.get_mut(&new_hash).is_none() {
                                 debug!("write notice");
-                                if let Ok(mut env) = Envelope::from_bytes(
-                                    unsafe {
-                                        &Mmap::open_path(&pathbuf, Protection::Read)?.as_slice()
-                                    },
-                                    Some(pathbuf.flags()),
-                                ) {
+                                if let Ok(mut env) =
+                                    Envelope::from_bytes(buf.as_slice(), Some(pathbuf.flags()))
+                                {
                                     env.set_hash(new_hash);
                                     debug!("{}\t{:?}", new_hash, &pathbuf);
                                     debug!(
@@ -543,9 +543,13 @@ impl MailBackend for MaildirType {
                                 });
                                 continue;
                             }
-                            *mailbox_counts[&mailbox_hash].1.lock().unwrap() -= 1;
+                            {
+                                let mut lck = mailbox_counts[&mailbox_hash].1.lock().unwrap();
+                                *lck = lck.saturating_sub(1);
+                            }
                             if !pathbuf.flags().contains(Flag::SEEN) {
-                                *mailbox_counts[&mailbox_hash].0.lock().unwrap() -= 1;
+                                let mut lck = mailbox_counts[&mailbox_hash].0.lock().unwrap();
+                                *lck = lck.saturating_sub(1);
                             }
 
                             index_lock.entry(hash).and_modify(|e| {
@@ -611,6 +615,7 @@ impl MailBackend for MaildirType {
                                         dest.as_path(),
                                         &cache_dir,
                                         file_name,
+                                        &mut buf,
                                     ) {
                                         mailbox_index
                                             .lock()
@@ -701,6 +706,7 @@ impl MailBackend for MaildirType {
                                     dest.as_path(),
                                     &cache_dir,
                                     file_name,
+                                    &mut buf,
                                 ) {
                                     mailbox_index
                                         .lock()
@@ -747,6 +753,7 @@ impl MailBackend for MaildirType {
                                     dest.as_path(),
                                     &cache_dir,
                                     file_name,
+                                    &mut buf,
                                 ) {
                                     mailbox_index
                                         .lock()
@@ -913,6 +920,37 @@ impl MailBackend for MaildirType {
         }))
     }
 
+    fn delete_messages(
+        &mut self,
+        env_hashes: EnvelopeHashBatch,
+        mailbox_hash: MailboxHash,
+    ) -> ResultFuture<()> {
+        let hash_index = self.hash_indexes.clone();
+        Ok(Box::pin(async move {
+            let mut hash_indexes_lck = hash_index.lock().unwrap();
+            let hash_index = hash_indexes_lck.entry(mailbox_hash).or_default();
+
+            for env_hash in env_hashes.iter() {
+                let _path = {
+                    if !hash_index.contains_key(&env_hash) {
+                        continue;
+                    }
+                    if let Some(modif) = &hash_index[&env_hash].modified {
+                        match modif {
+                            PathMod::Path(ref path) => path.clone(),
+                            PathMod::Hash(hash) => hash_index[&hash].to_path_buf(),
+                        }
+                    } else {
+                        hash_index[&env_hash].to_path_buf()
+                    }
+                };
+
+                fs::remove_file(&_path)?;
+            }
+            Ok(())
+        }))
+    }
+
     fn copy_messages(
         &mut self,
         env_hashes: EnvelopeHashBatch,
@@ -967,6 +1005,10 @@ impl MailBackend for MaildirType {
         }))
     }
 
+    fn collection(&self) -> Collection {
+        self.collection.clone()
+    }
+
     fn create_mailbox(
         &mut self,
         new_path: String,
@@ -1200,6 +1242,7 @@ impl MaildirType {
             hash_indexes: Arc::new(Mutex::new(hash_indexes)),
             mailbox_index: Default::default(),
             event_consumer,
+            collection: Default::default(),
             path: root_path,
         }))
     }
@@ -1299,6 +1342,7 @@ fn add_path_to_index(
     path: &Path,
     cache_dir: &xdg::BaseDirectories,
     file_name: PathBuf,
+    buf: &mut Vec<u8>,
 ) -> Result<Envelope> {
     debug!("add_path_to_index path {:?} filename{:?}", path, file_name);
     let env_hash = get_file_hash(path);
@@ -1313,11 +1357,10 @@ fn add_path_to_index(
             map.len()
         );
     }
-    //Mmap::open_path(self.path(), Protection::Read)?
-    let mut env = Envelope::from_bytes(
-        unsafe { &Mmap::open_path(path, Protection::Read)?.as_slice() },
-        Some(path.flags()),
-    )?;
+    let mut reader = io::BufReader::new(fs::File::open(&path)?);
+    buf.clear();
+    reader.read_to_end(buf)?;
+    let mut env = Envelope::from_bytes(buf.as_slice(), Some(path.flags()))?;
     env.set_hash(env_hash);
     debug!(
         "add_path_to_index gen {}\t{}",
@@ -1334,7 +1377,7 @@ fn add_path_to_index(
         permissions.set_mode(0o600); // Read/write for owner only.
         f.set_permissions(permissions)?;
         let writer = io::BufWriter::new(f);
-        bincode::serialize_into(writer, &env)?;
+        bincode::Options::serialize_into(bincode::config::DefaultOptions::new(), writer, &env)?;
     }
     Ok(env)
 }

melib/src/backends/maildir/stream.rs

@@ -25,8 +25,7 @@ use core::future::Future;
 use core::pin::Pin;
 use futures::stream::{FuturesUnordered, StreamExt};
 use futures::task::{Context, Poll};
-use memmap::{Mmap, Protection};
-use std::io::{self};
+use std::io::{self, Read};
 use std::os::unix::fs::PermissionsExt;
 use std::path::PathBuf;
 use std::result;
@@ -51,6 +50,7 @@ impl MaildirStream {
         map: HashIndexes,
         mailbox_index: Arc<Mutex<HashMap<EnvelopeHash, MailboxHash>>>,
     ) -> Result<Pin<Box<dyn Stream<Item = Result<Vec<Envelope>>> + Send + 'static>>> {
+        let chunk_size = 2048;
         path.push("new");
         for d in path.read_dir()? {
             if let Ok(p) = d {
@@ -58,7 +58,6 @@ impl MaildirStream {
             }
         }
         path.pop();
-
         path.push("cur");
         let iter = path.read_dir()?;
         let count = path.read_dir()?.count();
@@ -71,16 +70,9 @@ impl MaildirStream {
             files.push(e);
         }
         let payloads = Box::pin(if !files.is_empty() {
-            let cores = 4_usize;
-            let chunk_size = if count / cores > 0 {
-                count / cores
-            } else {
-                count
-            };
             files
                 .chunks(chunk_size)
                 .map(|chunk| {
-                    //Self::chunk(chunk, name, mailbox_hash, unseen, total, path, root_path, map, mailbox_index)})
                     let cache_dir = xdg::BaseDirectories::with_profile("meli", &name).unwrap();
                     Box::pin(Self::chunk(
                         SmallVec::from(chunk),
@@ -110,79 +102,90 @@ impl MaildirStream {
         map: HashIndexes,
         mailbox_index: Arc<Mutex<HashMap<EnvelopeHash, MailboxHash>>>,
     ) -> Result<Vec<Envelope>> {
-        let len = chunk.len();
-        let size = if len <= 100 { 100 } else { (len / 100) * 100 };
         let mut local_r: Vec<Envelope> = Vec::with_capacity(chunk.len());
-        for c in chunk.chunks(size) {
-            let map = map.clone();
-            for file in c {
-                /* Check if we have a cache file with this email's
-                 * filename */
-                let file_name = PathBuf::from(file)
-                    .strip_prefix(&root_path)
-                    .unwrap()
-                    .to_path_buf();
-                if let Some(cached) = cache_dir.find_cache_file(&file_name) {
-                    /* Cached struct exists, try to load it */
-                    let reader = io::BufReader::new(fs::File::open(&cached).unwrap());
-                    let result: result::Result<Envelope, _> = bincode::deserialize_from(reader);
-                    if let Ok(env) = result {
-                        let mut map = map.lock().unwrap();
-                        let map = map.entry(mailbox_hash).or_default();
-                        let hash = env.hash();
-                        map.insert(hash, file.clone().into());
-                        mailbox_index.lock().unwrap().insert(hash, mailbox_hash);
-                        if !env.is_seen() {
-                            *unseen.lock().unwrap() += 1;
-                        }
-                        *total.lock().unwrap() += 1;
-                        local_r.push(env);
-                        continue;
-                    }
-                };
-                let env_hash = get_file_hash(file);
-                {
+        let mut unseen_total: usize = 0;
+        let mut buf = Vec::with_capacity(4096);
+        for file in chunk {
+            /* Check if we have a cache file with this email's
+             * filename */
+            let file_name = PathBuf::from(&file)
+                .strip_prefix(&root_path)
+                .unwrap()
+                .to_path_buf();
+            if let Some(cached) = cache_dir.find_cache_file(&file_name) {
+                /* Cached struct exists, try to load it */
+                let cached_file = fs::File::open(&cached)?;
+                let filesize = cached_file.metadata()?.len();
+                let reader = io::BufReader::new(cached_file);
+                let result: result::Result<Envelope, _> = bincode::Options::deserialize_from(
+                    bincode::Options::with_limit(
+                        bincode::config::DefaultOptions::new(),
+                        2 * filesize,
+                    ),
+                    reader,
+                );
+                if let Ok(env) = result {
                     let mut map = map.lock().unwrap();
                     let map = map.entry(mailbox_hash).or_default();
-                    (*map).insert(env_hash, PathBuf::from(file).into());
+                    let hash = env.hash();
+                    map.insert(hash, file.clone().into());
+                    mailbox_index.lock().unwrap().insert(hash, mailbox_hash);
+                    if !env.is_seen() {
+                        unseen_total += 1;
+                    }
+                    local_r.push(env);
+                    continue;
                 }
-                match Envelope::from_bytes(
-                    unsafe { &Mmap::open_path(&file, Protection::Read)?.as_slice() },
-                    Some(file.flags()),
-                ) {
-                    Ok(mut env) => {
-                        env.set_hash(env_hash);
-                        mailbox_index.lock().unwrap().insert(env_hash, mailbox_hash);
-                        if let Ok(cached) = cache_dir.place_cache_file(file_name) {
-                            /* place result in cache directory */
-                            let f = fs::File::create(cached)?;
-                            let metadata = f.metadata()?;
-                            let mut permissions = metadata.permissions();
+                /* Try delete invalid file */
+                let _ = fs::remove_file(&cached);
+            };
+            let env_hash = get_file_hash(&file);
+            {
+                let mut map = map.lock().unwrap();
+                let map = map.entry(mailbox_hash).or_default();
+                map.insert(env_hash, PathBuf::from(&file).into());
+            }
+            let mut reader = io::BufReader::new(fs::File::open(&file)?);
+            buf.clear();
+            reader.read_to_end(&mut buf)?;
+            match Envelope::from_bytes(buf.as_slice(), Some(file.flags())) {
+                Ok(mut env) => {
+                    env.set_hash(env_hash);
+                    mailbox_index.lock().unwrap().insert(env_hash, mailbox_hash);
+                    if let Ok(cached) = cache_dir.place_cache_file(file_name) {
+                        /* place result in cache directory */
+                        let f = fs::File::create(cached)?;
+                        let metadata = f.metadata()?;
+                        let mut permissions = metadata.permissions();
 
-                            permissions.set_mode(0o600); // Read/write for owner only.
-                            f.set_permissions(permissions)?;
+                        permissions.set_mode(0o600); // Read/write for owner only.
+                        f.set_permissions(permissions)?;
 
-                            let writer = io::BufWriter::new(f);
-                            bincode::serialize_into(writer, &env)?;
-                        }
-                        if !env.is_seen() {
-                            *unseen.lock().unwrap() += 1;
-                        }
-                        *total.lock().unwrap() += 1;
-                        local_r.push(env);
+                        let writer = io::BufWriter::new(f);
+                        bincode::Options::serialize_into(
+                            bincode::config::DefaultOptions::new(),
+                            writer,
+                            &env,
+                        )?;
                     }
-                    Err(err) => {
-                        debug!(
-                            "DEBUG: hash {}, path: {} couldn't be parsed, {}",
-                            env_hash,
-                            file.as_path().display(),
-                            err,
-                        );
-                        continue;
+                    if !env.is_seen() {
+                        unseen_total += 1;
                     }
+                    local_r.push(env);
+                }
+                Err(err) => {
+                    debug!(
+                        "DEBUG: hash {}, path: {} couldn't be parsed, {}",
+                        env_hash,
+                        file.as_path().display(),
+                        err,
+                    );
+                    continue;
                 }
             }
         }
+        *total.lock().unwrap() += local_r.len();
+        *unseen.lock().unwrap() += unseen_total;
         Ok(local_r)
     }
 }
@@ -190,7 +193,6 @@ impl MaildirStream {
 impl Stream for MaildirStream {
     type Item = Result<Vec<Envelope>>;
     fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        //todo!()
         let payloads = self.payloads.as_mut();
         payloads.poll_next(cx)
     }

melib/src/backends/mbox.rs

@@ -19,18 +19,115 @@
  * along with meli. If not, see <http://www.gnu.org/licenses/>.
  */
 
-/*!
- * https://wiki2.dovecot.org/MailboxFormat/mbox
- */
+//! # Mbox formats
+//!
+//! ## Resources
+//!
+//! - [0] <https://web.archive.org/web/20160812091518/https://jdebp.eu./FGA/mail-mbox-formats.html>
+//! - [1] <https://wiki2.dovecot.org/MailboxFormat/mbox>
+//! - [2] <https://manpages.debian.org/buster/mutt/mbox.5.en.html>
+//!
+//! ## `mbox` format
+//! `mbox` describes a family of incompatible legacy formats.
+//!
+//! "All of the 'mbox' formats store all of the messages in the mailbox in a single file. Delivery appends new messages to the end of the file." [0]
+//!
+//! "Each message is preceded by a From_ line and followed by a blank line. A From_ line is a line that begins with the five characters 'F', 'r', 'o', 'm', and ' '." [0]
+//!
+//! ## `From ` / postmark line
+//!
+//! "An mbox is a text file containing an arbitrary number of e-mail messages. Each message
+//! consists of a postmark, followed by an e-mail message formatted according to RFC822, RFC2822.
+//! The file format is line-oriented. Lines are separated by line feed characters (ASCII 10).
+//!
+//! "A postmark line consists of the four characters 'From', followed by a space character,
+//! followed by the message's envelope sender address, followed by whitespace, and followed by a
+//! time stamp. This line is often called From_ line.
+//!
+//! "The sender address is expected to be addr-spec as defined in RFC2822 3.4.1. The date is expected
+//! to be date-time as output by asctime(3). For compatibility reasons with legacy software,
+//! two-digit years greater than or equal to 70 should be interpreted as the years 1970+, while
+//! two-digit years less than 70 should be interpreted as the years 2000-2069. Software reading
+//! files in this format should also be prepared to accept non-numeric timezone information such as
+//! 'CET DST' for Central European Time, daylight saving time.
+//!
+//! "Example:
+//!
+//!```text
+//!From example@example.com Fri Jun 23 02:56:55 2000
+//!```
+//!
+//! "In order to avoid misinterpretation of lines in message bodies which begin with the four
+//! characters 'From', followed by a space character, the mail delivery agent must quote
+//! any occurrence of 'From ' at the start of a body line." [2]
+//!
+//! ## Metadata
+//!
+//! `melib` recognizes the CClient (a [Pine client API](https://web.archive.org/web/20050203003235/http://www.washington.edu/imap/)) convention for metadata in `mbox` format:
+//!
+//! - `Status`: R (Seen) and O (non-Recent) flags
+//! - `X-Status`: A (Answered), F (Flagged), T (Draft) and D (Deleted) flags
+//! - `X-Keywords`: Message’s keywords
+//!
+//! ## Parsing an mbox file
+//!
+//! ```
+//! # use melib::{Result, Envelope, EnvelopeHash, mbox::*};
+//! # use std::collections::HashMap;
+//! # use std::sync::{Arc, Mutex};
+//! let file_contents = vec![]; // Replace with actual mbox file contents
+//! let index: Arc<Mutex<HashMap<EnvelopeHash, (Offset, Length)>>> = Arc::new(Mutex::new(HashMap::default()));
+//! let mut message_iter = MessageIterator {
+//!     index: index.clone(),
+//!     input: &file_contents.as_slice(),
+//!     offset: 0,
+//!     file_offset: 0,
+//!     format: Some(MboxFormat::MboxCl2),
+//! };
+//! let envelopes: Result<Vec<Envelope>> = message_iter.collect();
+//! ```
+//!
+//! ## Writing / Appending an mbox file
+//!
+//! ```no_run
+//! # use melib::mbox::*;
+//! # use std::io::Write;
+//! let mbox_1: &[u8] = br#"From: <a@b.c>\n\nHello World"#;
+//! let mbox_2: &[u8] = br#"From: <d@e.f>\n\nHello World #2"#;
+//! let mut file = std::io::BufWriter::new(std::fs::File::create(&"out.mbox")?);
+//! let format = MboxFormat::MboxCl2;
+//! format.append(
+//!     &mut file,
+//!     mbox_1,
+//!     None, // Envelope From
+//!     Some(melib::datetime::now()), // Delivered date
+//!     Default::default(), // Flags and tags
+//!     MboxMetadata::None,
+//!     true,
+//!     false,
+//! )?;
+//! format.append(
+//!     &mut file,
+//!     mbox_2,
+//!     None,
+//!     Some(melib::datetime::now()),
+//!     Default::default(), // Flags and tags
+//!     MboxMetadata::None,
+//!     false,
+//!     false,
+//! )?;
+//! file.flush()?;
+//! # Ok::<(), melib::MeliError>(())
+//! ```
 
 use crate::backends::*;
+use crate::collection::Collection;
 use crate::conf::AccountSettings;
 use crate::email::parser::BytesExt;
 use crate::email::*;
 use crate::error::{MeliError, Result};
 use crate::get_path_hash;
 use crate::shellexpand::ShellExpandTrait;
-use memmap::{Mmap, Protection};
 use nom::bytes::complete::tag;
 use nom::character::complete::digit1;
 use nom::combinator::map_res;
@@ -41,22 +138,24 @@ use self::notify::{watcher, DebouncedEvent, RecursiveMode, Watcher};
 use std::collections::hash_map::{DefaultHasher, HashMap};
 use std::fs::File;
 use std::hash::Hasher;
-use std::io::BufReader;
-use std::io::Read;
+use std::io::{BufReader, Read};
 use std::os::unix::io::AsRawFd;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 use std::sync::mpsc::channel;
 use std::sync::{Arc, Mutex, RwLock};
 
-type Offset = usize;
-type Length = usize;
+pub mod write;
 
+pub type Offset = usize;
+pub type Length = usize;
+
+#[cfg(target_os = "linux")]
 const F_OFD_SETLKW: libc::c_int = 38;
 
 // Open file description locking
 // # man fcntl
-fn get_rw_lock_blocking(f: &File) {
+fn get_rw_lock_blocking(f: &File, path: &Path) -> Result<()> {
     let fd: libc::c_int = f.as_raw_fd();
     let mut flock: libc::flock = libc::flock {
         l_type: libc::F_WRLCK as libc::c_short,
@@ -65,11 +164,23 @@ fn get_rw_lock_blocking(f: &File) {
         l_len: 0, /* "Specifying 0 for l_len has the special meaning: lock all bytes starting at the location
                   specified by l_whence and l_start through to the end of file, no matter how large the file grows." */
         l_pid: 0, /* "By contrast with traditional record locks, the l_pid field of that structure must be set to zero when using the commands described below." */
+        #[cfg(target_os = "freebsd")]
+        l_sysid: 0,
     };
     let ptr: *mut libc::flock = &mut flock;
+    #[cfg(not(target_os = "linux"))]
+    let ret_val = unsafe { libc::fcntl(fd, libc::F_SETLKW, ptr as *mut libc::c_void) };
+    #[cfg(target_os = "linux")]
     let ret_val = unsafe { libc::fcntl(fd, F_OFD_SETLKW, ptr as *mut libc::c_void) };
-    debug!(&ret_val);
-    assert!(-1 != ret_val);
+    if ret_val == -1 {
+        let err = nix::errno::Errno::from_i32(nix::errno::errno());
+        return Err(MeliError::new(format!(
+            "Could not lock {}: fcntl() returned {}",
+            path.display(),
+            err.desc()
+        )));
+    }
+    Ok(())
 }
 
 #[derive(Debug)]
@@ -164,7 +275,7 @@ pub struct MboxOp {
     path: PathBuf,
     offset: Offset,
     length: Length,
-    slice: Option<Mmap>,
+    slice: std::cell::RefCell<Option<Vec<u8>>>,
 }
 
 impl MboxOp {
@@ -172,7 +283,7 @@ impl MboxOp {
         MboxOp {
             hash,
             path: path.to_path_buf(),
-            slice: None,
+            slice: std::cell::RefCell::new(None),
             offset,
             length,
         }
@@ -181,28 +292,38 @@ impl MboxOp {
 
 impl BackendOp for MboxOp {
     fn as_bytes(&mut self) -> ResultFuture<Vec<u8>> {
-        if self.slice.is_none() {
-            self.slice = Some(Mmap::open_path(&self.path, Protection::Read)?);
+        if self.slice.get_mut().is_none() {
+            let file = std::fs::OpenOptions::new()
+                .read(true)
+                .write(true)
+                .open(&self.path)?;
+            get_rw_lock_blocking(&file, &self.path)?;
+            let mut buf_reader = BufReader::new(file);
+            let mut contents = Vec::new();
+            buf_reader.read_to_end(&mut contents)?;
+            *self.slice.get_mut() = Some(contents);
         }
-        /* Unwrap is safe since we use ? above. */
-        let ret = Ok((unsafe {
-            &self.slice.as_ref().unwrap().as_slice()[self.offset..self.offset + self.length]
-        })
-        .to_vec());
+        let ret = Ok(self.slice.get_mut().as_ref().unwrap().as_slice()
+            [self.offset..self.offset + self.length]
+            .to_vec());
         Ok(Box::pin(async move { ret }))
     }
 
     fn fetch_flags(&self) -> ResultFuture<Flag> {
         let mut flags = Flag::empty();
-        let file = std::fs::OpenOptions::new()
-            .read(true)
-            .write(true)
-            .open(&self.path)?;
-        get_rw_lock_blocking(&file);
-        let mut buf_reader = BufReader::new(file);
-        let mut contents = Vec::new();
-        buf_reader.read_to_end(&mut contents)?;
-        let (_, headers) = parser::headers::headers_raw(contents.as_slice())?;
+        if self.slice.borrow().is_none() {
+            let file = std::fs::OpenOptions::new()
+                .read(true)
+                .write(true)
+                .open(&self.path)?;
+            get_rw_lock_blocking(&file, &self.path)?;
+            let mut buf_reader = BufReader::new(file);
+            let mut contents = Vec::new();
+            buf_reader.read_to_end(&mut contents)?;
+            *self.slice.borrow_mut() = Some(contents);
+        }
+        let slice_ref = self.slice.borrow();
+        let (_, headers) = parser::headers::headers_raw(slice_ref.as_ref().unwrap().as_slice())?;
         if let Some(start) = headers.find(b"Status:") {
             if let Some(end) = headers[start..].find(b"\n") {
                 let start = start + b"Status:".len();
@@ -250,14 +371,30 @@ impl BackendOp for MboxOp {
 }
 
 #[derive(Debug, Clone, Copy)]
-pub enum MboxReader {
+pub enum MboxMetadata {
+    /// Dovecot uses C-Client (ie. UW-IMAP, Pine) compatible headers in mbox messages to store me
+    /// - X-IMAPbase: Contains UIDVALIDITY, last used UID and list of used keywords
+    /// - X-IMAP: Same as X-IMAPbase but also specifies that the message is a “pseudo message”
+    /// - X-UID: Message’s allocated UID
+    /// - Status: R (Seen) and O (non-Recent) flags
+    /// - X-Status: A (Answered), F (Flagged), T (Draft) and D (Deleted) flags
+    /// - X-Keywords: Message’s keywords
+    /// - Content-Length: Length of the message body in bytes
+    CClient,
+    None,
+}
+
+/// Choose between "mboxo", "mboxrd", "mboxcl", "mboxcl2". For new mailboxes, prefer "mboxcl2"
+/// which does not alter the mail body.
+#[derive(Debug, Clone, Copy)]
+pub enum MboxFormat {
     MboxO,
     MboxRd,
     MboxCl,
     MboxCl2,
 }
 
-impl Default for MboxReader {
+impl Default for MboxFormat {
     fn default() -> Self {
         Self::MboxCl2
     }
@@ -300,8 +437,8 @@ macro_rules! find_From__line {
     }};
 }
 
-impl MboxReader {
-    fn parse<'i>(&self, input: &'i [u8]) -> IResult<&'i [u8], Envelope> {
+impl MboxFormat {
+    pub fn parse<'i>(&self, input: &'i [u8]) -> IResult<&'i [u8], Envelope> {
         let orig_input = input;
         let mut input = input;
         match self {
@@ -584,7 +721,7 @@ pub fn mbox_parse(
     index: Arc<Mutex<HashMap<EnvelopeHash, (Offset, Length)>>>,
     input: &[u8],
     file_offset: usize,
-    reader: Option<MboxReader>,
+    format: Option<MboxFormat>,
 ) -> IResult<&[u8], Vec<Envelope>> {
     if input.is_empty() {
         return Err(nom::Err::Error((input, ErrorKind::Tag)));
@@ -593,9 +730,9 @@ pub fn mbox_parse(
     let mut index = index.lock().unwrap();
     let mut envelopes = Vec::with_capacity(32);
 
-    let reader = reader.unwrap_or(MboxReader::MboxCl2);
+    let format = format.unwrap_or(MboxFormat::MboxCl2);
     while !input[offset + file_offset..].is_empty() {
-        let (next_input, env) = match reader.parse(&input[offset + file_offset..]) {
+        let (next_input, env) = match format.parse(&input[offset + file_offset..]) {
             Ok(v) => v,
             Err(e) => {
                 // Try to recover from this error by finding a new candidate From_ line
@@ -627,12 +764,12 @@ pub fn mbox_parse(
     Ok((&[], envelopes))
 }
 
-struct MessageIterator<'a> {
-    index: Arc<Mutex<HashMap<EnvelopeHash, (Offset, Length)>>>,
-    input: &'a [u8],
-    file_offset: usize,
-    offset: usize,
-    reader: Option<MboxReader>,
+pub struct MessageIterator<'a> {
+    pub index: Arc<Mutex<HashMap<EnvelopeHash, (Offset, Length)>>>,
+    pub input: &'a [u8],
+    pub file_offset: usize,
+    pub offset: usize,
+    pub format: Option<MboxFormat>,
 }
 
 impl<'a> Iterator for MessageIterator<'a> {
@@ -643,10 +780,10 @@ impl<'a> Iterator for MessageIterator<'a> {
         }
         let mut index = self.index.lock().unwrap();
 
-        let reader = self.reader.unwrap_or(MboxReader::MboxCl2);
+        let format = self.format.unwrap_or(MboxFormat::MboxCl2);
         while !self.input[self.offset + self.file_offset..].is_empty() {
             let (next_input, env) =
-                match reader.parse(&self.input[self.offset + self.file_offset..]) {
+                match format.parse(&self.input[self.offset + self.file_offset..]) {
                     Ok(v) => v,
                     Err(e) => {
                         // Try to recover from this error by finding a new candidate From_ line
@@ -687,9 +824,10 @@ impl<'a> Iterator for MessageIterator<'a> {
 pub struct MboxType {
     account_name: String,
     path: PathBuf,
+    collection: Collection,
     mailbox_index: Arc<Mutex<HashMap<EnvelopeHash, MailboxHash>>>,
     mailboxes: Arc<Mutex<HashMap<MailboxHash, MboxMailbox>>>,
-    prefer_mbox_type: Option<MboxReader>,
+    prefer_mbox_type: Option<MboxFormat>,
     event_consumer: BackendEventConsumer,
 }
 
@@ -718,7 +856,7 @@ impl MailBackend for MboxType {
             mailbox_hash: MailboxHash,
             mailbox_index: Arc<Mutex<HashMap<EnvelopeHash, MailboxHash>>>,
             mailboxes: Arc<Mutex<HashMap<MailboxHash, MboxMailbox>>>,
-            prefer_mbox_type: Option<MboxReader>,
+            prefer_mbox_type: Option<MboxFormat>,
             offset: usize,
             file_offset: usize,
             contents: Vec<u8>,
@@ -733,7 +871,7 @@ impl MailBackend for MboxType {
                     input: &self.contents.as_slice(),
                     offset: self.offset,
                     file_offset: self.file_offset,
-                    reader: self.prefer_mbox_type,
+                    format: self.prefer_mbox_type,
                 };
                 let mut payload = vec![];
                 let mut done = false;
@@ -782,7 +920,7 @@ impl MailBackend for MboxType {
             .read(true)
             .write(true)
             .open(&mailbox_path)?;
-        get_rw_lock_blocking(&file);
+        get_rw_lock_blocking(&file, &mailbox_path)?;
         let mut buf_reader = BufReader::new(file);
         let mut contents = Vec::new();
         buf_reader.read_to_end(&mut contents)?;
@@ -860,7 +998,7 @@ impl MailBackend for MboxType {
                                     continue;
                                 }
                             };
-                            get_rw_lock_blocking(&file);
+                            get_rw_lock_blocking(&file, &pathbuf)?;
                             let mut mailbox_lock = mailboxes.lock().unwrap();
                             let mut buf_reader = BufReader::new(file);
                             let mut contents = Vec::new();
@@ -1012,6 +1150,14 @@ impl MailBackend for MboxType {
         Err(MeliError::new("Unimplemented."))
     }
 
+    fn delete_messages(
+        &mut self,
+        _env_hashes: EnvelopeHashBatch,
+        _mailbox_hash: MailboxHash,
+    ) -> ResultFuture<()> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
     fn save(
         &self,
         _bytes: Vec<u8>,
@@ -1028,6 +1174,10 @@ impl MailBackend for MboxType {
     fn as_any_mut(&mut self) -> &mut dyn Any {
         self
     }
+
+    fn collection(&self) -> Collection {
+        self.collection.clone()
+    }
 }
 
 macro_rules! get_conf_val {
@@ -1079,10 +1229,10 @@ impl MboxType {
             path,
             prefer_mbox_type: match prefer_mbox_type.as_str() {
                 "auto" => None,
-                "mboxo" => Some(MboxReader::MboxO),
-                "mboxrd" => Some(MboxReader::MboxRd),
-                "mboxcl" => Some(MboxReader::MboxCl),
-                "mboxcl2" => Some(MboxReader::MboxCl2),
+                "mboxo" => Some(MboxFormat::MboxO),
+                "mboxrd" => Some(MboxFormat::MboxRd),
+                "mboxcl" => Some(MboxFormat::MboxCl),
+                "mboxcl2" => Some(MboxFormat::MboxCl2),
                 _ => {
                     return Err(MeliError::new(format!(
                         "{} invalid `prefer_mbox_type` value: `{}`",
@@ -1091,6 +1241,7 @@ impl MboxType {
                     )))
                 }
             },
+            collection: Collection::default(),
             mailbox_index: Default::default(),
             mailboxes: Default::default(),
         };

melib/src/backends/mbox/write.rs

@@ -0,0 +1,260 @@
+/*
+ * meli - mailbox module.
+ *
+ * Copyright 2021  Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+
+impl MboxFormat {
+    pub fn append(
+        &self,
+        writer: &mut dyn std::io::Write,
+        input: &[u8],
+        envelope_from: Option<&Address>,
+        delivery_date: Option<crate::UnixTimestamp>,
+        (flags, tags): (Flag, Vec<&str>),
+        metadata_format: MboxMetadata,
+        is_empty: bool,
+        crlf: bool,
+    ) -> Result<()> {
+        if tags.iter().any(|t| t.contains(' ')) {
+            return Err(MeliError::new("mbox tags/keywords can't contain spaces"));
+        }
+        let line_ending: &'static [u8] = if crlf { &b"\r\n"[..] } else { &b"\n"[..] };
+        if !is_empty {
+            writer.write_all(line_ending)?;
+            writer.write_all(line_ending)?;
+        }
+        writer.write_all(&b"From "[..])?;
+        if let Some(from) = envelope_from {
+            writer.write_all(from.address_spec_raw())?;
+        } else {
+            writer.write_all(&b"MAILER-DAEMON"[..])?;
+        }
+        writer.write_all(&b" "[..])?;
+        writer.write_all(
+            crate::datetime::timestamp_to_string(
+                delivery_date.unwrap_or_else(|| crate::datetime::now()),
+                Some(crate::datetime::ASCTIME_FMT),
+                true,
+            )
+            .trim()
+            .as_bytes(),
+        )?;
+        writer.write_all(line_ending)?;
+        let (mut headers, body) = parser::mail(input)?;
+        headers.retain(|(header_name, _)| {
+            !header_name.eq_ignore_ascii_case(b"Status")
+                && !header_name.eq_ignore_ascii_case(b"X-Status")
+                && !header_name.eq_ignore_ascii_case(b"X-Keywords")
+                && !header_name.eq_ignore_ascii_case(b"Content-Length")
+        });
+        let write_header_val_fn = |writer: &mut dyn std::io::Write, bytes: &[u8]| {
+            let mut i = 0;
+            if crlf {
+                while i < bytes.len() {
+                    if bytes[i..].starts_with(b"\r\n") {
+                        writer.write_all(&[b'\r', b'\n'])?;
+                        i += 2;
+                        continue;
+                    } else if bytes[i] == b'\n' {
+                        writer.write_all(&[b'\r', b'\n'])?;
+                    } else {
+                        writer.write_all(&[bytes[i]])?;
+                    }
+                    i += 1;
+                }
+            } else {
+                while i < bytes.len() {
+                    if bytes[i..].starts_with(b"\r\n") {
+                        writer.write_all(&[b'\n'])?;
+                        i += 2;
+                    } else {
+                        writer.write_all(&[bytes[i]])?;
+                        i += 1;
+                    }
+                }
+            }
+            Ok::<(), MeliError>(())
+        };
+        let write_metadata_fn = |writer: &mut dyn std::io::Write| match metadata_format {
+            MboxMetadata::CClient => {
+                for (h, v) in {
+                    if flags.is_seen() {
+                        Some((&b"Status"[..], "R".into()))
+                    } else {
+                        None
+                    }
+                    .into_iter()
+                    .chain(
+                        if !flags.is_flagged()
+                            && !flags.is_replied()
+                            && !flags.is_draft()
+                            && !flags.is_trashed()
+                        {
+                            None
+                        } else {
+                            Some((
+                                &b"X-Status"[..],
+                                format!(
+                                    "{flagged}{replied}{draft}{trashed}",
+                                    flagged = if flags.is_flagged() { "F" } else { "" },
+                                    replied = if flags.is_replied() { "A" } else { "" },
+                                    draft = if flags.is_draft() { "T" } else { "" },
+                                    trashed = if flags.is_trashed() { "D" } else { "" }
+                                ),
+                            ))
+                        },
+                    )
+                    .chain(if tags.is_empty() {
+                        None
+                    } else {
+                        Some((&b"X-Keywords"[..], tags.as_slice().join(" ")))
+                    })
+                } {
+                    writer.write_all(h)?;
+                    writer.write_all(&b": "[..])?;
+                    writer.write_all(v.as_bytes())?;
+                    writer.write_all(line_ending)?;
+                }
+                Ok::<(), MeliError>(())
+            }
+            MboxMetadata::None => Ok(()),
+        };
+
+        let body_len = {
+            let mut len = body.len();
+            if crlf {
+                let stray_lfs = body.iter().filter(|b| **b == b'\n').count()
+                    - body.windows(b"\r\n".len()).filter(|w| w == b"\r\n").count();
+                len += stray_lfs;
+            } else {
+                let crlfs = body.windows(b"\r\n".len()).filter(|w| w == b"\r\n").count();
+                len -= crlfs;
+            }
+            len
+        };
+
+        match self {
+            MboxFormat::MboxO | MboxFormat::MboxRd => Err(MeliError::new("Unimplemented.")),
+            MboxFormat::MboxCl => {
+                let len = (body_len
+                    + body
+                        .windows(b"\nFrom ".len())
+                        .filter(|w| w == b"\nFrom ")
+                        .count()
+                    + if body.starts_with(b"From ") { 1 } else { 0 })
+                .to_string();
+                for (h, v) in headers
+                    .into_iter()
+                    .chain(Some((&b"Content-Length"[..], len.as_bytes())))
+                {
+                    writer.write_all(h)?;
+                    writer.write_all(&b": "[..])?;
+                    write_header_val_fn(writer, v)?;
+                    writer.write_all(line_ending)?;
+                }
+                write_metadata_fn(writer)?;
+                writer.write_all(line_ending)?;
+
+                if body.starts_with(b"From ") {
+                    writer.write_all(&[b'>'])?;
+                }
+                let mut i = 0;
+                if crlf {
+                    while i < body.len() {
+                        if body[i..].starts_with(b"\r\n") {
+                            writer.write_all(&[b'\r', b'\n'])?;
+                            if body[i..].starts_with(b"\r\nFrom ") {
+                                writer.write_all(&[b'>'])?;
+                            }
+                            i += 2;
+                        } else if body[i] == b'\n' {
+                            writer.write_all(&[b'\r', b'\n'])?;
+                            if body[i..].starts_with(b"\nFrom ") {
+                                writer.write_all(&[b'>'])?;
+                            }
+                            i += 1;
+                        } else {
+                            writer.write_all(&[body[i]])?;
+                            i += 1;
+                        }
+                    }
+                } else {
+                    while i < body.len() {
+                        if body[i..].starts_with(b"\r\n") {
+                            writer.write_all(&[b'\n'])?;
+                            if body[i..].starts_with(b"\r\nFrom ") {
+                                writer.write_all(&[b'>'])?;
+                            }
+                            i += 2;
+                        } else {
+                            writer.write_all(&[body[i]])?;
+                            if body[i..].starts_with(b"\nFrom ") {
+                                writer.write_all(&[b'>'])?;
+                            }
+                            i += 1;
+                        }
+                    }
+                }
+                Ok(())
+            }
+            MboxFormat::MboxCl2 => {
+                let len = body_len.to_string();
+                for (h, v) in headers
+                    .into_iter()
+                    .chain(Some((&b"Content-Length"[..], len.as_bytes())))
+                {
+                    writer.write_all(h)?;
+                    writer.write_all(&b": "[..])?;
+                    write_header_val_fn(writer, v)?;
+                    writer.write_all(line_ending)?;
+                }
+                write_metadata_fn(writer)?;
+                writer.write_all(line_ending)?;
+                let mut i = 0;
+                if crlf {
+                    while i < body.len() {
+                        if body[i..].starts_with(b"\r\n") {
+                            writer.write_all(&[b'\r', b'\n'])?;
+                            i += 2;
+                            continue;
+                        } else if body[i] == b'\n' {
+                            writer.write_all(&[b'\r', b'\n'])?;
+                        } else {
+                            writer.write_all(&[body[i]])?;
+                        }
+                        i += 1;
+                    }
+                } else {
+                    while i < body.len() {
+                        if body[i..].starts_with(b"\r\n") {
+                            writer.write_all(&[b'\n'])?;
+                            i += 2;
+                        } else {
+                            writer.write_all(&[body[i]])?;
+                            i += 1;
+                        }
+                    }
+                }
+                Ok(())
+            }
+        }
+    }
+}

melib/src/backends/nntp.rs

@@ -32,19 +32,19 @@ pub use operations::*;
 mod connection;
 pub use connection::*;
 
-use crate::backends::*;
 use crate::conf::AccountSettings;
+use crate::connections::timeout;
 use crate::email::*;
 use crate::error::{MeliError, Result, ResultIntoMeliError};
+use crate::{backends::*, Collection};
 use futures::lock::Mutex as FutureMutex;
 use futures::stream::Stream;
-use std::collections::{hash_map::DefaultHasher, BTreeMap, BTreeSet, HashMap, HashSet};
-use std::future::Future;
+use std::collections::{hash_map::DefaultHasher, BTreeSet, HashMap, HashSet};
 use std::hash::Hasher;
 use std::pin::Pin;
 use std::str::FromStr;
-use std::sync::{Arc, Mutex, RwLock};
-use std::time::Instant;
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
 pub type UID = usize;
 
 pub static SUPPORTED_CAPABILITIES: &[&str] = &[
@@ -66,20 +66,6 @@ pub struct NntpServerConf {
     pub extension_use: NntpExtensionUse,
 }
 
-pub struct IsSubscribedFn(Box<dyn Fn(&str) -> bool + Send + Sync>);
-
-impl std::fmt::Debug for IsSubscribedFn {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(f, "IsSubscribedFn Box")
-    }
-}
-
-impl std::ops::Deref for IsSubscribedFn {
-    type Target = Box<dyn Fn(&str) -> bool + Send + Sync>;
-    fn deref(&self) -> &Box<dyn Fn(&str) -> bool + Send + Sync> {
-        &self.0
-    }
-}
 type Capabilities = HashSet<String>;
 
 #[derive(Debug)]
@@ -91,6 +77,7 @@ pub struct UIDStore {
     hash_index: Arc<Mutex<HashMap<EnvelopeHash, (UID, MailboxHash)>>>,
     uid_index: Arc<Mutex<HashMap<(MailboxHash, UID), EnvelopeHash>>>,
 
+    collection: Collection,
     mailboxes: Arc<FutureMutex<HashMap<MailboxHash, NntpMailbox>>>,
     is_online: Arc<Mutex<(Instant, Result<()>)>>,
     event_consumer: BackendEventConsumer,
@@ -111,6 +98,7 @@ impl UIDStore {
             hash_index: Default::default(),
             uid_index: Default::default(),
             mailboxes: Arc::new(FutureMutex::new(Default::default())),
+            collection: Collection::new(),
             is_online: Arc::new(Mutex::new((
                 Instant::now(),
                 Err(MeliError::new("Account is uninitialised.")),
@@ -234,10 +222,11 @@ impl MailBackend for NntpType {
     fn is_online(&self) -> ResultFuture<()> {
         let connection = self.connection.clone();
         Ok(Box::pin(async move {
-            match timeout(std::time::Duration::from_secs(3), connection.lock()).await {
+            match timeout(Some(Duration::from_secs(60 * 16)), connection.lock()).await {
                 Ok(mut conn) => {
                     debug!("is_online");
-                    match debug!(timeout(std::time::Duration::from_secs(3), conn.connect()).await) {
+                    match debug!(timeout(Some(Duration::from_secs(60 * 16)), conn.connect()).await)
+                    {
                         Ok(Ok(())) => Ok(()),
                         Err(err) | Ok(Err(err)) => {
                             conn.stream = Err(err.clone());
@@ -278,7 +267,7 @@ impl MailBackend for NntpType {
         _mailbox_hash: MailboxHash,
         _flags: Option<Flag>,
     ) -> ResultFuture<()> {
-        Err(MeliError::new("Unimplemented."))
+        Err(MeliError::new("NNTP doesn't support saving."))
     }
 
     fn copy_messages(
@@ -288,7 +277,7 @@ impl MailBackend for NntpType {
         _destination_mailbox_hash: MailboxHash,
         _move_: bool,
     ) -> ResultFuture<()> {
-        Err(MeliError::new("Unimplemented."))
+        Err(MeliError::new("NNTP doesn't support copying/moving."))
     }
 
     fn set_flags(
@@ -297,11 +286,15 @@ impl MailBackend for NntpType {
         _mailbox_hash: MailboxHash,
         _flags: SmallVec<[(std::result::Result<Flag, String>, bool); 8]>,
     ) -> ResultFuture<()> {
-        Err(MeliError::new("Unimplemented."))
+        Err(MeliError::new("NNTP doesn't support flags."))
     }
 
-    fn tags(&self) -> Option<Arc<RwLock<BTreeMap<u64, String>>>> {
-        None
+    fn delete_messages(
+        &mut self,
+        _env_hashes: EnvelopeHashBatch,
+        _mailbox_hash: MailboxHash,
+    ) -> ResultFuture<()> {
+        Err(MeliError::new("NNTP doesn't support deletion."))
     }
 
     fn as_any(&self) -> &dyn Any {
@@ -312,6 +305,10 @@ impl MailBackend for NntpType {
         self
     }
 
+    fn collection(&self) -> Collection {
+        self.uid_store.collection.clone()
+    }
+
     fn create_mailbox(
         &mut self,
         _path: String,
@@ -634,15 +631,3 @@ impl FetchState {
         Ok(Some(ret))
     }
 }
-
-use futures::future::{self, Either};
-
-async fn timeout<O>(dur: std::time::Duration, f: impl Future<Output = O>) -> Result<O> {
-    futures::pin_mut!(f);
-    match future::select(f, smol::Timer::after(dur)).await {
-        Either::Left((out, _)) => Ok(out),
-        Either::Right(_) => {
-            Err(MeliError::new("Timedout").set_kind(crate::error::ErrorKind::Network))
-        }
-    }
-}

melib/src/backends/nntp/connection.rs

@@ -90,7 +90,7 @@ impl NntpStream {
         let stream = {
             let addr = lookup_ipv4(path, server_conf.server_port)?;
             AsyncWrapper::new(Connection::Tcp(
-                TcpStream::connect_timeout(&addr, std::time::Duration::new(4, 0))
+                TcpStream::connect_timeout(&addr, std::time::Duration::new(16, 0))
                     .chain_err_kind(crate::error::ErrorKind::Network)?,
             ))
             .chain_err_kind(crate::error::ErrorKind::Network)?
@@ -130,7 +130,7 @@ impl NntpStream {
                     .any(|cap| cap.eq_ignore_ascii_case("VERSION 2"))
                 {
                     return Err(MeliError::new(format!(
-                        "Could not connect to {}: server is not NNTP compliant",
+                        "Could not connect to {}: server is not NNTP VERSION 2 compliant",
                         &server_conf.server_hostname
                     )));
                 }
@@ -190,8 +190,12 @@ impl NntpStream {
                 ret.stream = AsyncWrapper::new(Connection::Tls(
                     conn_result.chain_err_kind(crate::error::ErrorKind::Network)?,
                 ))
+                .chain_err_summary(|| format!("Could not initiate TLS negotiation to {}.", path))
                 .chain_err_kind(crate::error::ErrorKind::Network)?;
             }
+        } else {
+            ret.read_response(&mut res, false, &["200 ", "201 "])
+                .await?;
         }
         //ret.send_command(
         //    format!(
@@ -201,9 +205,17 @@ impl NntpStream {
         //    .as_bytes(),
         //)
         //.await?;
+        if let Err(err) = ret
+            .stream
+            .get_ref()
+            .set_keepalive(Some(std::time::Duration::new(60 * 9, 0)))
+        {
+            crate::log(
+                format!("Could not set TCP keepalive in NNTP connection: {}", err),
+                crate::LoggingLevel::WARN,
+            );
+        }
 
-        ret.read_response(&mut res, false, &["200 ", "201 "])
-            .await?;
         ret.send_command(b"CAPABILITIES").await?;
         ret.read_response(&mut res, true, command_to_replycodes("CAPABILITIES"))
             .await?;

melib/src/backends/nntp/mailbox.rs

@@ -18,9 +18,8 @@
  * You should have received a copy of the GNU General Public License
  * along with meli. If not, see <http://www.gnu.org/licenses/>.
  */
-use crate::backends::imap::LazyCountSet;
 use crate::backends::{
-    BackendMailbox, Mailbox, MailboxHash, MailboxPermissions, SpecialUsageMailbox,
+    BackendMailbox, LazyCountSet, Mailbox, MailboxHash, MailboxPermissions, SpecialUsageMailbox,
 };
 use crate::error::*;
 use std::sync::{Arc, Mutex};

melib/src/backends/nntp/protocol_parser.rs

@@ -144,15 +144,6 @@ pub fn over_article(input: &str) -> IResult<&str, (UID, Envelope)> {
             }
 
             if let Some(references) = references {
-                {
-                    if let Ok((_, r)) =
-                        crate::email::parser::address::msg_id_list(references.as_bytes())
-                    {
-                        for v in r {
-                            env.push_references(v);
-                        }
-                    }
-                }
                 env.set_references(references.as_bytes());
             }
 

melib/src/backends/notmuch.rs

@@ -19,11 +19,11 @@
  * along with meli. If not, see <http://www.gnu.org/licenses/>.
  */
 
-use crate::backends::*;
 use crate::conf::AccountSettings;
 use crate::email::{Envelope, EnvelopeHash, Flag};
-use crate::error::{MeliError, Result, ResultIntoMeliError};
+use crate::error::{MeliError, Result};
 use crate::shellexpand::ShellExpandTrait;
+use crate::{backends::*, Collection};
 use smallvec::SmallVec;
 use std::collections::{
     hash_map::{DefaultHasher, HashMap},
@@ -101,7 +101,7 @@ impl DbConnection {
             *self.revision_uuid.read().unwrap(),
             new_revision_uuid
         );
-        let query: Query = Query::new(self.lib.clone(), &self, &query_str)?;
+        let query: Query = Query::new(&self, &query_str)?;
         let iter = query.search()?;
         let mailbox_index_lck = mailbox_index.write().unwrap();
         let mailboxes_lck = mailboxes.read().unwrap();
@@ -130,31 +130,25 @@ impl DbConnection {
                 }
             } else {
                 let message_id = message.msg_id_cstr().to_string_lossy().to_string();
-                match message.into_envelope(index.clone(), tag_index.clone()) {
-                    Ok(env) => {
-                        for (&mailbox_hash, m) in mailboxes_lck.iter() {
-                            let query_str = format!("{} id:{}", m.query_str.as_str(), &message_id);
-                            let query: Query = Query::new(self.lib.clone(), self, &query_str)?;
-                            if query.count().unwrap_or(0) > 0 {
-                                let mut total_lck = m.total.lock().unwrap();
-                                let mut unseen_lck = m.unseen.lock().unwrap();
-                                *total_lck += 1;
-                                if !env.is_seen() {
-                                    *unseen_lck += 1;
-                                }
-                                (event_consumer)(
-                                    account_hash,
-                                    BackendEvent::Refresh(RefreshEvent {
-                                        account_hash,
-                                        mailbox_hash,
-                                        kind: Create(Box::new(env.clone())),
-                                    }),
-                                );
-                            }
+                let env = message.into_envelope(&index, &tag_index);
+                for (&mailbox_hash, m) in mailboxes_lck.iter() {
+                    let query_str = format!("{} id:{}", m.query_str.as_str(), &message_id);
+                    let query: Query = Query::new(self, &query_str)?;
+                    if query.count().unwrap_or(0) > 0 {
+                        let mut total_lck = m.total.lock().unwrap();
+                        let mut unseen_lck = m.unseen.lock().unwrap();
+                        *total_lck += 1;
+                        if !env.is_seen() {
+                            *unseen_lck += 1;
                         }
-                    }
-                    Err(err) => {
-                        debug!("could not parse message {:?}", err);
+                        (event_consumer)(
+                            account_hash,
+                            BackendEvent::Refresh(RefreshEvent {
+                                account_hash,
+                                mailbox_hash,
+                                kind: Create(Box::new(env.clone())),
+                            }),
+                        );
                     }
                 }
             }
@@ -226,9 +220,10 @@ pub struct NotmuchDb {
     mailboxes: Arc<RwLock<HashMap<MailboxHash, NotmuchMailbox>>>,
     index: Arc<RwLock<HashMap<EnvelopeHash, CString>>>,
     mailbox_index: Arc<RwLock<HashMap<EnvelopeHash, SmallVec<[MailboxHash; 16]>>>>,
-    tag_index: Arc<RwLock<BTreeMap<u64, String>>>,
+    collection: Collection,
     path: PathBuf,
     account_name: Arc<String>,
+    account_hash: AccountHash,
     event_consumer: BackendEventConsumer,
     save_messages_to: Option<PathBuf>,
 }
@@ -352,17 +347,23 @@ impl NotmuchDb {
             }
         }
 
+        let account_hash = {
+            let mut hasher = DefaultHasher::new();
+            hasher.write(s.name().as_bytes());
+            hasher.finish()
+        };
         Ok(Box::new(NotmuchDb {
             lib,
             revision_uuid: Arc::new(RwLock::new(0)),
             path,
             index: Arc::new(RwLock::new(Default::default())),
             mailbox_index: Arc::new(RwLock::new(Default::default())),
-            tag_index: Arc::new(RwLock::new(Default::default())),
+            collection: Collection::default(),
 
             mailboxes: Arc::new(RwLock::new(mailboxes)),
             save_messages_to: None,
             account_name: Arc::new(s.name().to_string()),
+            account_hash,
             event_consumer,
         }))
     }
@@ -474,21 +475,15 @@ impl MailBackend for NotmuchDb {
                             } else {
                                 continue;
                             };
-                        match message.into_envelope(self.index.clone(), self.tag_index.clone()) {
-                            Ok(env) => {
-                                mailbox_index_lck
-                                    .entry(env.hash())
-                                    .or_default()
-                                    .push(self.mailbox_hash);
-                                if !env.is_seen() {
-                                    unseen_count += 1;
-                                }
-                                ret.push(env);
-                            }
-                            Err(err) => {
-                                debug!("could not parse message {:?}", err);
-                            }
+                        let env = message.into_envelope(&self.index, &self.tag_index);
+                        mailbox_index_lck
+                            .entry(env.hash())
+                            .or_default()
+                            .push(self.mailbox_hash);
+                        if !env.is_seen() {
+                            unseen_count += 1;
                         }
+                        ret.push(env);
                     } else {
                         done = true;
                         break;
@@ -515,13 +510,13 @@ impl MailBackend for NotmuchDb {
         )?);
         let index = self.index.clone();
         let mailbox_index = self.mailbox_index.clone();
-        let tag_index = self.tag_index.clone();
+        let tag_index = self.collection.tag_index.clone();
         let mailboxes = self.mailboxes.clone();
         let v: Vec<CString>;
         {
             let mailboxes_lck = mailboxes.read().unwrap();
             let mailbox = mailboxes_lck.get(&mailbox_hash).unwrap();
-            let query: Query = Query::new(self.lib.clone(), &database, mailbox.query_str.as_str())?;
+            let query: Query = Query::new(&database, mailbox.query_str.as_str())?;
             {
                 let mut total_lck = mailbox.total.lock().unwrap();
                 let mut unseen_lck = mailbox.unseen.lock().unwrap();
@@ -556,11 +551,7 @@ impl MailBackend for NotmuchDb {
     }
 
     fn refresh(&mut self, _mailbox_hash: MailboxHash) -> ResultFuture<()> {
-        let account_hash = {
-            let mut hasher = DefaultHasher::new();
-            hasher.write(self.account_name.as_bytes());
-            hasher.finish()
-        };
+        let account_hash = self.account_hash;
         let mut database = NotmuchDb::new_connection(
             self.path.as_path(),
             self.revision_uuid.clone(),
@@ -570,7 +561,7 @@ impl MailBackend for NotmuchDb {
         let mailboxes = self.mailboxes.clone();
         let index = self.index.clone();
         let mailbox_index = self.mailbox_index.clone();
-        let tag_index = self.tag_index.clone();
+        let tag_index = self.collection.tag_index.clone();
         let event_consumer = self.event_consumer.clone();
         Ok(Box::pin(async move {
             let new_revision_uuid = database.get_revision_uuid();
@@ -594,18 +585,14 @@ impl MailBackend for NotmuchDb {
         extern crate notify;
         use notify::{watcher, RecursiveMode, Watcher};
 
-        let account_hash = {
-            let mut hasher = DefaultHasher::new();
-            hasher.write(self.account_name.as_bytes());
-            hasher.finish()
-        };
+        let account_hash = self.account_hash;
+        let collection = self.collection.clone();
         let lib = self.lib.clone();
         let path = self.path.clone();
         let revision_uuid = self.revision_uuid.clone();
         let mailboxes = self.mailboxes.clone();
         let index = self.index.clone();
         let mailbox_index = self.mailbox_index.clone();
-        let tag_index = self.tag_index.clone();
         let event_consumer = self.event_consumer.clone();
 
         let (tx, rx) = std::sync::mpsc::channel();
@@ -629,7 +616,7 @@ impl MailBackend for NotmuchDb {
                             mailboxes.clone(),
                             index.clone(),
                             mailbox_index.clone(),
-                            tag_index.clone(),
+                            collection.tag_index.clone(),
                             account_hash.clone(),
                             event_consumer.clone(),
                             new_revision_uuid,
@@ -664,7 +651,7 @@ impl MailBackend for NotmuchDb {
             hash,
             index: self.index.clone(),
             bytes: None,
-            tag_index: self.tag_index.clone(),
+            collection: self.collection.clone(),
         }))
     }
 
@@ -706,7 +693,7 @@ impl MailBackend for NotmuchDb {
             self.lib.clone(),
             true,
         )?;
-        let tag_index = self.tag_index.clone();
+        let collection = self.collection.clone();
         let index = self.index.clone();
 
         Ok(Box::pin(async move {
@@ -794,7 +781,11 @@ impl MailBackend for NotmuchDb {
             for (f, v) in flags.iter() {
                 if let (Err(tag), true) = (f, v) {
                     let hash = tag_hash!(tag);
-                    tag_index.write().unwrap().insert(hash, tag.to_string());
+                    collection
+                        .tag_index
+                        .write()
+                        .unwrap()
+                        .insert(hash, tag.to_string());
                 }
             }
 
@@ -802,6 +793,14 @@ impl MailBackend for NotmuchDb {
         }))
     }
 
+    fn delete_messages(
+        &mut self,
+        _env_hashes: EnvelopeHashBatch,
+        _mailbox_hash: MailboxHash,
+    ) -> ResultFuture<()> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
     fn search(
         &self,
         melib_query: crate::search::Query,
@@ -813,7 +812,6 @@ impl MailBackend for NotmuchDb {
             self.lib.clone(),
             false,
         )?;
-        let lib = self.lib.clone();
         let mailboxes = self.mailboxes.clone();
         Ok(Box::pin(async move {
             let mut ret = SmallVec::new();
@@ -830,7 +828,7 @@ impl MailBackend for NotmuchDb {
                 String::new()
             };
             melib_query.query_to_string(&mut query_s);
-            let query: Query = Query::new(lib.clone(), &database, &query_s)?;
+            let query: Query = Query::new(&database, &query_s)?;
             let iter = query.search()?;
             for message in iter {
                 ret.push(message.env_hash());
@@ -840,8 +838,8 @@ impl MailBackend for NotmuchDb {
         }))
     }
 
-    fn tags(&self) -> Option<Arc<RwLock<BTreeMap<u64, String>>>> {
-        Some(self.tag_index.clone())
+    fn collection(&self) -> Collection {
+        self.collection.clone()
     }
 
     fn as_any(&self) -> &dyn Any {
@@ -857,7 +855,7 @@ impl MailBackend for NotmuchDb {
 struct NotmuchOp {
     hash: EnvelopeHash,
     index: Arc<RwLock<HashMap<EnvelopeHash, CString>>>,
-    tag_index: Arc<RwLock<BTreeMap<u64, String>>>,
+    collection: Collection,
     database: Arc<DbConnection>,
     bytes: Option<Vec<u8>>,
     lib: Arc<libloading::Library>,
@@ -890,11 +888,8 @@ pub struct Query<'s> {
 }
 
 impl<'s> Query<'s> {
-    fn new(
-        lib: Arc<libloading::Library>,
-        database: &DbConnection,
-        query_str: &'s str,
-    ) -> Result<Self> {
+    fn new(database: &DbConnection, query_str: &'s str) -> Result<Self> {
+        let lib: Arc<libloading::Library> = database.lib.clone();
         let query_cstr = std::ffi::CString::new(query_str)?;
         let query: *mut notmuch_query_t = unsafe {
             call!(lib, notmuch_query_create)(*database.inner.read().unwrap(), query_cstr.as_ptr())

melib/src/backends/notmuch/message.rs

@@ -65,6 +65,16 @@ impl<'m> Message<'m> {
         }
     }
 
+    pub fn header(&self, header: &CStr) -> Option<&[u8]> {
+        let header_val =
+            unsafe { call!(self.lib, notmuch_message_get_header)(self.message, header.as_ptr()) };
+        if header_val.is_null() {
+            None
+        } else {
+            Some(unsafe { CStr::from_ptr(header_val).to_bytes() })
+        }
+    }
+
     pub fn msg_id(&self) -> &[u8] {
         let c_str = self.msg_id_cstr();
         c_str.to_bytes()
@@ -81,19 +91,11 @@ impl<'m> Message<'m> {
 
     pub fn into_envelope(
         self,
-        index: Arc<RwLock<HashMap<EnvelopeHash, CString>>>,
-        tag_index: Arc<RwLock<BTreeMap<u64, String>>>,
-    ) -> Result<Envelope> {
-        let mut contents = Vec::new();
-        let path = self.get_filename().to_os_string();
-        let mut f = std::fs::File::open(&path)?;
-        f.read_to_end(&mut contents)?;
+        index: &RwLock<HashMap<EnvelopeHash, CString>>,
+        tag_index: &RwLock<BTreeMap<u64, String>>,
+    ) -> Envelope {
         let env_hash = self.env_hash();
-        let mut env = Envelope::from_bytes(&contents, None).chain_err_summary(|| {
-            index.write().unwrap().remove(&env_hash);
-            format!("could not parse path {:?}", path)
-        })?;
-        env.set_hash(env_hash);
+        let mut env = Envelope::new(env_hash);
         index
             .write()
             .unwrap()
@@ -109,8 +111,63 @@ impl<'m> Message<'m> {
             }
             env.labels_mut().push(num);
         }
-        env.set_flags(flags);
-        Ok(env)
+        unsafe {
+            use crate::email::parser::address::rfc2822address_list;
+            env.set_message_id(self.msg_id())
+                .set_date(
+                    self.header(CStr::from_bytes_with_nul_unchecked(b"Date\0"))
+                        .unwrap_or_default(),
+                )
+                .set_from(
+                    rfc2822address_list(
+                        self.header(CStr::from_bytes_with_nul_unchecked(b"From\0"))
+                            .unwrap_or_default(),
+                    )
+                    .map(|(_, v)| v)
+                    .unwrap_or_default(),
+                )
+                .set_to(
+                    rfc2822address_list(
+                        self.header(CStr::from_bytes_with_nul_unchecked(b"To\0"))
+                            .unwrap_or_default(),
+                    )
+                    .map(|(_, v)| v)
+                    .unwrap_or_default(),
+                )
+                .set_cc(
+                    rfc2822address_list(
+                        self.header(CStr::from_bytes_with_nul_unchecked(b"Cc\0"))
+                            .unwrap_or_default(),
+                    )
+                    .map(|(_, v)| v)
+                    .unwrap_or_default(),
+                )
+                .set_bcc(
+                    rfc2822address_list(
+                        self.header(CStr::from_bytes_with_nul_unchecked(b"Bcc\0"))
+                            .unwrap_or_default(),
+                    )
+                    .map(|(_, v)| v)
+                    .unwrap_or_default()
+                    .to_vec(),
+                )
+                .set_subject(
+                    self.header(CStr::from_bytes_with_nul_unchecked(b"Subject\0"))
+                        .unwrap_or_default()
+                        .to_vec(),
+                )
+                .set_references(
+                    self.header(CStr::from_bytes_with_nul_unchecked(b"References\0"))
+                        .unwrap_or_default(),
+                )
+                .set_in_reply_to(
+                    self.header(CStr::from_bytes_with_nul_unchecked(b"In-Reply-To\0"))
+                        .unwrap_or_default(),
+                )
+                .set_datetime(self.date())
+                .set_flags(flags);
+        }
+        env
     }
 
     pub fn replies_iter(&self) -> Option<MessageIterator> {
@@ -137,6 +194,7 @@ impl<'m> Message<'m> {
             ThreadNode {
                 message: Some(self.env_hash()),
                 parent: None,
+                other_mailbox: false,
                 children: vec![],
                 date: self.date(),
                 show_subject: true,

melib/src/collection.rs

@@ -25,7 +25,7 @@ use smallvec::SmallVec;
 use std::ops::{Deref, DerefMut};
 use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
 
-use std::collections::{HashMap, HashSet};
+use std::collections::{BTreeMap, HashMap, HashSet};
 
 pub struct EnvelopeRef<'g> {
     guard: RwLockReadGuard<'g, HashMap<EnvelopeHash, Envelope>>,
@@ -64,8 +64,9 @@ pub struct Collection {
     pub envelopes: Arc<RwLock<HashMap<EnvelopeHash, Envelope>>>,
     pub message_id_index: Arc<RwLock<HashMap<Vec<u8>, EnvelopeHash>>>,
     pub threads: Arc<RwLock<HashMap<MailboxHash, Threads>>>,
-    sent_mailbox: Arc<RwLock<Option<MailboxHash>>>,
+    pub sent_mailbox: Arc<RwLock<Option<MailboxHash>>>,
     pub mailboxes: Arc<RwLock<HashMap<MailboxHash, HashSet<EnvelopeHash>>>>,
+    pub tag_index: Arc<RwLock<BTreeMap<u64, String>>>,
 }
 
 impl Default for Collection {
@@ -88,7 +89,11 @@ impl Drop for Collection {
                     }
                 };
                 let writer = io::BufWriter::new(f);
-                bincode::serialize_into(writer, &self.threads).unwrap();
+                let _ = bincode::Options::serialize_into(
+                    bincode::config::DefaultOptions::new(),
+                    writer,
+                    &self.thread,
+                );
             }
     }
 }
@@ -111,6 +116,7 @@ impl Collection {
 
         Collection {
             envelopes: Arc::new(RwLock::new(Default::default())),
+            tag_index: Arc::new(RwLock::new(BTreeMap::default())),
             message_id_index,
             threads,
             mailboxes,

melib/src/conf.rs

@@ -128,6 +128,7 @@ pub enum ToggleFlag {
     InternalVal(bool),
     False,
     True,
+    Ask,
 }
 
 impl From<bool> for ToggleFlag {
@@ -157,9 +158,15 @@ impl ToggleFlag {
             false
         }
     }
+
+    pub fn is_ask(&self) -> bool {
+        *self == ToggleFlag::Ask
+    }
+
     pub fn is_false(&self) -> bool {
         ToggleFlag::False == *self || ToggleFlag::InternalVal(false) == *self
     }
+
     pub fn is_true(&self) -> bool {
         ToggleFlag::True == *self || ToggleFlag::InternalVal(true) == *self
     }
@@ -174,6 +181,7 @@ impl Serialize for ToggleFlag {
             ToggleFlag::Unset | ToggleFlag::InternalVal(_) => serializer.serialize_none(),
             ToggleFlag::False => serializer.serialize_bool(false),
             ToggleFlag::True => serializer.serialize_bool(true),
+            ToggleFlag::Ask => serializer.serialize_str("ask"),
         }
     }
 }
@@ -183,10 +191,17 @@ impl<'de> Deserialize<'de> for ToggleFlag {
     where
         D: Deserializer<'de>,
     {
-        let s = <bool>::deserialize(deserializer);
+        let s = <String>::deserialize(deserializer);
         Ok(match s? {
-            true => ToggleFlag::True,
-            false => ToggleFlag::False,
+            s if s.eq_ignore_ascii_case("true") => ToggleFlag::True,
+            s if s.eq_ignore_ascii_case("false") => ToggleFlag::False,
+            s if s.eq_ignore_ascii_case("ask") => ToggleFlag::Ask,
+            s => {
+                return Err(serde::de::Error::custom(format!(
+                    r#"expected one of "true", "false", "ask", found `{}`"#,
+                    s
+                )))
+            }
         })
     }
 }

melib/src/connections.rs

@@ -291,3 +291,7 @@ pub async fn timeout<O>(dur: Option<Duration>, f: impl Future<Output = O>) -> cr
         Ok(f.await)
     }
 }
+
+pub async fn sleep(dur: Duration) {
+    smol::Timer::after(dur).await;
+}

melib/src/datetime.rs

@@ -34,67 +34,133 @@
 //! assert_eq!(timestamp, 1578509043);
 //!
 //! // Convert timestamp back to string
-//! let s = timestamp_to_string(timestamp, Some("%Y-%m-%d"));
+//! let s = timestamp_to_string(timestamp, Some("%Y-%m-%d"), true);
 //! assert_eq!(s, "2020-01-08");
 //! ```
-use crate::error::Result;
+use crate::error::{Result, ResultIntoMeliError};
+use std::borrow::Cow;
 use std::convert::TryInto;
 use std::ffi::{CStr, CString};
 
 pub type UnixTimestamp = u64;
-
-use libc::{timeval, timezone};
+pub const RFC3339_FMT_WITH_TIME: &str = "%Y-%m-%dT%H:%M:%S\0";
+pub const RFC3339_FMT: &str = "%Y-%m-%d\0";
+pub const RFC822_FMT_WITH_TIME: &str = "%a, %e %h %Y %H:%M:%S \0";
+pub const RFC822_FMT: &str = "%e %h %Y %H:%M:%S \0";
+pub const DEFAULT_FMT: &str = "%a, %d %b %Y %R\0";
+//"Tue May 21 13:46:22 1991\n"
+pub const ASCTIME_FMT: &str = "%a %b %d %H:%M:%S %Y\n\0";
 
 extern "C" {
     fn strptime(
-        s: *const ::std::os::raw::c_char,
-        format: *const ::std::os::raw::c_char,
-        tm: *mut ::libc::tm,
-    ) -> *const ::std::os::raw::c_char;
+        s: *const std::os::raw::c_char,
+        format: *const std::os::raw::c_char,
+        tm: *mut libc::tm,
+    ) -> *const std::os::raw::c_char;
 
     fn strftime(
-        s: *mut ::std::os::raw::c_char,
-        max: ::libc::size_t,
-        format: *const ::std::os::raw::c_char,
-        tm: *const ::libc::tm,
-    ) -> ::libc::size_t;
+        s: *mut std::os::raw::c_char,
+        max: libc::size_t,
+        format: *const std::os::raw::c_char,
+        tm: *const libc::tm,
+    ) -> libc::size_t;
 
-    fn mktime(tm: *const ::libc::tm) -> ::libc::time_t;
+    fn mktime(tm: *const libc::tm) -> libc::time_t;
 
-    fn localtime_r(timep: *const ::libc::time_t, tm: *mut ::libc::tm) -> *mut ::libc::tm;
+    fn localtime_r(timep: *const libc::time_t, tm: *mut libc::tm) -> *mut libc::tm;
 
-    fn gettimeofday(tv: *mut timeval, tz: *mut timezone) -> i32;
+    fn gettimeofday(tv: *mut libc::timeval, tz: *mut libc::timezone) -> i32;
 }
 
-pub fn timestamp_to_string(timestamp: UnixTimestamp, fmt: Option<&str>) -> String {
-    let mut new_tm: ::libc::tm = unsafe { std::mem::zeroed() };
+struct Locale {
+    new_locale: libc::locale_t,
+    old_locale: libc::locale_t,
+}
+
+impl Drop for Locale {
+    fn drop(&mut self) {
+        unsafe {
+            let _ = libc::uselocale(self.old_locale);
+            libc::freelocale(self.new_locale);
+        }
+    }
+}
+
+// How to unit test this? Test machine is not guaranteed to have non-english locales.
+impl Locale {
+    fn new(
+        mask: std::os::raw::c_int,
+        locale: *const std::os::raw::c_char,
+        base: libc::locale_t,
+    ) -> Result<Self> {
+        let new_locale = unsafe { libc::newlocale(mask, locale, base) };
+        if new_locale.is_null() {
+            return Err(nix::Error::last().into());
+        }
+        let old_locale = unsafe { libc::uselocale(new_locale) };
+        if old_locale.is_null() {
+            unsafe { libc::freelocale(new_locale) };
+            return Err(nix::Error::last().into());
+        }
+        Ok(Locale {
+            new_locale,
+            old_locale,
+        })
+    }
+}
+
+pub fn timestamp_to_string(timestamp: UnixTimestamp, fmt: Option<&str>, posix: bool) -> String {
+    let mut new_tm: libc::tm = unsafe { std::mem::zeroed() };
     unsafe {
         let i: i64 = timestamp.try_into().unwrap_or(0);
-        localtime_r(&i as *const i64, &mut new_tm as *mut ::libc::tm);
+        localtime_r(&i as *const i64, &mut new_tm as *mut libc::tm);
     }
-    let fmt = fmt
+    let format: Cow<'_, CStr> = if let Some(cs) = fmt
+        .map(str::as_bytes)
+        .map(CStr::from_bytes_with_nul)
+        .and_then(|res| res.ok())
+    {
+        Cow::from(cs)
+    } else if let Some(cstring) = fmt
+        .map(str::as_bytes)
         .map(CString::new)
-        .map(|res| res.ok())
-        .and_then(|opt| opt);
-    let format: &CStr = if let Some(ref s) = fmt {
-        &s
+        .and_then(|res| res.ok())
+    {
+        Cow::from(cstring)
     } else {
-        unsafe { CStr::from_bytes_with_nul_unchecked(b"%a, %d %b %Y %T %z\0") }
+        unsafe { CStr::from_bytes_with_nul_unchecked(DEFAULT_FMT.as_bytes()).into() }
     };
+
     let mut vec: [u8; 256] = [0; 256];
-    let ret = unsafe {
-        strftime(
-            vec.as_mut_ptr() as *mut _,
-            256,
-            format.as_ptr(),
-            &new_tm as *const _,
-        )
+    let ret = {
+        let _with_locale: Option<Result<Locale>> = if posix {
+            Some(
+                Locale::new(
+                    libc::LC_TIME,
+                    b"C\0".as_ptr() as *const i8,
+                    std::ptr::null_mut(),
+                )
+                .chain_err_summary(|| "Could not set locale for datetime conversion")
+                .chain_err_kind(crate::error::ErrorKind::External),
+            )
+        } else {
+            None
+        };
+
+        unsafe {
+            strftime(
+                vec.as_mut_ptr() as *mut _,
+                256,
+                format.as_ptr(),
+                &new_tm as *const _,
+            )
+        }
     };
 
     String::from_utf8_lossy(&vec[0..ret]).into_owned()
 }
 
-fn tm_to_secs(tm: ::libc::tm) -> std::result::Result<i64, ()> {
+fn tm_to_secs(tm: libc::tm) -> std::result::Result<i64, ()> {
     let mut is_leap = false;
     let mut year = tm.tm_year;
     let mut month = tm.tm_mon;
@@ -207,52 +273,58 @@ where
     T: Into<Vec<u8>>,
 {
     let s = CString::new(s)?;
-    let mut new_tm: ::libc::tm = unsafe { std::mem::zeroed() };
-    for fmt in &[
-        &b"%a, %e %h %Y %H:%M:%S \0"[..],
-        &b"%e %h %Y %H:%M:%S \0"[..],
-    ] {
-        unsafe {
-            let fmt = CStr::from_bytes_with_nul_unchecked(fmt);
-            let ret = strptime(s.as_ptr(), fmt.as_ptr(), &mut new_tm as *mut _);
-            if ret.is_null() {
-                continue;
-            }
-            let rest = CStr::from_ptr(ret);
-            let tm_gmtoff = if rest.to_bytes().len() > 4
-                && rest.to_bytes().is_ascii()
-                && rest.to_bytes()[1..5].iter().all(u8::is_ascii_digit)
-            {
-                let offset = std::str::from_utf8_unchecked(&rest.to_bytes()[0..5]);
-                if let (Ok(mut hr_offset), Ok(mut min_offset)) =
-                    (offset[1..3].parse::<i64>(), offset[3..5].parse::<i64>())
-                {
-                    if rest.to_bytes()[0] == b'-' {
-                        hr_offset = -hr_offset;
-                        min_offset = -min_offset;
-                    }
-                    hr_offset * 60 * 60 + min_offset * 60
-                } else {
-                    0
-                }
-            } else {
-                let rest = if rest.to_bytes().starts_with(b"(") && rest.to_bytes().ends_with(b")") {
-                    &rest.to_bytes()[1..rest.to_bytes().len() - 1]
-                } else {
-                    rest.to_bytes()
-                };
+    let mut new_tm: libc::tm = unsafe { std::mem::zeroed() };
+    for fmt in &[RFC822_FMT_WITH_TIME, RFC822_FMT] {
+        let fmt = unsafe { CStr::from_bytes_with_nul_unchecked(fmt.as_bytes()) };
+        let ret = {
+            let _with_locale = Locale::new(
+                libc::LC_TIME,
+                b"C\0".as_ptr() as *const i8,
+                std::ptr::null_mut(),
+            )
+            .chain_err_summary(|| "Could not set locale for datetime conversion")
+            .chain_err_kind(crate::error::ErrorKind::External)?;
+            unsafe { strptime(s.as_ptr(), fmt.as_ptr(), &mut new_tm as *mut _) }
+        };
 
-                if let Ok(idx) = TIMEZONE_ABBR.binary_search_by(|probe| probe.0.cmp(rest)) {
-                    let (hr_offset, min_offset) = TIMEZONE_ABBR[idx].1;
-                    (hr_offset as i64) * 60 * 60 + (min_offset as i64) * 60
-                } else {
-                    0
-                }
-            };
-            return Ok(tm_to_secs(new_tm)
-                .map(|res| (res - tm_gmtoff) as u64)
-                .unwrap_or(0));
+        if ret.is_null() {
+            continue;
         }
+        let rest = unsafe { CStr::from_ptr(ret) };
+        let tm_gmtoff = if rest.to_bytes().len() > 4
+            && rest.to_bytes().is_ascii()
+            && rest.to_bytes()[1..5].iter().all(u8::is_ascii_digit)
+        {
+            // safe since rest.to_bytes().is_ascii()
+            let offset = unsafe { std::str::from_utf8_unchecked(&rest.to_bytes()[0..5]) };
+            if let (Ok(mut hr_offset), Ok(mut min_offset)) =
+                (offset[1..3].parse::<i64>(), offset[3..5].parse::<i64>())
+            {
+                if rest.to_bytes()[0] == b'-' {
+                    hr_offset = -hr_offset;
+                    min_offset = -min_offset;
+                }
+                hr_offset * 60 * 60 + min_offset * 60
+            } else {
+                0
+            }
+        } else {
+            let rest = if rest.to_bytes().starts_with(b"(") && rest.to_bytes().ends_with(b")") {
+                &rest.to_bytes()[1..rest.to_bytes().len() - 1]
+            } else {
+                rest.to_bytes()
+            };
+
+            if let Ok(idx) = TIMEZONE_ABBR.binary_search_by(|probe| probe.0.cmp(rest)) {
+                let (hr_offset, min_offset) = TIMEZONE_ABBR[idx].1;
+                (hr_offset as i64) * 60 * 60 + (min_offset as i64) * 60
+            } else {
+                0
+            }
+        };
+        return Ok(tm_to_secs(new_tm)
+            .map(|res| (res - tm_gmtoff) as u64)
+            .unwrap_or(0));
     }
     Ok(0)
 }
@@ -262,50 +334,58 @@ where
     T: Into<Vec<u8>>,
 {
     let s = CString::new(s)?;
-    let mut new_tm: ::libc::tm = unsafe { std::mem::zeroed() };
-    for fmt in &[&b"%Y-%m-%dT%H:%M:%S\0"[..], &b"%Y-%m-%d\0"[..]] {
-        unsafe {
-            let fmt = CStr::from_bytes_with_nul_unchecked(fmt);
-            let ret = strptime(s.as_ptr(), fmt.as_ptr(), &mut new_tm as *mut _);
-            if ret.is_null() {
-                continue;
-            }
-            let rest = CStr::from_ptr(ret);
-            let tm_gmtoff = if rest.to_bytes().len() > 4
-                && rest.to_bytes().is_ascii()
-                && rest.to_bytes()[1..3].iter().all(u8::is_ascii_digit)
-                && rest.to_bytes()[4..6].iter().all(u8::is_ascii_digit)
-            {
-                let offset = std::str::from_utf8_unchecked(&rest.to_bytes()[0..6]);
-                if let (Ok(mut hr_offset), Ok(mut min_offset)) =
-                    (offset[1..3].parse::<i64>(), offset[4..6].parse::<i64>())
-                {
-                    if rest.to_bytes()[0] == b'-' {
-                        hr_offset = -hr_offset;
-                        min_offset = -min_offset;
-                    }
-                    hr_offset * 60 * 60 + min_offset * 60
-                } else {
-                    0
-                }
-            } else {
-                let rest = if rest.to_bytes().starts_with(b"(") && rest.to_bytes().ends_with(b")") {
-                    &rest.to_bytes()[1..rest.to_bytes().len() - 1]
-                } else {
-                    rest.to_bytes()
-                };
-
-                if let Ok(idx) = TIMEZONE_ABBR.binary_search_by(|probe| probe.0.cmp(rest)) {
-                    let (hr_offset, min_offset) = debug!(TIMEZONE_ABBR[idx]).1;
-                    (hr_offset as i64) * 60 * 60 + (min_offset as i64) * 60
-                } else {
-                    0
-                }
-            };
-            return Ok(tm_to_secs(new_tm)
-                .map(|res| (res - tm_gmtoff) as u64)
-                .unwrap_or(0));
+    let mut new_tm: libc::tm = unsafe { std::mem::zeroed() };
+    for fmt in &[RFC3339_FMT_WITH_TIME, RFC3339_FMT] {
+        let fmt = unsafe { CStr::from_bytes_with_nul_unchecked(fmt.as_bytes()) };
+        let ret = {
+            let _with_locale = Locale::new(
+                libc::LC_TIME,
+                b"C\0".as_ptr() as *const i8,
+                std::ptr::null_mut(),
+            )
+            .chain_err_summary(|| "Could not set locale for datetime conversion")
+            .chain_err_kind(crate::error::ErrorKind::External)?;
+            unsafe { strptime(s.as_ptr(), fmt.as_ptr(), &mut new_tm as *mut _) }
+        };
+        if ret.is_null() {
+            continue;
         }
+        let rest = unsafe { CStr::from_ptr(ret) };
+        let tm_gmtoff = if rest.to_bytes().len() > 4
+            && rest.to_bytes().is_ascii()
+            && rest.to_bytes()[1..3].iter().all(u8::is_ascii_digit)
+            && rest.to_bytes()[4..6].iter().all(u8::is_ascii_digit)
+        {
+            // safe since rest.to_bytes().is_ascii()
+            let offset = unsafe { std::str::from_utf8_unchecked(&rest.to_bytes()[0..6]) };
+            if let (Ok(mut hr_offset), Ok(mut min_offset)) =
+                (offset[1..3].parse::<i64>(), offset[4..6].parse::<i64>())
+            {
+                if rest.to_bytes()[0] == b'-' {
+                    hr_offset = -hr_offset;
+                    min_offset = -min_offset;
+                }
+                hr_offset * 60 * 60 + min_offset * 60
+            } else {
+                0
+            }
+        } else {
+            let rest = if rest.to_bytes().starts_with(b"(") && rest.to_bytes().ends_with(b")") {
+                &rest.to_bytes()[1..rest.to_bytes().len() - 1]
+            } else {
+                rest.to_bytes()
+            };
+
+            if let Ok(idx) = TIMEZONE_ABBR.binary_search_by(|probe| probe.0.cmp(rest)) {
+                let (hr_offset, min_offset) = TIMEZONE_ABBR[idx].1;
+                (hr_offset as i64) * 60 * 60 + (min_offset as i64) * 60
+            } else {
+                0
+            }
+        };
+        return Ok(tm_to_secs(new_tm)
+            .map(|res| (res - tm_gmtoff) as u64)
+            .unwrap_or(0));
     }
     Ok(0)
 }
@@ -315,8 +395,12 @@ pub fn timestamp_from_string<T>(s: T, fmt: &str) -> Result<Option<UnixTimestamp>
 where
     T: Into<Vec<u8>>,
 {
-    let mut new_tm: ::libc::tm = unsafe { std::mem::zeroed() };
-    let fmt = CString::new(fmt)?;
+    let mut new_tm: libc::tm = unsafe { std::mem::zeroed() };
+    let fmt: Cow<'_, CStr> = if let Ok(cs) = CStr::from_bytes_with_nul(fmt.as_bytes()) {
+        Cow::from(cs)
+    } else {
+        Cow::from(CString::new(fmt.as_bytes())?)
+    };
     unsafe {
         let ret = strptime(
             CString::new(s)?.as_ptr(),
@@ -332,8 +416,8 @@ where
 
 pub fn now() -> UnixTimestamp {
     use std::mem::MaybeUninit;
-    let mut tv = MaybeUninit::<::libc::timeval>::uninit();
-    let mut tz = MaybeUninit::<::libc::timezone>::uninit();
+    let mut tv = MaybeUninit::<libc::timeval>::uninit();
+    let mut tz = MaybeUninit::<libc::timezone>::uninit();
     unsafe {
         let ret = gettimeofday(tv.as_mut_ptr(), tz.as_mut_ptr());
         if ret == -1 {
@@ -344,12 +428,15 @@ pub fn now() -> UnixTimestamp {
 }
 
 #[test]
-fn test_timestamp() {
-    timestamp_to_string(0, None);
+fn test_datetime_timestamp() {
+    timestamp_to_string(0, None, false);
 }
 
 #[test]
-fn test_rfcs() {
+fn test_datetime_rfcs() {
+    if unsafe { libc::setlocale(libc::LC_ALL, b"\0".as_ptr() as _) }.is_null() {
+        println!("Unable to set locale.");
+    }
     /* Some tests were lazily stolen from https://rachelbythebay.com/w/2013/06/11/time/ */
 
     assert_eq!(
@@ -360,7 +447,7 @@ fn test_rfcs() {
     /*
     macro_rules! mkt {
         ($year:literal, $month:literal, $day:literal, $hour:literal, $minute:literal, $second:literal) => {
-            ::libc::tm {
+            libc::tm {
                 tm_sec: $second,
                 tm_min: $minute,
                 tm_hour: $hour,

melib/src/email.rs

@@ -20,8 +20,75 @@
  */
 
 /*!
- * Email parsing, handling, sending etc.
+ * Email parsing and composing.
+ *
+ * # Parsing bytes into an `Envelope`
+ *
+ * An [`Envelope`](Envelope) represents the information you can get from an email's headers and body
+ * structure. Addresses in `To`, `From` fields etc are parsed into [`Address`](email::address::Address) types.
+ *
+ * ```
+ * use melib::{Attachment, Envelope};
+ *
+ * let raw_mail = r#"From: "some name" <some@example.com>
+ * To: "me" <myself@example.com>
+ * Cc:
+ * Subject: =?utf-8?Q?gratuitously_encoded_subject?=
+ * Message-ID: <h2g7f.z0gy2pgaen5m@example.com>
+ * MIME-Version: 1.0
+ * Content-Type: multipart/mixed; charset="utf-8";
+ *  boundary="bzz_bzz__bzz__"
+ *
+ * This is a MIME formatted message with attachments. Use a MIME-compliant client to view it properly.
+ * --bzz_bzz__bzz__
+ *
+ * hello world.
+ * --bzz_bzz__bzz__
+ * Content-Type: image/gif; name="test_image.gif"; charset="utf-8"
+ * Content-Disposition: attachment
+ * Content-Transfer-Encoding: base64
+ *
+ * R0lGODdhKAAXAOfZAAABzAADzQAEzgQFtBEAxAAGxBcAxwALvRcFwAAPwBcLugATuQEUuxoNuxYQ
+ * sxwOvAYVvBsStSAVtx8YsRUcuhwhth4iuCQsyDAwuDc1vTc3uDg4uT85rkc9ukJBvENCvURGukdF
+ * wUVKt0hLuUxPvVZSvFlYu1hbt2BZuFxdul5joGhqlnNuf3FvlnBvwXJyt3Jxw3N0oXx1gH12gV99
+ * z317f3N7spFxwHp5wH99gYB+goF/g25+26tziIOBhWqD3oiBjICAuudkjIN+zHeC2n6Bzc1vh4eF
+ * iYaBw8F0kImHi4KFxYyHmIWIvI2Lj4uIvYaJyY+IuJGMi5iJl4qKxZSMmIuLxpONnpGPk42NvI2M
+ * 1LKGl46OvZePm5ORlZiQnJqSnpaUmLyJnJuTn5iVmZyUoJGVyZ2VoZSVw5iXoZmWrO18rJiUyp6W
+ * opuYnKaVnZ+Xo5yZncaMoaCYpJiaqo+Z2Z2annuf5qGZpa2WoJybpZmayZ2Z0KCZypydrZ6dp6Cd
+ * oZ6a0aGay5ucy5+eqKGeouWMgp+b0qKbzKCfqdqPnp2ezaGgqqOgpKafqrScpp+gz6ajqKujr62j
+ * qayksKmmq62lsaiosqqorOyWnaqqtKeqzLGptaurta2rr7Kqtq+ssLOrt6+uuLGusuqhfbWtubCv
+ * ubKvs7GwurOwtPSazbevu+ali7SxtbiwvOykjLOyvLWytuCmqOankrSzvbazuLmyvrW0vre0uba1
+ * wLi1ury0wLm2u721wbe3wbq3vMC2vLi4wr+3w7m5w8C4xLi6yry6vsG5xbu7xcC6zMK6xry8xry+
+ * u8O7x729x8C9wb++yMG+wsO+vMK/w8a+y8e/zMnBzcXH18nL2///////////////////////////
+ * ////////////////////////////////////////////////////////////////////////////
+ * /////////////////////////////////////////////////////ywAAAAAKAAXAAAI/gBP4Cjh
+ * IYMLEh0w4EgBgsMLEyFGFBEB5cOFABgzatS4AVssZAOsLOHCxooVMzCyoNmzaBOkJlS0VEDyZMjG
+ * mxk3XOMF60CDBgsoPABK9KcDCRImPCiQYAECAgQCRMU4VSrGCjFarBgUSJCgQ10FBTrkNRCfPnz4
+ * dA3UNa1btnDZqgU7Ntqzu3ej2X2mFy9eaHuhNRtMGJrhwYYN930G2K7eaNIY34U2mfJkwpgzI9Yr
+ * GBqwR2KSvAlMOXHnw5pTNzPdLNoWIWtU9XjGjDEYS8LAlFm1SrVvzIKj5TH0KpORSZOryPgCZgqL
+ * Ob+jG0YVRBErUrOiiGJ8KxgtYsh27xWL/tswnTtEbsiRVYdJNMHk4yOGhswGjR88UKjQ9Ey+/8TL
+ * XKKGGn7Akph/8XX2WDTTcAYfguVt9hhrEPqmzIOJ3VUheb48WJiHG6amC4i+WVJKKCimqGIoYxyj
+ * WWK8kKjaJ9bA18sxvXjYhourmbbMMrjI+OIn1QymDCVXANGFK4S1gQw0PxozzC+33FLLKUJq9gk1
+ * gyWDhyNwrMLkYGUEM4wvuLRiCiieXIJJJVlmJskcZ9TZRht1lnFGGmTMkMoonVQSSSOFAGJHHI0w
+ * ouiijDaaCCGQRgrpH3q4QYYXWDihxBE+7KCDDjnUIEVAADs=
+ * --bzz_bzz__bzz__--"#;
+ *
+ * let envelope = Envelope::from_bytes(raw_mail.as_bytes(), None).expect("Could not parse mail");
+ * assert_eq!(envelope.subject().as_ref(), "gratuitously encoded subject");
+ * assert_eq!(envelope.message_id_display().as_ref(), "<h2g7f.z0gy2pgaen5m@example.com>");
+ *
+ * let body = envelope.body_bytes(raw_mail.as_bytes());
+ * assert_eq!(body.content_type().to_string().as_str(), "multipart/mixed");
+ *
+ * let body_text = body.text();
+ * assert_eq!(body_text.as_str(), "hello world.");
+ *
+ * let subattachments: Vec<Attachment> = body.attachments();
+ * assert_eq!(subattachments.len(), 3);
+ * assert_eq!(subattachments[2].content_type().name().unwrap(), "test_image.gif");
+ * ```
  */
+
 pub mod address;
 pub mod attachment_types;
 pub mod attachments;
@@ -30,7 +97,7 @@ pub mod headers;
 pub mod list_management;
 pub mod mailto;
 pub mod parser;
-pub mod signatures;
+pub mod pgp;
 
 pub use address::{Address, MessageID, References, StrBuild, StrBuilder};
 pub use attachments::{Attachment, AttachmentBuilder};
@@ -76,6 +143,23 @@ impl PartialEq<&str> for Flag {
     }
 }
 
+macro_rules! flag_impl {
+    (fn $name:ident, $val:expr) => {
+        pub const fn $name(&self) -> bool {
+            self.contains($val)
+        }
+    };
+}
+
+impl Flag {
+    flag_impl!(fn is_seen, Flag::SEEN);
+    flag_impl!(fn is_draft, Flag::DRAFT);
+    flag_impl!(fn is_trashed, Flag::TRASHED);
+    flag_impl!(fn is_passed, Flag::PASSED);
+    flag_impl!(fn is_replied, Flag::REPLIED);
+    flag_impl!(fn is_flagged, Flag::FLAGGED);
+}
+
 ///`Mail` holds both the envelope info of an email in its `envelope` field and the raw bytes that
 ///describe the email in `bytes`. Its body as an `melib::email::Attachment` can be parsed on demand
 ///with the `melib::email::Mail::body` method.
@@ -152,6 +236,7 @@ impl core::fmt::Debug for Envelope {
             .field("Message-ID", &self.message_id_display())
             .field("In-Reply-To", &self.in_reply_to_display())
             .field("References", &self.references)
+            .field("Flags", &self.flags)
             .field("Hash", &self.hash)
             .finish()
     }
@@ -185,8 +270,9 @@ impl Envelope {
         }
     }
 
-    pub fn set_hash(&mut self, new_hash: EnvelopeHash) {
+    pub fn set_hash(&mut self, new_hash: EnvelopeHash) -> &mut Self {
         self.hash = new_hash;
+        self
     }
 
     pub fn from_bytes(bytes: &[u8], flags: Option<Flag>) -> Result<Envelope> {
@@ -252,14 +338,6 @@ impl Envelope {
             } else if name == "message-id" {
                 self.set_message_id(value);
             } else if name == "references" {
-                {
-                    let parse_result = parser::address::msg_id_list(value);
-                    if let Ok((_, value)) = parse_result {
-                        for v in value {
-                            self.push_references(v);
-                        }
-                    }
-                }
                 self.set_references(value);
             } else if name == "in-reply-to" {
                 self.set_in_reply_to(value);
@@ -506,41 +584,51 @@ impl Envelope {
         String::from_utf8_lossy(self.message_id.raw())
     }
 
-    pub fn set_date(&mut self, new_val: &[u8]) {
+    pub fn set_date(&mut self, new_val: &[u8]) -> &mut Self {
         let new_val = new_val.trim();
         self.date = String::from_utf8_lossy(new_val).into_owned();
+        self
     }
 
-    pub fn set_bcc(&mut self, new_val: Vec<Address>) {
+    pub fn set_bcc(&mut self, new_val: Vec<Address>) -> &mut Self {
         self.bcc = new_val;
+        self
     }
 
-    pub fn set_cc(&mut self, new_val: SmallVec<[Address; 1]>) {
+    pub fn set_cc(&mut self, new_val: SmallVec<[Address; 1]>) -> &mut Self {
         self.cc = new_val;
+        self
     }
 
-    pub fn set_from(&mut self, new_val: SmallVec<[Address; 1]>) {
+    pub fn set_from(&mut self, new_val: SmallVec<[Address; 1]>) -> &mut Self {
         self.from = new_val;
+        self
     }
 
-    pub fn set_to(&mut self, new_val: SmallVec<[Address; 1]>) {
+    pub fn set_to(&mut self, new_val: SmallVec<[Address; 1]>) -> &mut Self {
         self.to = new_val;
+        self
     }
 
-    pub fn set_in_reply_to(&mut self, new_val: &[u8]) {
+    pub fn set_in_reply_to(&mut self, new_val: &[u8]) -> &mut Self {
         // FIXME msg_id_list
         let new_val = new_val.trim();
-        let val = match parser::address::msg_id(new_val) {
-            Ok(v) => v.1,
-            Err(_) => {
-                self.in_reply_to = Some(MessageID::new(new_val, new_val));
-                return;
-            }
-        };
-        self.in_reply_to = Some(val);
+        if !new_val.is_empty() {
+            let val = match parser::address::msg_id(new_val) {
+                Ok(v) => v.1,
+                Err(_) => {
+                    self.in_reply_to = Some(MessageID::new(new_val, new_val));
+                    return self;
+                }
+            };
+            self.in_reply_to = Some(val);
+        } else {
+            self.in_reply_to = None;
+        }
+        self
     }
 
-    pub fn set_subject(&mut self, new_val: Vec<u8>) {
+    pub fn set_subject(&mut self, new_val: Vec<u8>) -> &mut Self {
         let mut new_val = String::from_utf8(new_val)
             .unwrap_or_else(|err| String::from_utf8_lossy(&err.into_bytes()).into());
         while new_val
@@ -553,9 +641,10 @@ impl Envelope {
         }
 
         self.subject = Some(new_val);
+        self
     }
 
-    pub fn set_message_id(&mut self, new_val: &[u8]) {
+    pub fn set_message_id(&mut self, new_val: &[u8]) -> &mut Self {
         let new_val = new_val.trim();
         match parser::address::msg_id(new_val) {
             Ok((_, val)) => {
@@ -565,6 +654,7 @@ impl Envelope {
                 self.message_id = MessageID::new(new_val, new_val);
             }
         }
+        self
     }
 
     pub fn push_references(&mut self, new_ref: MessageID) {
@@ -593,19 +683,31 @@ impl Envelope {
         }
     }
 
-    pub fn set_references(&mut self, new_val: &[u8]) {
+    pub fn set_references(&mut self, new_val: &[u8]) -> &mut Self {
         let new_val = new_val.trim();
-        match self.references {
-            Some(ref mut s) => {
-                s.raw = new_val.into();
+        if !new_val.is_empty() {
+            self.references = None;
+            {
+                let parse_result = parser::address::msg_id_list(new_val);
+                if let Ok((_, value)) = parse_result {
+                    for v in value {
+                        self.push_references(v);
+                    }
+                }
             }
-            None => {
-                self.references = Some(References {
-                    raw: new_val.into(),
-                    refs: Vec::new(),
-                });
+            match self.references {
+                Some(ref mut s) => {
+                    s.raw = new_val.into();
+                }
+                None => {
+                    self.references = Some(References {
+                        raw: new_val.into(),
+                        refs: Vec::new(),
+                    });
+                }
             }
         }
+        self
     }
 
     pub fn references(&self) -> SmallVec<[&MessageID; 8]> {
@@ -630,40 +732,47 @@ impl Envelope {
         self.thread
     }
 
-    pub fn set_thread(&mut self, new_val: ThreadNodeHash) {
+    pub fn set_thread(&mut self, new_val: ThreadNodeHash) -> &mut Self {
         self.thread = new_val;
+        self
     }
 
-    pub fn set_datetime(&mut self, new_val: UnixTimestamp) {
+    pub fn set_datetime(&mut self, new_val: UnixTimestamp) -> &mut Self {
         self.timestamp = new_val;
+        self
     }
 
-    pub fn set_flag(&mut self, f: Flag, value: bool) {
+    pub fn set_flag(&mut self, f: Flag, value: bool) -> &mut Self {
         self.flags.set(f, value);
+        self
     }
 
-    pub fn set_flags(&mut self, f: Flag) {
+    pub fn set_flags(&mut self, f: Flag) -> &mut Self {
         self.flags = f;
+        self
     }
 
     pub fn flags(&self) -> Flag {
         self.flags
     }
 
-    pub fn set_seen(&mut self) {
-        self.set_flag(Flag::SEEN, true)
+    pub fn set_seen(&mut self) -> &mut Self {
+        self.set_flag(Flag::SEEN, true);
+        self
     }
 
-    pub fn set_unseen(&mut self) {
-        self.set_flag(Flag::SEEN, false)
+    pub fn set_unseen(&mut self) -> &mut Self {
+        self.set_flag(Flag::SEEN, false);
+        self
     }
 
     pub fn is_seen(&self) -> bool {
         self.flags.contains(Flag::SEEN)
     }
 
-    pub fn set_has_attachments(&mut self, new_val: bool) {
+    pub fn set_has_attachments(&mut self, new_val: bool) -> &mut Self {
         self.has_attachments = new_val;
+        self
     }
 
     pub fn has_attachments(&self) -> bool {

melib/src/email/attachment_types.rs

@@ -31,8 +31,17 @@ pub enum Charset {
     UTF16,
     ISO8859_1,
     ISO8859_2,
+    ISO8859_3,
+    ISO8859_4,
+    ISO8859_5,
+    ISO8859_6,
     ISO8859_7,
+    ISO8859_8,
+    ISO8859_10,
+    ISO8859_13,
+    ISO8859_14,
     ISO8859_15,
+    ISO8859_16,
     Windows1250,
     Windows1251,
     Windows1252,
@@ -41,6 +50,9 @@ pub enum Charset {
     GB2312,
     BIG5,
     ISO2022JP,
+    EUCJP,
+    KOI8R,
+    KOI8U,
 }
 
 impl Default for Charset {
@@ -67,14 +79,49 @@ impl<'a> From<&'a [u8]> for Charset {
             b if b.eq_ignore_ascii_case(b"iso-8859-2") || b.eq_ignore_ascii_case(b"iso8859-2") => {
                 Charset::ISO8859_2
             }
+            b if b.eq_ignore_ascii_case(b"iso-8859-3") || b.eq_ignore_ascii_case(b"iso8859-3") => {
+                Charset::ISO8859_3
+            }
+            b if b.eq_ignore_ascii_case(b"iso-8859-4") || b.eq_ignore_ascii_case(b"iso8859-4") => {
+                Charset::ISO8859_4
+            }
+            b if b.eq_ignore_ascii_case(b"iso-8859-5") || b.eq_ignore_ascii_case(b"iso8859-5") => {
+                Charset::ISO8859_5
+            }
+            b if b.eq_ignore_ascii_case(b"iso-8859-6") || b.eq_ignore_ascii_case(b"iso8859-6") => {
+                Charset::ISO8859_6
+            }
             b if b.eq_ignore_ascii_case(b"iso-8859-7") || b.eq_ignore_ascii_case(b"iso8859-7") => {
                 Charset::ISO8859_7
             }
+            b if b.eq_ignore_ascii_case(b"iso-8859-8") || b.eq_ignore_ascii_case(b"iso8859-8") => {
+                Charset::ISO8859_8
+            }
+            b if b.eq_ignore_ascii_case(b"iso-8859-10")
+                || b.eq_ignore_ascii_case(b"iso8859-10") =>
+            {
+                Charset::ISO8859_10
+            }
+            b if b.eq_ignore_ascii_case(b"iso-8859-13")
+                || b.eq_ignore_ascii_case(b"iso8859-13") =>
+            {
+                Charset::ISO8859_13
+            }
+            b if b.eq_ignore_ascii_case(b"iso-8859-14")
+                || b.eq_ignore_ascii_case(b"iso8859-14") =>
+            {
+                Charset::ISO8859_14
+            }
             b if b.eq_ignore_ascii_case(b"iso-8859-15")
                 || b.eq_ignore_ascii_case(b"iso8859-15") =>
             {
                 Charset::ISO8859_15
             }
+            b if b.eq_ignore_ascii_case(b"iso-8859-16")
+                || b.eq_ignore_ascii_case(b"iso8859-16") =>
+            {
+                Charset::ISO8859_16
+            }
             b if b.eq_ignore_ascii_case(b"windows-1250")
                 || b.eq_ignore_ascii_case(b"windows1250") =>
             {
@@ -101,6 +148,9 @@ impl<'a> From<&'a [u8]> for Charset {
             }
             b if b.eq_ignore_ascii_case(b"big5") => Charset::BIG5,
             b if b.eq_ignore_ascii_case(b"iso-2022-jp") => Charset::ISO2022JP,
+            b if b.eq_ignore_ascii_case(b"euc-jp") => Charset::EUCJP,
+            b if b.eq_ignore_ascii_case(b"koi8-r") => Charset::KOI8R,
+            b if b.eq_ignore_ascii_case(b"koi8-u") => Charset::KOI8U,
             _ => {
                 debug!("unknown tag is {:?}", str::from_utf8(b));
                 Charset::Ascii
@@ -117,16 +167,28 @@ impl Display for Charset {
             Charset::UTF16 => write!(f, "utf-16"),
             Charset::ISO8859_1 => write!(f, "iso-8859-1"),
             Charset::ISO8859_2 => write!(f, "iso-8859-2"),
+            Charset::ISO8859_3 => write!(f, "iso-8859-3"),
+            Charset::ISO8859_4 => write!(f, "iso-8859-4"),
+            Charset::ISO8859_5 => write!(f, "iso-8859-5"),
+            Charset::ISO8859_6 => write!(f, "iso-8859-6"),
             Charset::ISO8859_7 => write!(f, "iso-8859-7"),
+            Charset::ISO8859_8 => write!(f, "iso-8859-8"),
+            Charset::ISO8859_10 => write!(f, "iso-8859-10"),
+            Charset::ISO8859_13 => write!(f, "iso-8859-13"),
+            Charset::ISO8859_14 => write!(f, "iso-8859-14"),
             Charset::ISO8859_15 => write!(f, "iso-8859-15"),
+            Charset::ISO8859_16 => write!(f, "iso-8859-16"),
             Charset::Windows1250 => write!(f, "windows-1250"),
             Charset::Windows1251 => write!(f, "windows-1251"),
             Charset::Windows1252 => write!(f, "windows-1252"),
             Charset::Windows1253 => write!(f, "windows-1253"),
-            Charset::GBK => write!(f, "GBK"),
+            Charset::GBK => write!(f, "gbk"),
             Charset::GB2312 => write!(f, "gb2312"),
-            Charset::BIG5 => write!(f, "BIG5"),
-            Charset::ISO2022JP => write!(f, "ISO-2022-JP"),
+            Charset::BIG5 => write!(f, "big5"),
+            Charset::ISO2022JP => write!(f, "iso-2022-jp"),
+            Charset::EUCJP => write!(f, "euc-jp"),
+            Charset::KOI8R => write!(f, "koi8-r"),
+            Charset::KOI8U => write!(f, "koi8-u"),
         }
     }
 }
@@ -135,6 +197,7 @@ impl Display for Charset {
 pub enum MultipartType {
     Alternative,
     Digest,
+    Encrypted,
     Mixed,
     Related,
     Signed,
@@ -148,13 +211,18 @@ impl Default for MultipartType {
 
 impl Display for MultipartType {
     fn fmt(&self, f: &mut Formatter) -> FmtResult {
-        match self {
-            MultipartType::Alternative => write!(f, "multipart/alternative"),
-            MultipartType::Digest => write!(f, "multipart/digest"),
-            MultipartType::Mixed => write!(f, "multipart/mixed"),
-            MultipartType::Related => write!(f, "multipart/related"),
-            MultipartType::Signed => write!(f, "multipart/signed"),
-        }
+        write!(
+            f,
+            "{}",
+            match self {
+                MultipartType::Alternative => "multipart/alternative",
+                MultipartType::Digest => "multipart/digest",
+                MultipartType::Encrypted => "multipart/encrypted",
+                MultipartType::Mixed => "multipart/mixed",
+                MultipartType::Related => "multipart/related",
+                MultipartType::Signed => "multipart/signed",
+            }
+        )
     }
 }
 
@@ -166,6 +234,8 @@ impl From<&[u8]> for MultipartType {
             MultipartType::Alternative
         } else if val.eq_ignore_ascii_case(b"digest") {
             MultipartType::Digest
+        } else if val.eq_ignore_ascii_case(b"encrypted") {
+            MultipartType::Encrypted
         } else if val.eq_ignore_ascii_case(b"signed") {
             MultipartType::Signed
         } else if val.eq_ignore_ascii_case(b"related") {
@@ -190,6 +260,7 @@ pub enum ContentType {
     },
     MessageRfc822,
     PGPSignature,
+    CMSSignature,
     Other {
         tag: Vec<u8>,
         name: Option<String>,
@@ -209,6 +280,75 @@ impl Default for ContentType {
     }
 }
 
+impl PartialEq<&str> for ContentType {
+    fn eq(&self, other: &&str) -> bool {
+        match (self, *other) {
+            (
+                ContentType::Text {
+                    kind: Text::Plain, ..
+                },
+                "text/plain",
+            ) => true,
+            (
+                ContentType::Text {
+                    kind: Text::Html, ..
+                },
+                "text/html",
+            ) => true,
+            (
+                ContentType::Multipart {
+                    kind: MultipartType::Alternative,
+                    ..
+                },
+                "multipart/alternative",
+            ) => true,
+            (
+                ContentType::Multipart {
+                    kind: MultipartType::Digest,
+                    ..
+                },
+                "multipart/digest",
+            ) => true,
+            (
+                ContentType::Multipart {
+                    kind: MultipartType::Encrypted,
+                    ..
+                },
+                "multipart/encrypted",
+            ) => true,
+            (
+                ContentType::Multipart {
+                    kind: MultipartType::Mixed,
+                    ..
+                },
+                "multipart/mixed",
+            ) => true,
+            (
+                ContentType::Multipart {
+                    kind: MultipartType::Related,
+                    ..
+                },
+                "multipart/related",
+            ) => true,
+            (
+                ContentType::Multipart {
+                    kind: MultipartType::Signed,
+                    ..
+                },
+                "multipart/signed",
+            ) => true,
+            (ContentType::PGPSignature, "application/pgp-signature") => true,
+            (ContentType::CMSSignature, "application/pkcs7-signature") => true,
+            (ContentType::MessageRfc822, "message/rfc822") => true,
+            (ContentType::Other { tag, .. }, _) => {
+                other.eq_ignore_ascii_case(&String::from_utf8_lossy(&tag))
+            }
+            (ContentType::OctetStream { .. }, "application/octet-stream") => true,
+            _ => false,
+        }
+    }
+}
+
 impl Display for ContentType {
     fn fmt(&self, f: &mut Formatter) -> FmtResult {
         match self {
@@ -216,6 +356,7 @@ impl Display for ContentType {
             ContentType::Multipart { kind: k, .. } => k.fmt(f),
             ContentType::Other { ref tag, .. } => write!(f, "{}", String::from_utf8_lossy(tag)),
             ContentType::PGPSignature => write!(f, "application/pgp-signature"),
+            ContentType::CMSSignature => write!(f, "application/pkcs7-signature"),
             ContentType::MessageRfc822 => write!(f, "message/rfc822"),
             ContentType::OctetStream { .. } => write!(f, "application/octet-stream"),
         }

melib/src/email/attachments.rs

@@ -19,6 +19,7 @@
  * along with meli. If not, see <http://www.gnu.org/licenses/>.
  */
 
+/*! Encoding/decoding of attachments */
 use crate::email::{
     address::StrBuilder,
     parser::{self, BytesExt},
@@ -201,6 +202,10 @@ impl AttachmentBuilder {
                     && cst.eq_ignore_ascii_case(b"pgp-signature")
                 {
                     self.content_type = ContentType::PGPSignature;
+                } else if ct.eq_ignore_ascii_case(b"application")
+                    && cst.eq_ignore_ascii_case(b"pkcs7-signature")
+                {
+                    self.content_type = ContentType::CMSSignature;
                 } else {
                     let mut name: Option<String> = None;
                     for (n, v) in params {
@@ -350,16 +355,14 @@ pub struct Attachment {
 
 impl fmt::Debug for Attachment {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "Attachment {{\n content_type: {:?},\n content_transfer_encoding: {:?},\n raw: Vec of {} bytes\n, body:\n{}\n}}",
-        self.content_type,
-        self.content_transfer_encoding,
-        self.raw.len(),
-        {
-            let mut text = Vec::with_capacity(4096);
-            self.get_text_recursive(&mut text);
-            std::str::from_utf8(&text).map(std::string::ToString::to_string).unwrap_or_else(|e| format!("Unicode error {}", e))
-        }
-        )
+        let mut text = Vec::with_capacity(4096);
+        self.get_text_recursive(&mut text);
+        f.debug_struct("Attachment")
+            .field("content_type", &self.content_type)
+            .field("content_transfer_encoding", &self.content_transfer_encoding)
+            .field("raw bytes length", &self.raw.len())
+            .field("body", &String::from_utf8_lossy(&text))
+            .finish()
     }
 }
 
@@ -370,44 +373,64 @@ impl fmt::Display for Attachment {
                 match Mail::new(self.body.display_bytes(&self.raw).to_vec(), None) {
                     Ok(wrapper) => write!(
                         f,
-                        "message/rfc822: {} - {} - {}",
+                        "{} - {} - {} [message/rfc822] {}",
                         wrapper.date(),
                         wrapper.field_from_to_string(),
-                        wrapper.subject()
+                        wrapper.subject(),
+                        crate::Bytes(self.raw.len()),
+                    ),
+                    Err(err) => write!(
+                        f,
+                        "could not parse: {} [message/rfc822] {}",
+                        err,
+                        crate::Bytes(self.raw.len()),
                     ),
-                    Err(e) => write!(f, "{}", e),
                 }
             }
-            ContentType::PGPSignature => write!(f, "pgp signature {}", self.mime_type()),
-            ContentType::OctetStream { ref name } => {
-                write!(f, "{}", name.clone().unwrap_or_else(|| self.mime_type()))
+            ContentType::PGPSignature => write!(f, "pgp signature [{}]", self.mime_type()),
+            ContentType::CMSSignature => write!(f, "S/MIME signature [{}]", self.mime_type()),
+            ContentType::OctetStream { .. } | ContentType::Other { .. } => {
+                if let Some(name) = self.filename() {
+                    write!(
+                        f,
+                        "\"{}\", [{}] {}",
+                        name,
+                        self.mime_type(),
+                        crate::Bytes(self.raw.len())
+                    )
+                } else {
+                    write!(
+                        f,
+                        "Data attachment [{}] {}",
+                        self.mime_type(),
+                        crate::Bytes(self.raw.len())
+                    )
+                }
             }
-            ContentType::Other {
-                name: Some(ref name),
-                ..
-            } => write!(f, "\"{}\", [{}]", name, self.mime_type()),
-            ContentType::Other { .. } => write!(f, "Data attachment of type {}", self.mime_type()),
-            ContentType::Text { ref parameters, .. }
-                if parameters
-                    .iter()
-                    .any(|(name, _)| name.eq_ignore_ascii_case(b"name")) =>
-            {
-                let name = String::from_utf8_lossy(
-                    parameters
-                        .iter()
-                        .find(|(name, _)| name.eq_ignore_ascii_case(b"name"))
-                        .map(|(_, value)| value)
-                        .unwrap(),
-                );
-                write!(f, "\"{}\", [{}]", name, self.mime_type())
+            ContentType::Text { .. } => {
+                if let Some(name) = self.filename() {
+                    write!(
+                        f,
+                        "\"{}\", [{}] {}",
+                        name,
+                        self.mime_type(),
+                        crate::Bytes(self.raw.len())
+                    )
+                } else {
+                    write!(
+                        f,
+                        "Text attachment [{}] {}",
+                        self.mime_type(),
+                        crate::Bytes(self.raw.len())
+                    )
+                }
             }
-            ContentType::Text { .. } => write!(f, "Text attachment of type {}", self.mime_type()),
             ContentType::Multipart {
                 parts: ref sub_att_vec,
                 ..
             } => write!(
                 f,
-                "{} attachment with {} subs",
+                "{} attachment with {} parts",
                 self.mime_type(),
                 sub_att_vec.len()
             ),
@@ -530,7 +553,7 @@ impl Attachment {
 
     fn get_text_recursive(&self, text: &mut Vec<u8>) {
         match self.content_type {
-            ContentType::Text { .. } | ContentType::PGPSignature => {
+            ContentType::Text { .. } | ContentType::PGPSignature | ContentType::CMSSignature => {
                 text.extend(decode(self, None));
             }
             ContentType::Multipart {
@@ -626,6 +649,16 @@ impl Attachment {
         }
     }
 
+    pub fn is_encrypted(&self) -> bool {
+        match self.content_type {
+            ContentType::Multipart {
+                kind: MultipartType::Encrypted,
+                ..
+            } => true,
+            _ => false,
+        }
+    }
+
     pub fn is_signed(&self) -> bool {
         match self.content_type {
             ContentType::Multipart {
@@ -640,7 +673,7 @@ impl Attachment {
         let mut ret = String::with_capacity(2 * self.raw.len());
         fn into_raw_helper(a: &Attachment, ret: &mut String) {
             ret.push_str(&format!(
-                "Content-Transfer-Encoding: {}\n",
+                "Content-Transfer-Encoding: {}\r\n",
                 a.content_transfer_encoding
             ));
             match &a.content_type {
@@ -666,7 +699,7 @@ impl Attachment {
                         }
                     }
 
-                    ret.push_str("\n\n");
+                    ret.push_str("\r\n\r\n");
                     ret.push_str(&String::from_utf8_lossy(a.body()));
                 }
                 ContentType::Multipart {
@@ -679,36 +712,36 @@ impl Attachment {
                     if *kind == MultipartType::Signed {
                         ret.push_str("; micalg=pgp-sha512; protocol=\"application/pgp-signature\"");
                     }
-                    ret.push('\n');
+                    ret.push_str("\r\n");
 
-                    let boundary_start = format!("\n--{}\n", boundary);
+                    let boundary_start = format!("\r\n--{}\r\n", boundary);
                     for p in parts {
                         ret.push_str(&boundary_start);
                         into_raw_helper(p, ret);
                     }
-                    ret.push_str(&format!("--{}--\n\n", boundary));
+                    ret.push_str(&format!("--{}--\r\n\r\n", boundary));
                 }
                 ContentType::MessageRfc822 => {
-                    ret.push_str(&format!("Content-Type: {}\n\n", a.content_type));
+                    ret.push_str(&format!("Content-Type: {}\r\n\r\n", a.content_type));
                     ret.push_str(&String::from_utf8_lossy(a.body()));
                 }
-                ContentType::PGPSignature => {
-                    ret.push_str(&format!("Content-Type: {}\n\n", a.content_type));
+                ContentType::CMSSignature | ContentType::PGPSignature => {
+                    ret.push_str(&format!("Content-Type: {}\r\n\r\n", a.content_type));
                     ret.push_str(&String::from_utf8_lossy(a.body()));
                 }
                 ContentType::OctetStream { ref name } => {
                     if let Some(name) = name {
                         ret.push_str(&format!(
-                            "Content-Type: {}; name={}\n\n",
+                            "Content-Type: {}; name={}\r\n\r\n",
                             a.content_type, name
                         ));
                     } else {
-                        ret.push_str(&format!("Content-Type: {}\n\n", a.content_type));
+                        ret.push_str(&format!("Content-Type: {}\r\n\r\n", a.content_type));
                     }
                     ret.push_str(&BASE64_MIME.encode(a.body()).trim());
                 }
                 _ => {
-                    ret.push_str(&format!("Content-Type: {}\n\n", a.content_type));
+                    ret.push_str(&format!("Content-Type: {}\r\n\r\n", a.content_type));
                     ret.push_str(&String::from_utf8_lossy(a.body()));
                 }
             }
@@ -751,9 +784,18 @@ impl Attachment {
                     h.eq_ignore_ascii_case(b"name") | h.eq_ignore_ascii_case(b"filename")
                 })
                 .map(|(_, v)| String::from_utf8_lossy(v).to_string()),
-            ContentType::Other { name, .. } | ContentType::OctetStream { name, .. } => name.clone(),
+            ContentType::Other { .. } | ContentType::OctetStream { .. } => {
+                self.content_type.name().map(|s| s.to_string())
+            }
             _ => None,
         })
+        .map(|s| {
+            crate::email::parser::encodings::phrase(s.as_bytes(), false)
+                .map(|(_, v)| v)
+                .ok()
+                .and_then(|n| String::from_utf8(n).ok())
+                .unwrap_or_else(|| s)
+        })
         .map(|n| n.replace(|c| std::path::is_separator(c) || c.is_ascii_control(), "_"))
     }
 }
@@ -762,25 +804,24 @@ pub fn interpret_format_flowed(_t: &str) -> String {
     unimplemented!()
 }
 
-fn decode_rfc822(_raw: &[u8]) -> Attachment {
-    // FIXME
-    let builder = AttachmentBuilder::new(b"message/rfc822 cannot be displayed");
-    builder.build()
-}
+type Filter<'a> = Box<dyn FnMut(&Attachment, &mut Vec<u8>) -> () + 'a>;
 
-type Filter<'a> = Box<dyn FnMut(&'a Attachment, &mut Vec<u8>) -> () + 'a>;
-
-fn decode_rec_helper<'a>(a: &'a Attachment, filter: &mut Option<Filter<'a>>) -> Vec<u8> {
+fn decode_rec_helper<'a, 'b>(a: &'a Attachment, filter: &mut Option<Filter<'b>>) -> Vec<u8> {
     match a.content_type {
         ContentType::Other { .. } => Vec::new(),
         ContentType::Text { .. } => decode_helper(a, filter),
         ContentType::OctetStream { ref name } => {
             name.clone().unwrap_or_else(|| a.mime_type()).into_bytes()
         }
-        ContentType::PGPSignature => Vec::new(),
+        ContentType::CMSSignature | ContentType::PGPSignature => Vec::new(),
         ContentType::MessageRfc822 => {
-            let temp = decode_rfc822(a.body());
-            decode_rec(&temp, None)
+            if a.content_disposition.kind.is_inline() {
+                let b = AttachmentBuilder::new(a.body()).build();
+                let ret = decode_rec_helper(&b, filter);
+                ret
+            } else {
+                b"message/rfc822 attachment".to_vec()
+            }
         }
         ContentType::Multipart {
             ref kind,
@@ -806,6 +847,16 @@ fn decode_rec_helper<'a>(a: &'a Attachment, filter: &mut Option<Filter<'a>>) ->
                 vec.extend(decode_helper(a, filter));
                 vec
             }
+            MultipartType::Encrypted => {
+                let mut vec = Vec::new();
+                for a in parts {
+                    if a.content_type == "application/octet-stream" {
+                        vec.extend(decode_rec_helper(a, filter));
+                    }
+                }
+                vec.extend(decode_helper(a, filter));
+                vec
+            }
             _ => {
                 let mut vec = Vec::new();
                 for a in parts {
@@ -819,11 +870,11 @@ fn decode_rec_helper<'a>(a: &'a Attachment, filter: &mut Option<Filter<'a>>) ->
     }
 }
 
-pub fn decode_rec<'a>(a: &'a Attachment, mut filter: Option<Filter<'a>>) -> Vec<u8> {
+pub fn decode_rec<'a, 'b>(a: &'a Attachment, mut filter: Option<Filter<'b>>) -> Vec<u8> {
     decode_rec_helper(a, &mut filter)
 }
 
-fn decode_helper<'a>(a: &'a Attachment, filter: &mut Option<Filter<'a>>) -> Vec<u8> {
+fn decode_helper<'a, 'b>(a: &'a Attachment, filter: &mut Option<Filter<'b>>) -> Vec<u8> {
     let charset = match a.content_type {
         ContentType::Text { charset: c, .. } => c,
         _ => Default::default(),
@@ -860,6 +911,6 @@ fn decode_helper<'a>(a: &'a Attachment, filter: &mut Option<Filter<'a>>) -> Vec<
     ret
 }
 
-pub fn decode<'a>(a: &'a Attachment, mut filter: Option<Filter<'a>>) -> Vec<u8> {
+pub fn decode<'a, 'b>(a: &'a Attachment, mut filter: Option<Filter<'b>>) -> Vec<u8> {
     decode_helper(a, &mut filter)
 }

melib/src/email/compose.rs

@@ -19,6 +19,7 @@
  * along with meli. If not, see <http://www.gnu.org/licenses/>.
  */
 
+/*! Compose a `Draft`, with MIME and attachment support */
 use super::*;
 use crate::email::attachment_types::{
     Charset, ContentTransferEncoding, ContentType, MultipartType,
@@ -52,7 +53,7 @@ impl Default for Draft {
         let mut headers = HeaderMap::default();
         headers.insert(
             HeaderName::new_unchecked("Date"),
-            crate::datetime::timestamp_to_string(crate::datetime::now(), None),
+            crate::datetime::timestamp_to_string(crate::datetime::now(), None, true),
         );
         headers.insert(HeaderName::new_unchecked("From"), "".into());
         headers.insert(HeaderName::new_unchecked("To"), "".into());
@@ -220,7 +221,7 @@ impl Draft {
         let mut ret = String::new();
 
         for (k, v) in self.headers.deref() {
-            ret.extend(format!("{}: {}\n", k, v).chars());
+            ret.push_str(&format!("{}: {}\n", k, v));
         }
 
         ret.push('\n');
@@ -245,9 +246,9 @@ impl Draft {
         }
         for (k, v) in self.headers.deref() {
             if v.is_ascii() {
-                ret.extend(format!("{}: {}\r\n", k, v).chars());
+                ret.push_str(&format!("{}: {}\r\n", k, v));
             } else {
-                ret.extend(format!("{}: {}\r\n", k, mime::encode_header(v)).chars());
+                ret.push_str(&format!("{}: {}\r\n", k, mime::encode_header(v)));
             }
         }
         ret.push_str("MIME-Version: 1.0\r\n");
@@ -255,19 +256,22 @@ impl Draft {
         if self.attachments.is_empty() {
             let content_type: ContentType = Default::default();
             let content_transfer_encoding: ContentTransferEncoding = ContentTransferEncoding::_8Bit;
-            ret.extend(format!("Content-Type: {}; charset=\"utf-8\"\r\n", content_type).chars());
-            ret.extend(
-                format!(
-                    "Content-Transfer-Encoding: {}\r\n",
-                    content_transfer_encoding
-                )
-                .chars(),
-            );
+            ret.push_str(&format!(
+                "Content-Type: {}; charset=\"utf-8\"\r\n",
+                content_type
+            ));
+            ret.push_str(&format!(
+                "Content-Transfer-Encoding: {}\r\n",
+                content_transfer_encoding
+            ));
             ret.push_str("\r\n");
-            ret.push_str(&self.body);
+            for line in self.body.lines() {
+                ret.push_str(line);
+                ret.push_str("\r\n");
+            }
         } else if self.body.is_empty() && self.attachments.len() == 1 {
             let attachment = std::mem::replace(&mut self.attachments, Vec::new()).remove(0);
-            print_attachment(&mut ret, &Default::default(), attachment);
+            print_attachment(&mut ret, attachment);
         } else {
             let mut parts = Vec::with_capacity(self.attachments.len() + 1);
             let attachments = std::mem::replace(&mut self.attachments, Vec::new());
@@ -286,28 +290,28 @@ impl Draft {
 
 fn build_multipart(ret: &mut String, kind: MultipartType, parts: Vec<AttachmentBuilder>) {
     let boundary = ContentType::make_boundary(&parts);
-    ret.extend(
-        format!(
-            "Content-Type: {}; charset=\"utf-8\"; boundary=\"{}\"\r\n",
-            kind, boundary
-        )
-        .chars(),
-    );
-    ret.push_str("\r\n");
+    ret.push_str(&format!(
+        r#"Content-Type: {}; charset="utf-8"; boundary="{}""#,
+        kind, boundary
+    ));
+    if kind == MultipartType::Encrypted {
+        ret.push_str(r#"; protocol="application/pgp-encrypted""#);
+    }
+    ret.push_str("\r\n\r\n");
     /* rfc1341 */
     ret.push_str("This is a MIME formatted message with attachments. Use a MIME-compliant client to view it properly.\r\n");
     for sub in parts {
         ret.push_str("--");
         ret.push_str(&boundary);
         ret.push_str("\r\n");
-        print_attachment(ret, &kind, sub);
+        print_attachment(ret, sub);
     }
     ret.push_str("--");
     ret.push_str(&boundary);
-    ret.push_str("--\n");
+    ret.push_str("--\r\n");
 }
 
-fn print_attachment(ret: &mut String, kind: &MultipartType, a: AttachmentBuilder) {
+fn print_attachment(ret: &mut String, a: AttachmentBuilder) {
     use ContentType::*;
     match a.content_type {
         ContentType::Text {
@@ -316,63 +320,91 @@ fn print_attachment(ret: &mut String, kind: &MultipartType, a: AttachmentBuilder
             parameters: ref v,
         } if v.is_empty() => {
             ret.push_str("\r\n");
-            ret.push_str(&String::from_utf8_lossy(a.raw()));
-            ret.push_str("\r\n");
+            for line in String::from_utf8_lossy(a.raw()).lines() {
+                ret.push_str(line);
+                ret.push_str("\r\n");
+            }
         }
         Text { .. } => {
-            ret.push_str(&a.build().into_raw());
-            ret.push_str("\r\n");
+            for line in a.build().into_raw().lines() {
+                ret.push_str(line);
+                ret.push_str("\r\n");
+            }
         }
         Multipart {
-            boundary: _boundary,
+            boundary: _,
             kind,
-            parts: subparts,
+            parts,
         } => {
             build_multipart(
                 ret,
                 kind,
-                subparts
+                parts
                     .into_iter()
                     .map(|s| s.into())
                     .collect::<Vec<AttachmentBuilder>>(),
             );
-            ret.push_str("\r\n");
         }
-        MessageRfc822 | PGPSignature => {
-            ret.push_str(&format!("Content-Type: {}; charset=\"utf-8\"\r\n", kind));
+        MessageRfc822 => {
+            ret.push_str(&format!(
+                "Content-Type: {}; charset=\"utf-8\"\r\n",
+                a.content_type
+            ));
             ret.push_str("Content-Disposition: attachment\r\n");
             ret.push_str("\r\n");
-            ret.push_str(&String::from_utf8_lossy(a.raw()));
+            for line in String::from_utf8_lossy(a.raw()).lines() {
+                ret.push_str(line);
+                ret.push_str("\r\n");
+            }
+        }
+        PGPSignature => {
+            ret.push_str(&format!(
+                "Content-Type: {}; charset=\"utf-8\"; name=\"signature.asc\"\r\n",
+                a.content_type
+            ));
+            ret.push_str("Content-Description: Digital signature\r\n");
+            ret.push_str("Content-Disposition: inline\r\n");
             ret.push_str("\r\n");
+            for line in String::from_utf8_lossy(a.raw()).lines() {
+                ret.push_str(line);
+                ret.push_str("\r\n");
+            }
         }
         _ => {
-            let content_transfer_encoding: ContentTransferEncoding =
-                ContentTransferEncoding::Base64;
-            if let Some(name) = a.content_type().name() {
-                ret.extend(
-                    format!(
-                        "Content-Type: {}; name=\"{}\"; charset=\"utf-8\"\r\n",
-                        a.content_type(),
-                        name
-                    )
-                    .chars(),
-                );
+            let content_transfer_encoding: ContentTransferEncoding = if a.raw().is_ascii() {
+                ContentTransferEncoding::_8Bit
             } else {
-                ret.extend(
-                    format!("Content-Type: {}; charset=\"utf-8\"\r\n", a.content_type()).chars(),
-                );
+                ContentTransferEncoding::Base64
+            };
+            if let Some(name) = a.content_type().name() {
+                ret.push_str(&format!(
+                    "Content-Type: {}; name=\"{}\"; charset=\"utf-8\"\r\n",
+                    a.content_type(),
+                    name
+                ));
+            } else {
+                ret.push_str(&format!(
+                    "Content-Type: {}; charset=\"utf-8\"\r\n",
+                    a.content_type()
+                ));
             }
             ret.push_str("Content-Disposition: attachment\r\n");
-            ret.extend(
-                format!(
-                    "Content-Transfer-Encoding: {}\r\n",
-                    content_transfer_encoding
-                )
-                .chars(),
-            );
-            ret.push_str("\r\n");
-            ret.push_str(&BASE64_MIME.encode(a.raw()).trim());
+            ret.push_str(&format!(
+                "Content-Transfer-Encoding: {}\r\n",
+                content_transfer_encoding
+            ));
             ret.push_str("\r\n");
+            if content_transfer_encoding == ContentTransferEncoding::Base64 {
+                for line in BASE64_MIME.encode(a.raw()).trim().lines() {
+                    ret.push_str(line);
+                    ret.push_str("\r\n");
+                }
+            } else {
+                for line in String::from_utf8_lossy(a.raw()).lines() {
+                    ret.push_str(line);
+                    ret.push_str("\r\n");
+                }
+            }
         }
     }
 }

melib/src/email/compose/mime.rs

@@ -179,7 +179,7 @@ fn test_encode_header() {
         )
         .unwrap(),
     );
-    let words = "[Advcomparch] =?utf-8?b?zqPPhc68z4DOtc+BzrnPhs6/z4HOrCDPg861IGZs?=\n\t=?utf-8?b?dXNoIM67z4zOs8+JIG1pc3ByZWRpY3Rpb24gzrrOsc+Ezqwgz4TOt869?=\n\t=?utf-8?b?IM61zrrPhM6tzrvOtc+Dzrcgc3RvcmU=?=";
+    //let words = "[Advcomparch] =?utf-8?b?zqPPhc68z4DOtc+BzrnPhs6/z4HOrCDPg861IGZs?=\n\t=?utf-8?b?dXNoIM67z4zOs8+JIG1pc3ByZWRpY3Rpb24gzrrOsc+Ezqwgz4TOt869?=\n\t=?utf-8?b?IM61zrrPhM6tzrvOtc+Dzrcgc3RvcmU=?=";
     let words_enc = "[Advcomparch] Συμπεριφορά σε flush λόγω misprediction κατά την εκτέλεση store";
     assert_eq!(
         "[Advcomparch] Συμπεριφορά σε flush λόγω misprediction κατά την εκτέλεση store",

melib/src/email/headers.rs

@@ -19,6 +19,7 @@
  * along with meli. If not, see <http://www.gnu.org/licenses/>.
  */
 
+/*! Wrapper type `HeaderName` for case-insensitive comparisons */
 use crate::error::MeliError;
 use indexmap::IndexMap;
 use smallvec::SmallVec;

melib/src/email/list_management.rs

@@ -19,15 +19,18 @@
  * along with meli. If not, see <http://www.gnu.org/licenses/>.
  */
 
+/*! Parsing of rfc2369/rfc2919 `List-*` headers */
 use super::parser;
 use super::Envelope;
 use smallvec::SmallVec;
 use std::convert::From;
 
-#[derive(Debug, Copy)]
+#[derive(Debug, PartialEq, Clone, Copy)]
 pub enum ListAction<'a> {
     Url(&'a [u8]),
     Email(&'a [u8]),
+    ///`List-Post` field may contain the special value "NO".
+    No,
 }
 
 impl<'a> From<&'a [u8]> for ListAction<'a> {
@@ -37,6 +40,8 @@ impl<'a> From<&'a [u8]> for ListAction<'a> {
              * parser::mailto() will handle this if user tries to unsubscribe.
              */
             ListAction::Email(value)
+        } else if value.starts_with(b"NO") {
+            ListAction::No
         } else {
             /* Otherwise treat it as url. There's no foolproof way to check if this is valid, so
              * postpone it until we try an HTTP request.
@@ -68,15 +73,6 @@ impl<'a> ListAction<'a> {
     }
 }
 
-impl<'a> Clone for ListAction<'a> {
-    fn clone(&self) -> Self {
-        match self {
-            ListAction::Url(a) => ListAction::Url(<&[u8]>::clone(a)),
-            ListAction::Email(a) => ListAction::Email(<&[u8]>::clone(a)),
-        }
-    }
-}
-
 #[derive(Default, Debug)]
 pub struct ListActions<'a> {
     pub id: Option<&'a str>,
@@ -128,6 +124,11 @@ impl<'a> ListActions<'a> {
 
         if let Some(post) = envelope.other_headers().get("List-Post") {
             ret.post = ListAction::parse_options_list(post.as_bytes());
+            if let Some(ref l) = ret.post {
+                if l.starts_with(&[ListAction::No]) {
+                    ret.post = None;
+                }
+            }
         }
 
         if let Some(unsubscribe) = envelope.other_headers().get("List-Unsubscribe") {

melib/src/email/mailto.rs

@@ -19,6 +19,7 @@
  * along with meli. If not, see <http://www.gnu.org/licenses/>.
  */
 
+/*! Parsing of `mailto` addresses */
 use super::*;
 use std::convert::TryFrom;
 

melib/src/email/parser.rs

@@ -19,6 +19,7 @@
  * along with meli. If not, see <http://www.gnu.org/licenses/>.
  */
 
+/*! Parsers for email. See submodules */
 use crate::error::{MeliError, Result, ResultIntoMeliError};
 use nom::{
     branch::alt,
@@ -281,6 +282,7 @@ pub fn mail(input: &[u8]) -> Result<(Vec<(&[u8], &[u8])>, &[u8])> {
 }
 
 pub mod dates {
+    /*! Date values in headers */
     use super::generic::*;
     use super::*;
     use crate::datetime::UnixTimestamp;
@@ -505,6 +507,7 @@ pub mod dates {
 }
 
 pub mod generic {
+    /*! Generally useful parser combinators */
     use super::*;
     #[inline(always)]
     pub fn byte_in_slice<'a>(slice: &'static [u8]) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], u8> {
@@ -1172,7 +1175,7 @@ pub mod mailing_lists {
                     map(tag("NO"), |_| ()),
                     map(opt(cfws), |_| ()),
                 ),
-                |_| vec![],
+                |_| vec![&b"NO"[..]],
             ),
         ))(input)?;
         let (input, _) = opt(cfws)(input)?;
@@ -1197,14 +1200,15 @@ List-Archive: <http://www.host.com/list/archive/> (Web Archive)
 "#;
         let (rest, headers) = headers::headers(s.as_bytes()).unwrap();
         assert!(rest.is_empty());
-        for (h, v) in headers {
-            let (rest, action_list) = rfc_2369_list_headers_action_list(v).unwrap();
+        for (_h, v) in headers {
+            let (rest, _action_list) = rfc_2369_list_headers_action_list(v).unwrap();
             assert!(rest.is_empty());
         }
     }
 }
 
 pub mod headers {
+    /*! Email headers */
     use super::*;
 
     pub fn headers(input: &[u8]) -> IResult<&[u8], Vec<(&[u8], &[u8])>> {
@@ -1465,6 +1469,7 @@ pub mod headers {
 }
 
 pub mod attachments {
+    /*! Email attachments */
     use super::*;
     use crate::email::address::*;
     use crate::email::attachment_types::{ContentDisposition, ContentDispositionKind};
@@ -1731,6 +1736,7 @@ pub mod attachments {
 }
 
 pub mod encodings {
+    /*! Email encodings (quoted printable, MIME) */
     use super::*;
     use crate::email::attachment_types::Charset;
     use data_encoding::BASE64_MIME;
@@ -1871,18 +1877,33 @@ pub mod encodings {
             Charset::UTF8 | Charset::Ascii => Ok(String::from_utf8_lossy(s).to_string()),
             Charset::ISO8859_1 => Ok(ISO_8859_1.decode(s, DecoderTrap::Strict)?),
             Charset::ISO8859_2 => Ok(ISO_8859_2.decode(s, DecoderTrap::Strict)?),
+            Charset::ISO8859_3 => Ok(ISO_8859_3.decode(s, DecoderTrap::Strict)?),
+            Charset::ISO8859_4 => Ok(ISO_8859_4.decode(s, DecoderTrap::Strict)?),
+            Charset::ISO8859_5 => Ok(ISO_8859_5.decode(s, DecoderTrap::Strict)?),
+            Charset::ISO8859_6 => Ok(ISO_8859_6.decode(s, DecoderTrap::Strict)?),
             Charset::ISO8859_7 => Ok(ISO_8859_7.decode(s, DecoderTrap::Strict)?),
+            Charset::ISO8859_8 => Ok(ISO_8859_8.decode(s, DecoderTrap::Strict)?),
+            Charset::ISO8859_10 => Ok(ISO_8859_10.decode(s, DecoderTrap::Strict)?),
+            Charset::ISO8859_13 => Ok(ISO_8859_13.decode(s, DecoderTrap::Strict)?),
+            Charset::ISO8859_14 => Ok(ISO_8859_14.decode(s, DecoderTrap::Strict)?),
             Charset::ISO8859_15 => Ok(ISO_8859_15.decode(s, DecoderTrap::Strict)?),
+            Charset::ISO8859_16 => Ok(ISO_8859_16.decode(s, DecoderTrap::Strict)?),
             Charset::GBK => Ok(GBK.decode(s, DecoderTrap::Strict)?),
             Charset::Windows1250 => Ok(WINDOWS_1250.decode(s, DecoderTrap::Strict)?),
             Charset::Windows1251 => Ok(WINDOWS_1251.decode(s, DecoderTrap::Strict)?),
             Charset::Windows1252 => Ok(WINDOWS_1252.decode(s, DecoderTrap::Strict)?),
             Charset::Windows1253 => Ok(WINDOWS_1253.decode(s, DecoderTrap::Strict)?),
-            // Unimplemented:
-            Charset::GB2312 => Ok(String::from_utf8_lossy(s).to_string()),
-            Charset::UTF16 => Ok(String::from_utf8_lossy(s).to_string()),
-            Charset::BIG5 => Ok(String::from_utf8_lossy(s).to_string()),
-            Charset::ISO2022JP => Ok(String::from_utf8_lossy(s).to_string()),
+            Charset::KOI8R => Ok(KOI8_R.decode(s, DecoderTrap::Strict)?),
+            Charset::KOI8U => Ok(KOI8_U.decode(s, DecoderTrap::Strict)?),
+            Charset::BIG5 => Ok(BIG5_2003.decode(s, DecoderTrap::Strict)?),
+            Charset::GB2312 => {
+                Ok(encoding::codec::simpchinese::GBK_ENCODING.decode(s, DecoderTrap::Strict)?)
+            }
+            Charset::UTF16 => {
+                Ok(encoding::codec::utf_16::UTF_16LE_ENCODING.decode(s, DecoderTrap::Strict)?)
+            }
+            Charset::ISO2022JP => Ok(ISO_2022_JP.decode(s, DecoderTrap::Strict)?),
+            Charset::EUCJP => Ok(EUC_JP.decode(s, DecoderTrap::Strict)?),
         }
     }
 
@@ -2465,7 +2486,7 @@ pub mod address {
 
 #[cfg(test)]
 mod tests {
-    use super::{address::*, encodings::*, generic::*, *};
+    use super::{address::*, encodings::*, *};
     use crate::email::address::*;
     use crate::make_address;
 

melib/src/email/pgp.rs

@@ -19,14 +19,17 @@
  * along with meli. If not, see <http://www.gnu.org/licenses/>.
  */
 
-use crate::email::parser::BytesExt;
+/*! Verification of OpenPGP signatures */
 use crate::email::{
     attachment_types::{ContentType, MultipartType},
     attachments::Attachment,
 };
 use crate::{MeliError, Result};
 
-/// rfc3156
+/// Convert raw attachment to the form needed for signature verification ([rfc3156](https://tools.ietf.org/html/rfc3156))
+///
+/// ## rfc3156
+/// ```text
 /// Upon receipt of a signed message, an application MUST:
 ///
 ///   (1)   Convert line endings to the canonical <CR><LF> sequence before
@@ -35,7 +38,7 @@ use crate::{MeliError, Result};
 ///   (2)   Pass both the signed data and its associated content headers
 ///         along with the OpenPGP signature to the signature verification
 ///         service.
-///
+/// ```
 pub fn convert_attachment_to_rfc_spec(input: &[u8]) -> Vec<u8> {
     if input.is_empty() {
         return Vec::new();
@@ -84,7 +87,7 @@ pub fn convert_attachment_to_rfc_spec(input: &[u8]) -> Vec<u8> {
     ret
 }
 
-pub fn verify_signature(a: &Attachment) -> Result<(Vec<u8>, &[u8])> {
+pub fn verify_signature(a: &Attachment) -> Result<(Vec<u8>, &Attachment)> {
     match a.content_type {
         ContentType::Multipart {
             kind: MultipartType::Signed,
@@ -103,7 +106,10 @@ pub fn verify_signature(a: &Attachment) -> Result<(Vec<u8>, &[u8])> {
             let signed_part: Vec<u8> = if let Some(v) = parts
                 .iter()
                 .zip(part_boundaries.iter())
-                .find(|(p, _)| p.content_type != ContentType::PGPSignature)
+                .find(|(p, _)| {
+                    p.content_type != ContentType::PGPSignature
+                        && p.content_type != ContentType::CMSSignature
+                })
                 .map(|(_, s)| convert_attachment_to_rfc_spec(s.display_bytes(a.body())))
             {
                 v
@@ -112,12 +118,11 @@ pub fn verify_signature(a: &Attachment) -> Result<(Vec<u8>, &[u8])> {
                     "multipart/signed attachment without a signed part".to_string(),
                 ));
             };
-            let signature = if let Some(sig) = parts
-                .iter()
-                .find(|s| s.content_type == ContentType::PGPSignature)
-                .map(|a| a.body())
-            {
-                sig.trim()
+            let signature = if let Some(sig) = parts.iter().find(|s| {
+                s.content_type == ContentType::PGPSignature
+                    || s.content_type == ContentType::CMSSignature
+            }) {
+                sig
             } else {
                 return Err(MeliError::new(
                     "multipart/signed attachment without a signature part".to_string(),
@@ -125,8 +130,29 @@ pub fn verify_signature(a: &Attachment) -> Result<(Vec<u8>, &[u8])> {
             };
             Ok((signed_part, signature))
         }
-        _ => {
-            unreachable!("Should not give non-signed attachments to this function");
-        }
+        _ => Err(MeliError::new(
+            "Should not give non-signed attachments to this function",
+        )),
     }
 }
+
+#[derive(Debug, Clone, Default)]
+pub struct DecryptionMetadata {
+    pub recipients: Vec<Recipient>,
+    pub file_name: Option<String>,
+    pub session_key: Option<String>,
+    pub is_mime: bool,
+}
+
+#[derive(Debug, Clone)]
+pub struct Recipient {
+    pub keyid: Option<String>,
+    pub status: Result<()>,
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct SignatureMetadata {
+    pub signatures: Vec<Recipient>,
+    pub file_name: Option<String>,
+    pub is_mime: bool,
+}

melib/src/gpgme/io.rs

@@ -0,0 +1,155 @@
+/*
+ * melib - gpgme module
+ *
+ * Copyright 2020 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+use std::io::{self, Read, Seek, Write};
+
+#[repr(C)]
+struct TagData {
+    idx: usize,
+    fd: ::std::os::raw::c_int,
+    io_state: Arc<Mutex<IoState>>,
+}
+
+pub unsafe extern "C" fn gpgme_register_io_cb(
+    data: *mut ::std::os::raw::c_void,
+    fd: ::std::os::raw::c_int,
+    dir: ::std::os::raw::c_int,
+    fnc: gpgme_io_cb_t,
+    fnc_data: *mut ::std::os::raw::c_void,
+    tag: *mut *mut ::std::os::raw::c_void,
+) -> gpgme_error_t {
+    let io_state: Arc<Mutex<IoState>> = Arc::from_raw(data as *const _);
+    let io_state_copy = io_state.clone();
+    let mut io_state_lck = io_state.lock().unwrap();
+    let idx = io_state_lck.max_idx;
+    io_state_lck.max_idx += 1;
+    let (sender, receiver) = smol::channel::unbounded();
+    let gpgfd = GpgmeFd {
+        fd,
+        fnc,
+        fnc_data,
+        idx,
+        write: dir == 0,
+        sender,
+        receiver,
+        io_state: io_state_copy.clone(),
+    };
+    let tag_data = Arc::into_raw(Arc::new(TagData {
+        idx,
+        fd,
+        io_state: io_state_copy,
+    }));
+    core::ptr::write(tag, tag_data as *mut _);
+    io_state_lck.ops.insert(idx, gpgfd);
+    drop(io_state_lck);
+    let _ = Arc::into_raw(io_state);
+    0
+}
+
+pub unsafe extern "C" fn gpgme_remove_io_cb(tag: *mut ::std::os::raw::c_void) {
+    let tag_data: Arc<TagData> = Arc::from_raw(tag as *const _);
+    let mut io_state_lck = tag_data.io_state.lock().unwrap();
+    let fd = io_state_lck.ops.remove(&tag_data.idx).unwrap();
+    fd.sender.try_send(()).unwrap();
+    drop(io_state_lck);
+    let _ = Arc::into_raw(tag_data);
+}
+
+pub unsafe extern "C" fn gpgme_event_io_cb(
+    data: *mut ::std::os::raw::c_void,
+    type_: gpgme_event_io_t,
+    type_data: *mut ::std::os::raw::c_void,
+) {
+    if type_ == gpgme_event_io_t_GPGME_EVENT_DONE {
+        let err = type_data as gpgme_io_event_done_data_t;
+        let io_state: Arc<Mutex<IoState>> = Arc::from_raw(data as *const _);
+        let io_state_lck = io_state.lock().unwrap();
+        io_state_lck.sender.try_send(()).unwrap();
+        *io_state_lck.done.lock().unwrap() = Some(gpgme_error_try(&io_state_lck.lib, (*err).err));
+        drop(io_state_lck);
+        let _ = Arc::into_raw(io_state);
+    } else if type_ == gpgme_event_io_t_GPGME_EVENT_NEXT_KEY {
+        if let Some(inner) = core::ptr::NonNull::new(type_data as gpgme_key_t) {
+            let io_state: Arc<Mutex<IoState>> = Arc::from_raw(data as *const _);
+            let io_state_lck = io_state.lock().unwrap();
+            io_state_lck
+                .key_sender
+                .try_send(KeyInner { inner })
+                .unwrap();
+            drop(io_state_lck);
+            let _ = Arc::into_raw(io_state);
+        }
+    }
+}
+
+impl Read for Data {
+    #[inline]
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        let result = unsafe {
+            let (buf, len) = (buf.as_mut_ptr() as *mut _, buf.len());
+            call!(self.lib, gpgme_data_read)(self.inner.as_ptr(), buf, len)
+        };
+        if result >= 0 {
+            Ok(result as usize)
+        } else {
+            Err(io::Error::last_os_error().into())
+        }
+    }
+}
+
+impl Write for Data {
+    #[inline]
+    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
+        let result = unsafe {
+            let (buf, len) = (buf.as_ptr() as *const _, buf.len());
+            call!(self.lib, gpgme_data_write)(self.inner.as_ptr(), buf, len)
+        };
+        if result >= 0 {
+            Ok(result as usize)
+        } else {
+            Err(io::Error::last_os_error().into())
+        }
+    }
+
+    #[inline]
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}
+
+impl Seek for Data {
+    #[inline]
+    fn seek(&mut self, pos: io::SeekFrom) -> io::Result<u64> {
+        use std::convert::TryInto;
+        let (off, whence) = match pos {
+            io::SeekFrom::Start(off) => (off.try_into().unwrap_or(i64::MAX), libc::SEEK_SET),
+            io::SeekFrom::End(off) => (off.saturating_abs(), libc::SEEK_END),
+            io::SeekFrom::Current(off) => (off, libc::SEEK_CUR),
+        };
+        let result = unsafe { call!(self.lib, gpgme_data_seek)(self.inner.as_ptr(), off, whence) };
+        if result >= 0 {
+            Ok(result as u64)
+        } else {
+            Err(io::Error::last_os_error().into())
+        }
+    }
+}

melib/src/lib.rs

@@ -41,7 +41,11 @@ pub mod dbg {
         () => {
             eprint!(
                 "[{}][{:?}] {}:{}_{}: ",
-                crate::datetime::timestamp_to_string(crate::datetime::now(), Some("%Y-%m-%d %T")),
+                crate::datetime::timestamp_to_string(
+                    crate::datetime::now(),
+                    Some("%Y-%m-%d %T"),
+                    false
+                ),
                 std::thread::current()
                     .name()
                     .map(std::string::ToString::to_string)
@@ -119,6 +123,8 @@ pub mod connections;
 pub mod parsec;
 pub mod search;
 
+#[cfg(feature = "gpgme")]
+pub mod gpgme;
 #[cfg(feature = "smtp")]
 pub mod smtp;
 #[cfg(feature = "sqlite3")]
@@ -140,6 +146,35 @@ pub extern crate smol;
 pub extern crate uuid;
 pub extern crate xdg_utils;
 
+#[derive(Debug, Copy, Clone)]
+pub struct Bytes(pub usize);
+
+impl Bytes {
+    pub const KILOBYTE: f64 = 1024.0;
+    pub const MEGABYTE: f64 = Self::KILOBYTE * 1024.0;
+    pub const GIGABYTE: f64 = Self::MEGABYTE * 1024.0;
+    pub const PETABYTE: f64 = Self::GIGABYTE * 1024.0;
+}
+
+impl core::fmt::Display for Bytes {
+    fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
+        let bytes: f64 = self.0 as f64;
+        if bytes == 0.0 {
+            write!(fmt, "0")
+        } else if bytes < Self::KILOBYTE {
+            write!(fmt, "{:.2} bytes", bytes)
+        } else if bytes < Self::MEGABYTE {
+            write!(fmt, "{:.2} KiB", bytes / Self::KILOBYTE)
+        } else if bytes < Self::GIGABYTE {
+            write!(fmt, "{:.2} MiB", bytes / Self::MEGABYTE)
+        } else if bytes < Self::PETABYTE {
+            write!(fmt, "{:.2} GiB", bytes / Self::GIGABYTE)
+        } else {
+            write!(fmt, "{:.2} PiB", bytes / Self::PETABYTE)
+        }
+    }
+}
+
 pub use shellexpand::ShellExpandTrait;
 pub mod shellexpand {
 

melib/src/logging.rs

@@ -85,7 +85,8 @@ pub fn log<S: AsRef<str>>(val: S, level: LoggingLevel) {
         if level <= b.level {
             b.dest
                 .write_all(
-                    crate::datetime::timestamp_to_string(crate::datetime::now(), None).as_bytes(),
+                    crate::datetime::timestamp_to_string(crate::datetime::now(), None, false)
+                        .as_bytes(),
                 )
                 .unwrap();
             b.dest.write_all(b" [").unwrap();

melib/src/smtp.rs

@@ -50,7 +50,11 @@
  *        require_auth: true,
  *    },
  *};
- *std::thread::spawn(|| smol::run(futures::future::pending::<()>()));
+ *
+ *std::thread::Builder::new().spawn(move || {
+ *    let ex = smol::Executor::new();
+ *    futures::executor::block_on(ex.run(futures::future::pending::<()>()));
+ *}).unwrap();
  *
  *let mut conn = futures::executor::block_on(SmtpConnection::new_connection(conf)).unwrap();
  *futures::executor::block_on(conn.mail_transaction(r#"To: l10@mail.gr
@@ -73,7 +77,7 @@ use crate::error::{MeliError, Result, ResultIntoMeliError};
 use futures::io::{AsyncReadExt, AsyncWriteExt};
 use native_tls::TlsConnector;
 use smallvec::SmallVec;
-use smol::blocking;
+use smol::unblock;
 use smol::Async as AsyncWrapper;
 use std::borrow::Cow;
 use std::convert::TryFrom;
@@ -133,10 +137,24 @@ pub enum SmtpAuth {
         password: Password,
         #[serde(default = "true_val")]
         require_auth: bool,
+        #[serde(skip_serializing, skip_deserializing, default)]
+        auth_type: SmtpAuthType,
+    },
+    #[serde(alias = "xoauth2")]
+    XOAuth2 {
+        token_command: String,
+        #[serde(default = "true_val")]
+        require_auth: bool,
     },
     // md5, sasl, etc
 }
 
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Default)]
+pub struct SmtpAuthType {
+    plain: bool,
+    login: bool,
+}
+
 fn true_val() -> bool {
     true
 }
@@ -150,7 +168,7 @@ impl SmtpAuth {
         use SmtpAuth::*;
         match self {
             None => false,
-            Auto { require_auth, .. } => *require_auth,
+            Auto { require_auth, .. } | XOAuth2 { require_auth, .. } => *require_auth,
         }
     }
 }
@@ -310,7 +328,9 @@ impl SmtpConnection {
                     let _path = path.clone();
 
                     socket.set_nonblocking(false)?;
-                    let conn_result = blocking!(connector.connect(&_path, socket));
+                    let conn = unblock(move || connector.connect(&_path, socket))
+                        .await
+                        .chain_err_kind(crate::error::ErrorKind::Network)?;
                     /*
                     if let Err(native_tls::HandshakeError::WouldBlock(midhandshake_stream)) =
                         conn_result
@@ -332,10 +352,8 @@ impl SmtpConnection {
                         }
                     }
                         */
-                    AsyncWrapper::new(Connection::Tls(
-                        conn_result.chain_err_kind(crate::error::ErrorKind::Network)?,
-                    ))
-                    .chain_err_kind(crate::error::ErrorKind::Network)?
+                    AsyncWrapper::new(Connection::Tls(conn))
+                        .chain_err_kind(crate::error::ErrorKind::Network)?
                 };
                 ret.write_all(b"EHLO meli.delivery\r\n")
                     .await
@@ -392,38 +410,56 @@ impl SmtpConnection {
                 return Err(MeliError::new(format!(
                         "SMTP Server doesn't advertise Authentication support. Server response was: {:?}",
                         pre_auth_extensions_reply
-                    )));
+                    )).set_kind(crate::error::ErrorKind::Authentication));
             }
             no_auth_needed =
                 ret.server_conf.auth == SmtpAuth::None || !ret.server_conf.auth.require_auth();
             if no_auth_needed {
                 ret.set_extension_support(pre_auth_extensions_reply);
+            } else if let SmtpAuth::Auto {
+                ref mut auth_type, ..
+            } = ret.server_conf.auth
+            {
+                for l in pre_auth_extensions_reply
+                    .lines
+                    .iter()
+                    .filter(|l| l.starts_with("AUTH"))
+                {
+                    let l = l["AUTH ".len()..].trim();
+                    for _type in l.split_whitespace() {
+                        if _type == "PLAIN" {
+                            auth_type.plain = true;
+                        } else if _type == "LOGIN" {
+                            auth_type.login = true;
+                        }
+                    }
+                    break;
+                }
             }
         }
         if !no_auth_needed {
             match &ret.server_conf.auth {
                 SmtpAuth::None => {}
                 SmtpAuth::Auto {
-                    username, password, ..
+                    username,
+                    password,
+                    auth_type,
+                    ..
                 } => {
-                    // # RFC 4616 The PLAIN SASL Mechanism
-                    // # https://www.ietf.org/rfc/rfc4616.txt
-                    // message   = [authzid] UTF8NUL authcid UTF8NUL passwd
-                    // authcid   = 1*SAFE ; MUST accept up to 255 octets
-                    // authzid   = 1*SAFE ; MUST accept up to 255 octets
-                    // passwd    = 1*SAFE ; MUST accept up to 255 octets
-                    // UTF8NUL   = %x00 ; UTF-8 encoded NUL character
-                    let username_password = match password {
-                        Password::Raw(p) => base64::encode(format!("\0{}\0{}", username, p)),
+                    let password = match password {
+                        Password::Raw(p) => p.as_bytes().to_vec(),
                         Password::CommandEval(command) => {
                             let _command = command.clone();
 
-                            let mut output = blocking!(Command::new("sh")
-                                .args(&["-c", &_command])
-                                .stdin(std::process::Stdio::piped())
-                                .stdout(std::process::Stdio::piped())
-                                .stderr(std::process::Stdio::piped())
-                                .output())?;
+                            let mut output = unblock(move || {
+                                Command::new("sh")
+                                    .args(&["-c", &_command])
+                                    .stdin(std::process::Stdio::piped())
+                                    .stdout(std::process::Stdio::piped())
+                                    .stderr(std::process::Stdio::piped())
+                                    .output()
+                            })
+                            .await?;
                             if !output.status.success() {
                                 return Err(MeliError::new(format!(
                                     "SMTP password evaluation command `{}` returned {}: {}",
@@ -432,22 +468,77 @@ impl SmtpConnection {
                                     String::from_utf8_lossy(&output.stderr)
                                 )));
                             }
-                            let mut buf =
-                                Vec::with_capacity(2 + username.len() + output.stdout.len());
-                            buf.push(b'\0');
-                            buf.extend(username.as_bytes().to_vec());
-                            buf.push(b'\0');
                             if output.stdout.ends_with(b"\n") {
                                 output.stdout.pop();
                             }
-                            buf.extend(output.stdout);
-                            base64::encode(buf)
+                            output.stdout
                         }
                     };
-                    let mut auth_command: SmallVec<[&[u8]; 16]> = SmallVec::new();
-                    auth_command.push(b"AUTH PLAIN ");
-                    auth_command.push(username_password.as_bytes());
-                    ret.send_command(&auth_command).await?;
+                    if auth_type.login {
+                        let username = username.to_string();
+                        ret.send_command(&[b"AUTH LOGIN"]).await?;
+                        ret.read_lines(&mut res, Some((ReplyCode::_334, &[])))
+                            .await
+                            .chain_err_kind(crate::error::ErrorKind::Authentication)?;
+                        let buf = base64::encode(&username);
+                        ret.send_command(&[buf.as_bytes()]).await?;
+                        ret.read_lines(&mut res, Some((ReplyCode::_334, &[])))
+                            .await
+                            .chain_err_kind(crate::error::ErrorKind::Authentication)?;
+                        let buf = base64::encode(&password);
+                        ret.send_command(&[buf.as_bytes()]).await?;
+                    } else {
+                        // # RFC 4616 The PLAIN SASL Mechanism
+                        // # https://www.ietf.org/rfc/rfc4616.txt
+                        // message   = [authzid] UTF8NUL authcid UTF8NUL passwd
+                        // authcid   = 1*SAFE ; MUST accept up to 255 octets
+                        // authzid   = 1*SAFE ; MUST accept up to 255 octets
+                        // passwd    = 1*SAFE ; MUST accept up to 255 octets
+                        // UTF8NUL   = %x00 ; UTF-8 encoded NUL character
+                        let username_password = {
+                            let mut buf = Vec::with_capacity(2 + username.len() + password.len());
+                            buf.push(b'\0');
+                            buf.extend(username.as_bytes().to_vec());
+                            buf.push(b'\0');
+                            buf.extend(password);
+                            base64::encode(buf)
+                        };
+                        ret.send_command(&[b"AUTH PLAIN ", username_password.as_bytes()])
+                            .await?;
+                    }
+                    ret.read_lines(&mut res, Some((ReplyCode::_235, &[])))
+                        .await
+                        .chain_err_kind(crate::error::ErrorKind::Authentication)?;
+                    ret.send_command(&[b"EHLO meli.delivery"]).await?;
+                }
+                SmtpAuth::XOAuth2 { token_command, .. } => {
+                    let password_token = {
+                        let _token_command = token_command.clone();
+                        let mut output = unblock(move || {
+                            Command::new("sh")
+                                .args(&["-c", &_token_command])
+                                .stdin(std::process::Stdio::piped())
+                                .stdout(std::process::Stdio::piped())
+                                .stderr(std::process::Stdio::piped())
+                                .output()
+                        })
+                        .await?;
+                        if !output.status.success() {
+                            return Err(MeliError::new(format!(
+                                "SMTP XOAUTH2 token evaluation command `{}` returned {}: {}",
+                                &token_command,
+                                output.status,
+                                String::from_utf8_lossy(&output.stderr)
+                            )));
+                        }
+                        if output.stdout.ends_with(b"\n") {
+                            output.stdout.pop();
+                        }
+                        output.stdout
+                    };
+                    // https://developers.google.com/gmail/imap/xoauth2-protocol#smtp_protocol_exchange
+                    ret.send_command(&[b"AUTH XOAUTH2 ", &password_token])
+                        .await?;
                     ret.read_lines(&mut res, Some((ReplyCode::_235, &[])))
                         .await
                         .chain_err_kind(crate::error::ErrorKind::Authentication)?;
@@ -565,7 +656,7 @@ impl SmtpConnection {
             current_command.push(b"RCPT TO:<");
             current_command.push(addr.address_spec_raw().trim());
             if let Some(dsn_notify) = dsn_notify.as_ref() {
-                current_command.push(b" NOTIFY=");
+                current_command.push(b"> NOTIFY=");
                 current_command.push(dsn_notify.as_bytes());
             } else {
                 current_command.push(b">");
@@ -704,6 +795,8 @@ pub enum ReplyCode {
     _251,
     ///Cannot VRFY user, but will accept message and attempt delivery (See Section 3.5.3)
     _252,
+    ///rfc4954 AUTH continuation request
+    _334,
     ///PRDR specific, eg "content analysis has started|
     _353,
     ///Start mail input; end with <CRLF>.<CRLF>
@@ -758,6 +851,7 @@ impl ReplyCode {
             _235 => "Authentication successful",
             _251 => "User not local; will forward",
             _252 => "Cannot VRFY user, but will accept message and attempt delivery",
+            _334 => "Intermediate response to the AUTH command",
             _353 => "PRDR specific notice",
             _354 => "Start mail input; end with <CRLF>.<CRLF>",
             _421 => "Service not available, closing transmission channel",
@@ -808,6 +902,7 @@ impl TryFrom<&'_ str> for ReplyCode {
             "250" => Ok(_250),
             "251" => Ok(_251),
             "252" => Ok(_252),
+            "334" => Ok(_334),
             "354" => Ok(_354),
             "421" => Ok(_421),
             "450" => Ok(_450),
@@ -910,6 +1005,9 @@ async fn read_lines<'r>(
             }
         }
     }
+    if ret.len() < 3 {
+        return Err(MeliError::new(format!("Invalid SMTP reply: {}", ret)));
+    }
     let code = ReplyCode::try_from(&ret[..3])?;
     let reply = Reply::new(ret, code);
     //debug!(&reply);

melib/src/sqlite3.rs

@@ -50,50 +50,65 @@ pub fn open_or_create_db(
     description: &DatabaseDescription,
     identifier: Option<&str>,
 ) -> Result<Connection> {
-    let db_path = if let Some(id) = identifier {
-        db_path(&format!("{}_{}", id, description.name))
-    } else {
-        db_path(description.name)
-    }?;
-    let mut set_mode = false;
-    if !db_path.exists() {
-        log(
-            format!(
-                "Creating {} database in {}",
-                description.name,
-                db_path.display()
-            ),
-            crate::INFO,
-        );
-        set_mode = true;
-    }
-    let conn = Connection::open(&db_path).map_err(|e| MeliError::new(e.to_string()))?;
-    if set_mode {
-        use std::os::unix::fs::PermissionsExt;
-        let file = std::fs::File::open(&db_path)?;
-        let metadata = file.metadata()?;
-        let mut permissions = metadata.permissions();
+    let mut second_try: bool = false;
+    loop {
+        let db_path = if let Some(id) = identifier {
+            db_path(&format!("{}_{}", id, description.name))
+        } else {
+            db_path(description.name)
+        }?;
+        let mut set_mode = false;
+        if !db_path.exists() {
+            log(
+                format!(
+                    "Creating {} database in {}",
+                    description.name,
+                    db_path.display()
+                ),
+                crate::INFO,
+            );
+            set_mode = true;
+        }
+        let conn = Connection::open(&db_path).map_err(|e| MeliError::new(e.to_string()))?;
+        if set_mode {
+            use std::os::unix::fs::PermissionsExt;
+            let file = std::fs::File::open(&db_path)?;
+            let metadata = file.metadata()?;
+            let mut permissions = metadata.permissions();
 
-        permissions.set_mode(0o600); // Read/write for owner only.
-        file.set_permissions(permissions)?;
-    }
-    let version: i32 = conn.pragma_query_value(None, "user_version", |row| row.get(0))?;
-    if version != 0_i32 && version as u32 != description.version {
-        return Err(MeliError::new(format!(
-            "Database version mismatch, is {} but expected {}",
-            version, description.version
-        )));
-    }
+            permissions.set_mode(0o600); // Read/write for owner only.
+            file.set_permissions(permissions)?;
+        }
+        let version: i32 = conn.pragma_query_value(None, "user_version", |row| row.get(0))?;
+        if version != 0_i32 && version as u32 != description.version {
+            log(
+                format!(
+                    "Database version mismatch, is {} but expected {}",
+                    version, description.version
+                ),
+                crate::INFO,
+            );
+            if second_try {
+                return Err(MeliError::new(format!(
+                            "Database version mismatch, is {} but expected {}. Could not recreate database.",
+                            version, description.version
+                )));
+            }
+            reset_db(description, identifier)?;
+            second_try = true;
+            continue;
+        }
 
-    if version == 0 {
-        conn.pragma_update(None, "user_version", &description.version)?;
-    }
-    if let Some(s) = description.init_script {
-        conn.execute_batch(s)
-            .map_err(|e| MeliError::new(e.to_string()))?;
-    }
+        if version == 0 {
+            conn.pragma_update(None, "user_version", &description.version)?;
+        }
+        if let Some(s) = description.init_script {
+            conn.execute_batch(s)
+                .map_err(|e| MeliError::new(e.to_string()))?;
+        }
 
-    Ok(conn)
+        return Ok(conn);
+    }
 }
 
 /// Return database to a clean slate.
@@ -120,17 +135,27 @@ pub fn reset_db(description: &DatabaseDescription, identifier: Option<&str>) ->
 
 impl ToSql for Envelope {
     fn to_sql(&self) -> rusqlite::Result<ToSqlOutput> {
-        let v: Vec<u8> = bincode::serialize(self).map_err(|e| {
-            rusqlite::Error::ToSqlConversionFailure(Box::new(MeliError::new(e.to_string())))
-        })?;
+        let v: Vec<u8> = bincode::Options::serialize(bincode::config::DefaultOptions::new(), self)
+            .map_err(|e| {
+                rusqlite::Error::ToSqlConversionFailure(Box::new(MeliError::new(e.to_string())))
+            })?;
         Ok(ToSqlOutput::from(v))
     }
 }
 
 impl FromSql for Envelope {
     fn column_result(value: rusqlite::types::ValueRef) -> FromSqlResult<Self> {
+        use std::convert::TryFrom;
+
         let b: Vec<u8> = FromSql::column_result(value)?;
-        Ok(bincode::deserialize(&b)
-            .map_err(|e| FromSqlError::Other(Box::new(MeliError::new(e.to_string()))))?)
+
+        Ok(bincode::Options::deserialize(
+            bincode::Options::with_limit(
+                bincode::config::DefaultOptions::new(),
+                2 * u64::try_from(b.len()).map_err(|e| FromSqlError::Other(Box::new(e)))?,
+            ),
+            &b,
+        )
+        .map_err(|e| FromSqlError::Other(Box::new(e)))?)
     }
 }

melib/src/text_processing/line_break.rs

@@ -28,6 +28,7 @@ use super::types::Reflow;
 use core::cmp::Ordering;
 use core::iter::Peekable;
 use core::str::FromStr;
+use std::collections::VecDeque;
 use LineBreakClass::*;
 
 #[derive(Debug, PartialEq, Copy, Clone)]
@@ -1102,12 +1103,23 @@ pub fn split_lines_reflow(text: &str, reflow: Reflow, width: Option<usize>) -> V
                         split(&mut ret, line, width);
                         continue;
                     }
+                    let segment_tree = {
+                        use std::iter::FromIterator;
+                        let mut t: smallvec::SmallVec<[usize; 1024]> =
+                            smallvec::SmallVec::from_iter(std::iter::repeat(0).take(line.len()));
+                        for (idx, _g) in UnicodeSegmentation::grapheme_indices(line, true) {
+                            t[idx] = 1;
+                        }
+                        segment_tree::SegmentTree::new(t)
+                    };
 
                     let mut prev = 0;
                     let mut prev_line_offset = 0;
                     while prev < breaks.len() {
                         let new_off = match breaks[prev..].binary_search_by(|(offset, _)| {
-                            line[prev_line_offset..*offset].grapheme_len().cmp(&width)
+                            segment_tree
+                                .get_sum(prev_line_offset, offset.saturating_sub(1))
+                                .cmp(&width)
                         }) {
                             Ok(v) => v,
                             Err(v) => v,
@@ -1233,3 +1245,554 @@ easy to take MORE than nothing.'"#;
         println!("{}", l);
     }
 }
+
+
+mod segment_tree {
+    /*! Simple segment tree implementation for maximum in range queries. This is useful if given an
+     *  array of numbers you want to get the maximum value inside an interval quickly.
+     */
+    use smallvec::SmallVec;
+    use std::convert::TryFrom;
+    use std::iter::FromIterator;
+
+    #[derive(Default, Debug, Clone)]
+    pub(super) struct SegmentTree {
+        array: SmallVec<[usize; 1024]>,
+        tree: SmallVec<[usize; 1024]>,
+    }
+
+    impl SegmentTree {
+        pub(super) fn new(val: SmallVec<[usize; 1024]>) -> SegmentTree {
+            if val.is_empty() {
+                return SegmentTree {
+                    array: val.clone(),
+                    tree: val,
+                };
+            }
+
+            let height = (f64::from(u32::try_from(val.len()).unwrap_or(0)))
+                .log2()
+                .ceil() as u32;
+            let max_size = 2 * (2_usize.pow(height));
+
+            let mut segment_tree: SmallVec<[usize; 1024]> =
+                SmallVec::from_iter(core::iter::repeat(0).take(max_size));
+            for i in 0..val.len() {
+                segment_tree[val.len() + i] = val[i];
+            }
+
+            for i in (1..val.len()).rev() {
+                segment_tree[i] = segment_tree[2 * i] + segment_tree[2 * i + 1];
+            }
+
+            SegmentTree {
+                array: val,
+                tree: segment_tree,
+            }
+        }
+
+        /// (left, right) is inclusive
+        pub(super) fn get_sum(&self, mut left: usize, mut right: usize) -> usize {
+            if self.array.is_empty() {
+                return 0;
+            }
+
+            let len = self.array.len();
+            if left > right {
+                return 0;
+            }
+            if right >= len {
+                right = len.saturating_sub(1);
+            }
+
+            left += len;
+            right += len + 1;
+
+            let mut sum = 0;
+
+            while left < right {
+                if (left & 1) > 0 {
+                    sum += self.tree[left];
+                    left += 1;
+                }
+
+                if (right & 1) > 0 {
+                    right -= 1;
+                    sum += self.tree[right];
+                }
+
+                left /= 2;
+                right /= 2;
+            }
+            sum
+        }
+    }
+}
+
+/// A lazy stateful iterator for line breaking text. Useful for very long text where you don't want
+/// to linebreak it completely before user requests specific lines.
+#[derive(Debug, Clone)]
+pub struct LineBreakText {
+    text: String,
+    reflow: Reflow,
+    paragraph: VecDeque<String>,
+    paragraph_start_index: usize,
+    width: Option<usize>,
+    state: ReflowState,
+}
+
+#[derive(Debug, Clone)]
+enum ReflowState {
+    ReflowNo {
+        cur_index: usize,
+    },
+    ReflowAllWidth {
+        width: usize,
+        state: LineBreakTextState,
+    },
+    ReflowAll {
+        cur_index: usize,
+    },
+    ReflowFormatFlowed {
+        cur_index: usize,
+    },
+}
+
+impl ReflowState {
+    fn new(reflow: Reflow, width: Option<usize>, cur_index: usize) -> ReflowState {
+        match reflow {
+            Reflow::All if width.is_some() => ReflowState::ReflowAllWidth {
+                width: width.unwrap(),
+                state: LineBreakTextState::AtLine { cur_index },
+            },
+            Reflow::All => ReflowState::ReflowAll { cur_index },
+            Reflow::FormatFlowed => ReflowState::ReflowFormatFlowed { cur_index },
+            Reflow::No => ReflowState::ReflowNo { cur_index },
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+enum LineBreakTextState {
+    AtLine {
+        cur_index: usize,
+    },
+    WithinLine {
+        line_index: usize,
+        line_length: usize,
+        within_line_index: usize,
+        breaks: Vec<(usize, LineBreakCandidate)>,
+        prev_break: usize,
+        segment_tree: segment_tree::SegmentTree,
+    },
+}
+
+impl Default for LineBreakText {
+    fn default() -> Self {
+        Self::new(String::new(), Reflow::default(), None)
+    }
+}
+
+impl LineBreakText {
+    pub fn new(text: String, reflow: Reflow, width: Option<usize>) -> Self {
+        LineBreakText {
+            text,
+            state: ReflowState::new(reflow, width, 0),
+            paragraph: VecDeque::new(),
+            paragraph_start_index: 0,
+            reflow,
+            width,
+        }
+    }
+
+    pub fn width(&self) -> Option<usize> {
+        self.width
+    }
+
+    pub fn set_reflow(&mut self, new_val: Reflow) -> &mut Self {
+        self.reflow = new_val;
+        self.paragraph.clear();
+        self.state = ReflowState::new(self.reflow, self.width, self.paragraph_start_index);
+        self
+    }
+
+    pub fn set_width(&mut self, new_val: Option<usize>) -> &mut Self {
+        self.width = new_val;
+        self.paragraph.clear();
+        self.state = ReflowState::new(self.reflow, self.width, self.paragraph_start_index);
+        self
+    }
+
+    pub fn set_text(&mut self, new_val: String) -> &mut Self {
+        self.text = new_val;
+        self.reset()
+    }
+
+    pub fn reset(&mut self) -> &mut Self {
+        self.paragraph.clear();
+        self.state = ReflowState::new(self.reflow, self.width, 0);
+        self.paragraph_start_index = 0;
+        self
+    }
+
+    pub fn is_finished(&self) -> bool {
+        match self.state {
+            ReflowState::ReflowNo { cur_index }
+            | ReflowState::ReflowAll { cur_index }
+            | ReflowState::ReflowFormatFlowed { cur_index }
+            | ReflowState::ReflowAllWidth {
+                width: _,
+                state: LineBreakTextState::AtLine { cur_index },
+            } => cur_index >= self.text.len(),
+            ReflowState::ReflowAllWidth {
+                width: _,
+                state: LineBreakTextState::WithinLine { .. },
+            } => false,
+        }
+    }
+}
+
+impl Iterator for LineBreakText {
+    type Item = String;
+    fn next(&mut self) -> Option<Self::Item> {
+        if !self.paragraph.is_empty() {
+            return self.paragraph.pop_front();
+        }
+        if self.is_finished() {
+            return None;
+        }
+        match self.state {
+            ReflowState::ReflowFormatFlowed { ref mut cur_index } => {
+                /* rfc3676 - The Text/Plain Format and DelSp Parameters
+                 * https://tools.ietf.org/html/rfc3676 */
+
+                /*
+                 * - Split lines with indices using str::match_indices()
+                 * - Iterate and reflow flow regions, and pass fixed regions through
+                 */
+                self.paragraph_start_index = *cur_index;
+                let line_indices_iter = self.text[*cur_index..].match_indices('\n').map(|(i, _)| i);
+                let start_offset = *cur_index;
+                let mut prev_index = *cur_index;
+                let mut in_paragraph = false;
+                let mut paragraph_start = *cur_index;
+
+                let mut prev_quote_depth = 0;
+                let mut paragraph = VecDeque::new();
+                for i in line_indices_iter {
+                    let i = i + start_offset + 1;
+                    let line = &self.text[prev_index..i];
+                    let mut trimmed = line.trim_start().lines().next().unwrap_or("");
+                    let mut quote_depth = 0;
+                    let p_str: usize = trimmed
+                        .as_bytes()
+                        .iter()
+                        .position(|&b| {
+                            if b != b'>' {
+                                /* position() is short-circuiting */
+                                true
+                            } else {
+                                quote_depth += 1;
+                                false
+                            }
+                        })
+                        .unwrap_or(0);
+                    trimmed = &trimmed[p_str..];
+                    if trimmed.starts_with(' ') {
+                        /* Remove space stuffing before checking for ending space character.
+                         * [rfc3676#section-4.4] */
+                        trimmed = &trimmed[1..];
+                    }
+
+                    if trimmed.ends_with(' ') {
+                        if !in_paragraph {
+                            in_paragraph = true;
+                            paragraph_start = prev_index;
+                        } else if prev_quote_depth == quote_depth {
+                            /* This becomes part of the paragraph we're in */
+                        } else {
+                            /*Malformed line, different quote depths can't be in the same paragraph. */
+                            let paragraph_s = &self.text[paragraph_start..prev_index];
+                            reflow_helper2(
+                                &mut paragraph,
+                                paragraph_s,
+                                prev_quote_depth,
+                                in_paragraph,
+                                self.width,
+                            );
+
+                            paragraph_start = prev_index;
+                        }
+                    } else {
+                        if prev_quote_depth == quote_depth || !in_paragraph {
+                            let paragraph_s = &self.text[paragraph_start..i];
+                            reflow_helper2(
+                                &mut paragraph,
+                                paragraph_s,
+                                quote_depth,
+                                in_paragraph,
+                                self.width,
+                            );
+                        } else {
+                            /*Malformed line, different quote depths can't be in the same paragraph. */
+                            let paragraph_s = &self.text[paragraph_start..prev_index];
+                            reflow_helper2(
+                                &mut paragraph,
+                                paragraph_s,
+                                prev_quote_depth,
+                                in_paragraph,
+                                self.width,
+                            );
+                            let paragraph_s = &self.text[prev_index..i];
+                            reflow_helper2(
+                                &mut paragraph,
+                                paragraph_s,
+                                quote_depth,
+                                false,
+                                self.width,
+                            );
+                        }
+                        *cur_index = i;
+                        std::mem::swap(&mut self.paragraph, &mut paragraph);
+                        paragraph_start = i;
+                        in_paragraph = false;
+                        break;
+                    }
+                    *cur_index = i;
+                    prev_quote_depth = quote_depth;
+                    prev_index = i;
+                }
+                if in_paragraph {
+                    let paragraph_s = &self.text[paragraph_start..self.text.len()];
+                    *cur_index = self.text.len();
+                    reflow_helper2(
+                        &mut paragraph,
+                        paragraph_s,
+                        prev_quote_depth,
+                        in_paragraph,
+                        self.width,
+                    );
+                    self.paragraph = paragraph;
+                }
+                return self.paragraph.pop_front();
+            }
+            ReflowState::ReflowAllWidth {
+                width,
+                ref mut state,
+            } => {
+                let width = width.saturating_sub(2);
+
+                loop {
+                    let line: &str;
+                    let cur_index: &mut usize;
+                    let within_line_index: &mut usize;
+                    let prev_break: &mut usize;
+                    let segment_tree: &segment_tree::SegmentTree;
+                    let breaks: &Vec<(usize, LineBreakCandidate)>;
+                    match state {
+                        LineBreakTextState::AtLine {
+                            cur_index: ref mut _cur_index,
+                        } => {
+                            line = if let Some(line) = self
+                                .text
+                                .get(*_cur_index..)
+                                .and_then(|slice| slice.split('\n').next())
+                            {
+                                line
+                            } else {
+                                *_cur_index = self.text.len();
+                                return None;
+                            };
+                            let _cur_index = *_cur_index;
+                            *state = LineBreakTextState::WithinLine {
+                                line_index: _cur_index,
+                                line_length: line.len(),
+                                within_line_index: 0,
+                                breaks: LineBreakCandidateIter::new(line).collect::<Vec<(
+                                    usize,
+                                    LineBreakCandidate,
+                                )>>(
+                                ),
+                                prev_break: 0,
+                                segment_tree: {
+                                    use std::iter::FromIterator;
+                                    let mut t: smallvec::SmallVec<[usize; 1024]> =
+                                        smallvec::SmallVec::from_iter(
+                                            std::iter::repeat(0).take(line.len()),
+                                        );
+                                    for (idx, _g) in
+                                        UnicodeSegmentation::grapheme_indices(line, true)
+                                    {
+                                        t[idx] = 1;
+                                    }
+                                    segment_tree::SegmentTree::new(t)
+                                },
+                            };
+                            if let LineBreakTextState::WithinLine {
+                                ref mut line_index,
+                                line_length: _,
+                                within_line_index: ref mut _within_line_index,
+                                breaks: ref _breaks,
+                                prev_break: ref mut _prev_break,
+                                segment_tree: ref _segment_tree,
+                            } = state
+                            {
+                                cur_index = line_index;
+                                within_line_index = _within_line_index;
+                                breaks = _breaks;
+                                prev_break = _prev_break;
+
+                                segment_tree = _segment_tree;
+                            } else {
+                                unreachable!()
+                            }
+                        }
+                        LineBreakTextState::WithinLine {
+                            ref mut line_index,
+                            ref line_length,
+                            within_line_index: ref mut _within_line_index,
+                            breaks: ref _breaks,
+                            prev_break: ref mut _prev_break,
+                            segment_tree: ref _segment_tree,
+                        } => {
+                            line = &self.text[*line_index..(*line_index + *line_length)];
+                            cur_index = line_index;
+                            within_line_index = _within_line_index;
+                            breaks = _breaks;
+                            prev_break = _prev_break;
+                            segment_tree = _segment_tree;
+                        }
+                    }
+
+                    if segment_tree.get_sum(0, line.len()) <= width {
+                        *state = LineBreakTextState::AtLine {
+                            cur_index: *cur_index + line.len() + 1,
+                        };
+                        return Some(
+                            line.trim_end_matches(|c| c == '\r' || c == '\n')
+                                .to_string(),
+                        );
+                    }
+                    if breaks.len() < 2 {
+                        let mut line = line;
+                        while !line.is_empty() {
+                            let mut chop_index = std::cmp::min(line.len().saturating_sub(1), width);
+                            while chop_index > 0 && !line.is_char_boundary(chop_index) {
+                                chop_index -= 1;
+                            }
+                            if chop_index == 0 {
+                                self.paragraph.push_back(format!("⤷{}", line));
+                                *cur_index += line.len();
+                                break;
+                            } else {
+                                self.paragraph
+                                    .push_back(format!("⤷{}", &line[..chop_index]));
+                                *cur_index += chop_index;
+                            }
+                            line = &line[chop_index..];
+                        }
+                        *state = LineBreakTextState::AtLine {
+                            cur_index: *cur_index,
+                        };
+                        if !self.paragraph.is_empty() {
+                            return self.paragraph.pop_front();
+                        }
+                        continue;
+                    }
+
+                    while *prev_break < breaks.len() {
+                        let new_off = match breaks[*prev_break..].binary_search_by(|(offset, _)| {
+                            segment_tree
+                                .get_sum(*within_line_index, offset.saturating_sub(1))
+                                .cmp(&width)
+                        }) {
+                            Ok(v) => v,
+                            Err(v) => v,
+                        } + *prev_break;
+                        let end_offset = if new_off >= breaks.len() {
+                            line.len()
+                        } else {
+                            breaks[new_off].0
+                        };
+                        if !line[*within_line_index..end_offset].is_empty() {
+                            if *within_line_index == 0 {
+                                let ret = line[*within_line_index..end_offset]
+                                    .trim_end_matches(|c| c == '\r' || c == '\n');
+                                *within_line_index = end_offset;
+                                return Some(ret.to_string());
+                            } else {
+                                let ret = format!(
+                                    "⤷{}",
+                                    &line[*within_line_index..end_offset]
+                                        .trim_end_matches(|c| c == '\r' || c == '\n')
+                                );
+                                *within_line_index = end_offset;
+                                return Some(ret);
+                            }
+                        }
+                        if *within_line_index == end_offset && *prev_break == new_off {
+                            break;
+                        }
+                        *within_line_index = end_offset + 1;
+                        *prev_break = new_off;
+                    }
+                    *state = LineBreakTextState::AtLine {
+                        cur_index: *cur_index + line.len() + 1,
+                    };
+                }
+            }
+            ReflowState::ReflowNo { ref mut cur_index }
+            | ReflowState::ReflowAll { ref mut cur_index } => {
+                for line in self.text[*cur_index..].split('\n') {
+                    let ret = line.to_string();
+                    *cur_index += line.len() + 2;
+                    return Some(ret);
+                }
+                return None;
+            }
+        }
+    }
+}
+
+fn reflow_helper2(
+    ret: &mut VecDeque<String>,
+    paragraph: &str,
+    quote_depth: usize,
+    in_paragraph: bool,
+    width: Option<usize>,
+) {
+    if quote_depth > 0 {
+        let quotes: String = ">".repeat(quote_depth);
+        let paragraph = paragraph
+            .trim_start_matches(&quotes)
+            .replace(&format!("\n{}", &quotes), "")
+            .replace("\n", "")
+            .replace("\r", "");
+        if in_paragraph {
+            if let Some(width) = width {
+                ret.extend(
+                    linear(&paragraph, width.saturating_sub(quote_depth))
+                        .into_iter()
+                        .map(|l| format!("{}{}", &quotes, l)),
+                );
+            } else {
+                ret.push_back(format!("{}{}", &quotes, &paragraph));
+            }
+        } else {
+            ret.push_back(format!("{}{}", &quotes, &paragraph));
+        }
+    } else {
+        let paragraph = paragraph.replace("\n", "").replace("\r", "");
+
+        if in_paragraph {
+            if let Some(width) = width {
+                let ex = linear(&paragraph, width);
+                ret.extend(ex.into_iter());
+            } else {
+                ret.push_back(paragraph);
+            }
+        } else {
+            ret.push_back(paragraph);
+        }
+    }
+}

melib/src/text_processing/mod.rs

@@ -33,6 +33,8 @@ pub use wcwidth::*;
 pub trait Truncate {
     fn truncate_at_boundary(&mut self, new_len: usize);
     fn trim_at_boundary(&self, new_len: usize) -> &str;
+    fn trim_left_at_boundary(&self, new_len: usize) -> &str;
+    fn truncate_left_at_boundary(&mut self, new_len: usize);
 }
 
 impl Truncate for &str {
@@ -67,6 +69,39 @@ impl Truncate for &str {
             self
         }
     }
+
+    fn trim_left_at_boundary(&self, skip_len: usize) -> &str {
+        if skip_len >= self.len() {
+            return "";
+        }
+
+        extern crate unicode_segmentation;
+        use unicode_segmentation::UnicodeSegmentation;
+        if let Some((first, _)) = UnicodeSegmentation::grapheme_indices(*self, true)
+            .skip(skip_len)
+            .next()
+        {
+            &self[first..]
+        } else {
+            self
+        }
+    }
+
+    fn truncate_left_at_boundary(&mut self, skip_len: usize) {
+        if skip_len >= self.len() {
+            *self = "";
+            return;
+        }
+
+        extern crate unicode_segmentation;
+        use unicode_segmentation::UnicodeSegmentation;
+        if let Some((first, _)) = UnicodeSegmentation::grapheme_indices(*self, true)
+            .skip(skip_len)
+            .next()
+        {
+            *self = &self[first..];
+        }
+    }
 }
 
 impl Truncate for String {
@@ -101,6 +136,39 @@ impl Truncate for String {
             self.as_str()
         }
     }
+
+    fn trim_left_at_boundary(&self, skip_len: usize) -> &str {
+        if skip_len >= self.len() {
+            return "";
+        }
+
+        extern crate unicode_segmentation;
+        use unicode_segmentation::UnicodeSegmentation;
+        if let Some((first, _)) = UnicodeSegmentation::grapheme_indices(self.as_str(), true)
+            .skip(skip_len)
+            .next()
+        {
+            &self[first..]
+        } else {
+            self.as_str()
+        }
+    }
+
+    fn truncate_left_at_boundary(&mut self, skip_len: usize) {
+        if skip_len >= self.len() {
+            self.clear();
+            return;
+        }
+
+        extern crate unicode_segmentation;
+        use unicode_segmentation::UnicodeSegmentation;
+        if let Some((first, _)) = UnicodeSegmentation::grapheme_indices(self.as_str(), true)
+            .skip(skip_len)
+            .next()
+        {
+            *self = self[first..].to_string();
+        }
+    }
 }
 
 pub trait GlobMatch {

melib/src/text_processing/wcwidth.rs

@@ -44,7 +44,7 @@ type WChar = u32;
 type Interval = (WChar, WChar);
 
 pub struct CodePointsIterator<'a> {
-    rest: &'a [u8],
+    rest: std::str::Chars<'a>,
 }
 
 /*
@@ -61,36 +61,7 @@ impl<'a> Iterator for CodePointsIterator<'a> {
     type Item = WChar;
 
     fn next(&mut self) -> Option<WChar> {
-        if self.rest.is_empty() {
-            return None;
-        }
-        /* Input is UTF-8 valid strings, guaranteed by Rust's std */
-        if self.rest[0] & 0b1000_0000 == 0x0 {
-            let ret: WChar = WChar::from(self.rest[0]);
-            self.rest = &self.rest[1..];
-            return Some(ret);
-        }
-        if self.rest[0] & 0b1110_0000 == 0b1100_0000 {
-            let ret: WChar = (WChar::from(self.rest[0]) & 0b0001_1111).rotate_left(6)
-                + (WChar::from(self.rest[1]) & 0b0111_1111);
-            self.rest = &self.rest[2..];
-            return Some(ret);
-        }
-
-        if self.rest[0] & 0b1111_0000 == 0b1110_0000 {
-            let ret: WChar = (WChar::from(self.rest[0]) & 0b0000_0111).rotate_left(12)
-                + (WChar::from(self.rest[1]) & 0b0011_1111).rotate_left(6)
-                + (WChar::from(self.rest[2]) & 0b0011_1111);
-            self.rest = &self.rest[3..];
-            return Some(ret);
-        }
-
-        let ret: WChar = (WChar::from(self.rest[0]) & 0b0000_0111).rotate_left(18)
-            + (WChar::from(self.rest[1]) & 0b0011_1111).rotate_left(12)
-            + (WChar::from(self.rest[2]) & 0b0011_1111).rotate_left(6)
-            + (WChar::from(self.rest[3]) & 0b0011_1111);
-        self.rest = &self.rest[4..];
-        Some(ret)
+        self.rest.next().map(|c| c as WChar)
     }
 }
 pub trait CodePointsIter {
@@ -99,16 +70,12 @@ pub trait CodePointsIter {
 
 impl CodePointsIter for str {
     fn code_points(&self) -> CodePointsIterator {
-        CodePointsIterator {
-            rest: self.as_bytes(),
-        }
+        CodePointsIterator { rest: self.chars() }
     }
 }
 impl CodePointsIter for &str {
     fn code_points(&self) -> CodePointsIterator {
-        CodePointsIterator {
-            rest: self.as_bytes(),
-        }
+        CodePointsIterator { rest: self.chars() }
     }
 }
 
@@ -136,174 +103,54 @@ fn bisearch(ucs: WChar, table: &'static [Interval]) -> bool {
     false
 }
 
-/* The following functions define the column width of an ISO 10646
- * character as follows:
- *
- *    - The null character (U+0000) has a column width of 0.
- *
- *    - Other C0/C1 control characters and DEL will lead to a return
- *      value of -1.
- *
- *    - Non-spacing and enclosing combining characters (general
- *      category code Mn or Me in the Unicode database) have a
- *      column width of 0.
- *
- *    - Other format characters (general category code Cf in the Unicode
- *      database) and ZERO WIDTH SPACE (U+200B) have a column width of 0.
- *
- *    - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF)
- *      have a column width of 0.
- *
- *    - Spacing characters in the East Asian Wide (W) or East Asian
- *      FullWidth (F) category as defined in Unicode Technical
- *      Report #11 have a column width of 2.
- *
- *    - All remaining characters (including all printable
- *      ISO 8859-1 and WGL4 characters, Unicode control characters,
- *      etc.) have a column width of 1.
- *
- * This implementation assumes that wchar_t characters are encoded
- * in ISO 10646.
- */
-
 pub fn wcwidth(ucs: WChar) -> Option<usize> {
-    /* sorted list of non-overlapping intervals of non-spacing characters */
-    const COMBINING: &[Interval] = &[
-        (0x0300, 0x034E),
-        (0x0360, 0x0362),
-        (0x0483, 0x0486),
-        (0x0488, 0x0489),
-        (0x0591, 0x05A1),
-        (0x05A3, 0x05B9),
-        (0x05BB, 0x05BD),
-        (0x05BF, 0x05BF),
-        (0x05C1, 0x05C2),
-        (0x05C4, 0x05C4),
-        (0x064B, 0x0655),
-        (0x0670, 0x0670),
-        (0x06D6, 0x06E4),
-        (0x06E7, 0x06E8),
-        (0x06EA, 0x06ED),
-        (0x070F, 0x070F),
-        (0x0711, 0x0711),
-        (0x0730, 0x074A),
-        (0x07A6, 0x07B0),
-        (0x0901, 0x0902),
-        (0x093C, 0x093C),
-        (0x0941, 0x0948),
-        (0x094D, 0x094D),
-        (0x0951, 0x0954),
-        (0x0962, 0x0963),
-        (0x0981, 0x0981),
-        (0x09BC, 0x09BC),
-        (0x09C1, 0x09C4),
-        (0x09CD, 0x09CD),
-        (0x09E2, 0x09E3),
-        (0x0A02, 0x0A02),
-        (0x0A3C, 0x0A3C),
-        (0x0A41, 0x0A42),
-        (0x0A47, 0x0A48),
-        (0x0A4B, 0x0A4D),
-        (0x0A70, 0x0A71),
-        (0x0A81, 0x0A82),
-        (0x0ABC, 0x0ABC),
-        (0x0AC1, 0x0AC5),
-        (0x0AC7, 0x0AC8),
-        (0x0ACD, 0x0ACD),
-        (0x0B01, 0x0B01),
-        (0x0B3C, 0x0B3C),
-        (0x0B3F, 0x0B3F),
-        (0x0B41, 0x0B43),
-        (0x0B4D, 0x0B4D),
-        (0x0B56, 0x0B56),
-        (0x0B82, 0x0B82),
-        (0x0BC0, 0x0BC0),
-        (0x0BCD, 0x0BCD),
-        (0x0C3E, 0x0C40),
-        (0x0C46, 0x0C48),
-        (0x0C4A, 0x0C4D),
-        (0x0C55, 0x0C56),
-        (0x0CBF, 0x0CBF),
-        (0x0CC6, 0x0CC6),
-        (0x0CCC, 0x0CCD),
-        (0x0D41, 0x0D43),
-        (0x0D4D, 0x0D4D),
-        (0x0DCA, 0x0DCA),
-        (0x0DD2, 0x0DD4),
-        (0x0DD6, 0x0DD6),
-        (0x0E31, 0x0E31),
-        (0x0E34, 0x0E3A),
-        (0x0E47, 0x0E4E),
-        (0x0EB1, 0x0EB1),
-        (0x0EB4, 0x0EB9),
-        (0x0EBB, 0x0EBC),
-        (0x0EC8, 0x0ECD),
-        (0x0F18, 0x0F19),
-        (0x0F35, 0x0F35),
-        (0x0F37, 0x0F37),
-        (0x0F39, 0x0F39),
-        (0x0F71, 0x0F7E),
-        (0x0F80, 0x0F84),
-        (0x0F86, 0x0F87),
-        (0x0F90, 0x0F97),
-        (0x0F99, 0x0FBC),
-        (0x0FC6, 0x0FC6),
-        (0x102D, 0x1030),
-        (0x1032, 0x1032),
-        (0x1036, 0x1037),
-        (0x1039, 0x1039),
-        (0x1058, 0x1059),
-        (0x1160, 0x11FF),
-        (0x17B7, 0x17BD),
-        (0x17C6, 0x17C6),
-        (0x17C9, 0x17D3),
-        (0x180B, 0x180E),
-        (0x18A9, 0x18A9),
-        (0x200B, 0x200F),
-        (0x202A, 0x202E),
-        (0x206A, 0x206F),
-        (0x20D0, 0x20E3),
-        (0x302A, 0x302F),
-        (0x3099, 0x309A),
-        (0xFB1E, 0xFB1E),
-        (0xFE20, 0xFE23),
-        (0xFEFF, 0xFEFF),
-        (0xFFF9, 0xFFFB),
-    ];
-
-    /* test for 8-bit control characters */
-    if ucs == 0 {
-        return Some(0);
-    }
-    if ucs < 32 || (ucs >= 0x7f && ucs < 0xa0) {
-        return None;
+    if bisearch(ucs, super::tables::ASCII) {
+        Some(1)
+    } else if bisearch(ucs, super::tables::PRIVATE) {
+        None
+    } else if bisearch(ucs, super::tables::NONPRINT) {
+        None
+    } else if bisearch(ucs, super::tables::COMBINING) {
+        None
+    } else if bisearch(ucs, super::tables::DOUBLEWIDE) {
+        Some(2)
+    } else if bisearch(ucs, super::tables::AMBIGUOUS) {
+        Some(1)
+    } else if bisearch(ucs, super::tables::UNASSIGNED) {
+        Some(2)
+    } else if bisearch(ucs, super::tables::WIDENEDIN9) {
+        Some(2)
+    } else {
+        Some(1)
     }
+}
 
-    /* binary search in table of emojis */
-    if bisearch(ucs, EMOJI_RANGES) {
-        return Some(2);
-    }
-    /* binary search in table of non-spacing characters */
-    if bisearch(ucs, COMBINING) {
-        return Some(1);
-    }
-
-    /* if we arrive here, ucs is not a combining or C0/C1 control character */
-
-    Some(
-        1 + big_if_true!(
-            ucs >= 0x1100
-                && (ucs <= 0x115f ||                    /* Hangul Jamo init. consonants */
-      (ucs >= 0x2e80 && ucs <= 0xa4cf && (ucs & !0x0011) != 0x300a &&
-       ucs != 0x303f) ||                  /* CJK ... Yi */
-      (ucs >= 0xac00 && ucs <= 0xd7a3) || /* Hangul Syllables */
-      (ucs >= 0xf900 && ucs <= 0xfaff) || /* CJK Compatibility Ideographs */
-      (ucs >= 0xfe30 && ucs <= 0xfe6f) || /* CJK Compatibility Forms */
-      (ucs >= 0xff00 && ucs <= 0xff5f) || /* Fullwidth Forms */
-      (ucs >= 0xffe0 && ucs <= 0xffe6) ||
-      (ucs >= 0x20000 && ucs <= 0x2ffff))
-        ),
-    )
+#[test]
+fn test_wcwidth() {
+    assert_eq!(
+        &"abc\0".code_points().collect::<Vec<_>>(),
+        &[0x61, 0x62, 0x63, 0x0]
+    );
+    assert_eq!(&"●".code_points().collect::<Vec<_>>(), &[0x25cf]);
+    assert_eq!(&"📎".code_points().collect::<Vec<_>>(), &[0x1f4ce]);
+    assert_eq!(
+        &"𐼹𐼺𐼻𐼼𐼽".code_points().collect::<Vec<_>>(),
+        &[0x10F39, 0x10F3A, 0x10F3B, 0x10F3C, 0x10F3D]
+    ); // Sogdian alphabet
+    assert_eq!(
+        &"𐼹a𐼽b".code_points().collect::<Vec<_>>(),
+        &[0x10F39, 0x61, 0x10F3D, 0x62]
+    ); // Sogdian alphabet
+    assert_eq!(
+        &"📎\u{FE0E}".code_points().collect::<Vec<_>>(),
+        &[0x1f4ce, 0xfe0e]
+    );
+    use crate::text_processing::grapheme_clusters::TextProcessing;
+    assert_eq!("●".grapheme_width(), 1);
+    assert_eq!("●📎".grapheme_width(), 3);
+    assert_eq!("●\u{FE0E}📎\u{FE0E}".grapheme_width(), 3);
+    assert_eq!("🎃".grapheme_width(), 2);
+    assert_eq!("👻".grapheme_width(), 2);
 }
 
 pub fn wcswidth(mut pwcs: WChar, mut n: usize) -> Option<usize> {
@@ -322,360 +169,3 @@ pub fn wcswidth(mut pwcs: WChar, mut n: usize) -> Option<usize> {
 
     Some(width)
 }
-
-const EMOJI_RANGES: &[Interval] = &[
-    (0x231A, 0x231B), //    ; Basic_Emoji              ; watch                                                          #  1.1  [2] (⌚..⌛)
-    (0x23E9, 0x23EC), //    ; Basic_Emoji              ; fast-forward button                                            #  6.0  [4] (⏩..⏬)
-    (0x23F0, 0x23F0), //          ; Basic_Emoji              ; alarm clock                                                    #  6.0  [1] (⏰)
-    (0x23F3, 0x23F3), //          ; Basic_Emoji              ; hourglass not done                                             #  6.0  [1] (⏳)
-    (0x25FD, 0x25FE), //    ; Basic_Emoji              ; white medium-small square                                      #  3.2  [2] (◽..◾)
-    (0x2614, 0x2615), //    ; Basic_Emoji              ; umbrella with rain drops                                       #  4.0  [2] (☔..☕)
-    (0x2648, 0x2653), //    ; Basic_Emoji              ; Aries                                                          #  1.1 [12] (♈..♓)
-    (0x267F, 0x267F), //          ; Basic_Emoji              ; wheelchair symbol                                              #  4.1  [1] (♿)
-    (0x2693, 0x2693), //          ; Basic_Emoji              ; anchor                                                         #  4.1  [1] (⚓)
-    (0x26A1, 0x26A1), //          ; Basic_Emoji              ; high voltage                                                   #  4.0  [1] (⚡)
-    (0x26AA, 0x26AB), //    ; Basic_Emoji              ; white circle                                                   #  4.1  [2] (⚪..⚫)
-    (0x26BD, 0x26BE), //    ; Basic_Emoji              ; soccer ball                                                    #  5.2  [2] (⚽..⚾)
-    (0x26C4, 0x26C5), //    ; Basic_Emoji              ; snowman without snow                                           #  5.2  [2] (⛄..⛅)
-    (0x26CE, 0x26CE), //          ; Basic_Emoji              ; Ophiuchus                                                      #  6.0  [1] (⛎)
-    (0x26D4, 0x26D4), //          ; Basic_Emoji              ; no entry                                                       #  5.2  [1] (⛔)
-    (0x26EA, 0x26EA), //          ; Basic_Emoji              ; church                                                         #  5.2  [1] (⛪)
-    (0x26F2, 0x26F3), //    ; Basic_Emoji              ; fountain                                                       #  5.2  [2] (⛲..⛳)
-    (0x26F5, 0x26F5), //          ; Basic_Emoji              ; sailboat                                                       #  5.2  [1] (⛵)
-    (0x26FA, 0x26FA), //          ; Basic_Emoji              ; tent                                                           #  5.2  [1] (⛺)
-    (0x26FD, 0x26FD), //          ; Basic_Emoji              ; fuel pump                                                      #  5.2  [1] (⛽)
-    (0x2705, 0x2705), //          ; Basic_Emoji              ; check mark button                                              #  6.0  [1] (✅)
-    (0x270A, 0x270B), //    ; Basic_Emoji              ; raised fist                                                    #  6.0  [2] (✊..✋)
-    (0x2728, 0x2728), //          ; Basic_Emoji              ; sparkles                                                       #  6.0  [1] (✨)
-    (0x274C, 0x274C), //          ; Basic_Emoji              ; cross mark                                                     #  6.0  [1] (❌)
-    (0x274E, 0x274E), //          ; Basic_Emoji              ; cross mark button                                              #  6.0  [1] (❎)
-    (0x2753, 0x2755), //    ; Basic_Emoji              ; question mark                                                  #  6.0  [3] (❓..❕)
-    (0x2757, 0x2757), //          ; Basic_Emoji              ; exclamation mark                                               #  5.2  [1] (❗)
-    (0x2795, 0x2797), //    ; Basic_Emoji              ; plus sign                                                      #  6.0  [3] (➕..➗)
-    (0x27B0, 0x27B0), //          ; Basic_Emoji              ; curly loop                                                     #  6.0  [1] (➰)
-    (0x27BF, 0x27BF), //          ; Basic_Emoji              ; double curly loop                                              #  6.0  [1] (➿)
-    (0x2B1B, 0x2B1C), //    ; Basic_Emoji              ; black large square                                             #  5.1  [2] (⬛..⬜)
-    (0x2B50, 0x2B50), //          ; Basic_Emoji              ; star                                                           #  5.1  [1] (⭐)
-    (0x2B55, 0x2B55), //          ; Basic_Emoji              ; hollow red circle                                              #  5.2  [1] (⭕)
-    (0x1F004, 0x1F004), //         ; Basic_Emoji              ; mahjong red dragon                                             #  5.1  [1] (🀄)
-    (0x1F0CF, 0x1F0CF), //         ; Basic_Emoji              ; joker                                                          #  6.0  [1] (🃏)
-    (0x1F18E, 0x1F18E), //         ; Basic_Emoji              ; AB button (blood type)                                         #  6.0  [1] (🆎)
-    (0x1F191, 0x1F19A), //  ; Basic_Emoji              ; CL button                                                      #  6.0 [10] (🆑..🆚)
-    (0x1F201, 0x1F201), //         ; Basic_Emoji              ; Japanese “here” button                                         #  6.0  [1] (🈁)
-    (0x1F21A, 0x1F21A), //         ; Basic_Emoji              ; Japanese “free of charge” button                               #  5.2  [1] (🈚)
-    (0x1F22F, 0x1F22F), //         ; Basic_Emoji              ; Japanese “reserved” button                                     #  5.2  [1] (🈯)
-    (0x1F232, 0x1F236), //  ; Basic_Emoji              ; Japanese “prohibited” button                                   #  6.0  [5] (🈲..🈶)
-    (0x1F238, 0x1F23A), //  ; Basic_Emoji              ; Japanese “application” button                                  #  6.0  [3] (🈸..🈺)
-    (0x1F250, 0x1F251), //  ; Basic_Emoji              ; Japanese “bargain” button                                      #  6.0  [2] (🉐..🉑)
-    (0x1F300, 0x1F320), //  ; Basic_Emoji              ; cyclone                                                        #  6.0 [33] (🌀..🌠)
-    (0x1F32D, 0x1F32F), //  ; Basic_Emoji              ; hot dog                                                        #  8.0  [3] (🌭..🌯)
-    (0x1F330, 0x1F335), //  ; Basic_Emoji              ; chestnut                                                       #  6.0  [6] (🌰..🌵)
-    (0x1F337, 0x1F37C), //  ; Basic_Emoji              ; tulip                                                          #  6.0 [70] (🌷..🍼)
-    (0x1F37E, 0x1F37F), //  ; Basic_Emoji              ; bottle with popping cork                                       #  8.0  [2] (🍾..🍿)
-    (0x1F380, 0x1F393), //  ; Basic_Emoji              ; ribbon                                                         #  6.0 [20] (🎀..🎓)
-    (0x1F3A0, 0x1F3C4), //  ; Basic_Emoji              ; carousel horse                                                 #  6.0 [37] (🎠..🏄)
-    (0x1F3C5, 0x1F3C5), //         ; Basic_Emoji              ; sports medal                                                   #  7.0  [1] (🏅)
-    (0x1F3C6, 0x1F3CA), //  ; Basic_Emoji              ; trophy                                                         #  6.0  [5] (🏆..🏊)
-    (0x1F3CF, 0x1F3D3), //  ; Basic_Emoji              ; cricket game                                                   #  8.0  [5] (🏏..🏓)
-    (0x1F3E0, 0x1F3F0), //  ; Basic_Emoji              ; house                                                          #  6.0 [17] (🏠..🏰)
-    (0x1F3F4, 0x1F3F4), //         ; Basic_Emoji              ; black flag                                                     #  7.0  [1] (🏴)
-    (0x1F3F8, 0x1F3FF), //  ; Basic_Emoji              ; badminton                                                      #  8.0  [8] (🏸..🏿)
-    (0x1F400, 0x1F43E), //  ; Basic_Emoji              ; rat                                                            #  6.0 [63] (🐀..🐾)
-    (0x1F440, 0x1F440), //         ; Basic_Emoji              ; eyes                                                           #  6.0  [1] (👀)
-    (0x1F442, 0x1F4F7), //  ; Basic_Emoji              ; ear                                                            #  6.0[182] (👂..📷)
-    (0x1F4F8, 0x1F4F8), //         ; Basic_Emoji              ; camera with flash                                              #  7.0  [1] (📸)
-    (0x1F4F9, 0x1F4FC), //  ; Basic_Emoji              ; video camera                                                   #  6.0  [4] (📹..📼)
-    (0x1F4FF, 0x1F4FF), //         ; Basic_Emoji              ; prayer beads                                                   #  8.0  [1] (📿)
-    (0x1F500, 0x1F53D), //  ; Basic_Emoji              ; shuffle tracks button                                          #  6.0 [62] (🔀..🔽)
-    (0x1F54B, 0x1F54E), //  ; Basic_Emoji              ; kaaba                                                          #  8.0  [4] (🕋..🕎)
-    (0x1F550, 0x1F567), //  ; Basic_Emoji              ; one o’clock                                                    #  6.0 [24] (🕐..🕧)
-    (0x1F57A, 0x1F57A), //         ; Basic_Emoji              ; man dancing                                                    #  9.0  [1] (🕺)
-    (0x1F595, 0x1F596), //  ; Basic_Emoji              ; middle finger                                                  #  7.0  [2] (🖕..🖖)
-    (0x1F5A4, 0x1F5A4), //         ; Basic_Emoji              ; black heart                                                    #  9.0  [1] (🖤)
-    (0x1F5FB, 0x1F5FF), //  ; Basic_Emoji              ; mount fuji                                                     #  6.0  [5] (🗻..🗿)
-    (0x1F600, 0x1F600), //         ; Basic_Emoji              ; grinning face                                                  #  6.1  [1] (😀)
-    (0x1F601, 0x1F610), //  ; Basic_Emoji              ; beaming face with smiling eyes                                 #  6.0 [16] (😁..😐)
-    (0x1F611, 0x1F611), //         ; Basic_Emoji              ; expressionless face                                            #  6.1  [1] (😑)
-    (0x1F612, 0x1F614), //  ; Basic_Emoji              ; unamused face                                                  #  6.0  [3] (😒..😔)
-    (0x1F615, 0x1F615), //         ; Basic_Emoji              ; confused face                                                  #  6.1  [1] (😕)
-    (0x1F616, 0x1F616), //         ; Basic_Emoji              ; confounded face                                                #  6.0  [1] (😖)
-    (0x1F617, 0x1F617), //         ; Basic_Emoji              ; kissing face                                                   #  6.1  [1] (😗)
-    (0x1F618, 0x1F618), //         ; Basic_Emoji              ; face blowing a kiss                                            #  6.0  [1] (😘)
-    (0x1F619, 0x1F619), //         ; Basic_Emoji              ; kissing face with smiling eyes                                 #  6.1  [1] (😙)
-    (0x1F61A, 0x1F61A), //         ; Basic_Emoji              ; kissing face with closed eyes                                  #  6.0  [1] (😚)
-    (0x1F61B, 0x1F61B), //         ; Basic_Emoji              ; face with tongue                                               #  6.1  [1] (😛)
-    (0x1F61C, 0x1F61E), //  ; Basic_Emoji              ; winking face with tongue                                       #  6.0  [3] (😜..😞)
-    (0x1F61F, 0x1F61F), //         ; Basic_Emoji              ; worried face                                                   #  6.1  [1] (😟)
-    (0x1F620, 0x1F625), //  ; Basic_Emoji              ; angry face                                                     #  6.0  [6] (😠..😥)
-    (0x1F626, 0x1F627), //  ; Basic_Emoji              ; frowning face with open mouth                                  #  6.1  [2] (😦..😧)
-    (0x1F628, 0x1F62B), //  ; Basic_Emoji              ; fearful face                                                   #  6.0  [4] (😨..😫)
-    (0x1F62C, 0x1F62C), //         ; Basic_Emoji              ; grimacing face                                                 #  6.1  [1] (😬)
-    (0x1F62D, 0x1F62D), //         ; Basic_Emoji              ; loudly crying face                                             #  6.0  [1] (😭)
-    (0x1F62E, 0x1F62F), //  ; Basic_Emoji              ; face with open mouth                                           #  6.1  [2] (😮..😯)
-    (0x1F630, 0x1F633), //  ; Basic_Emoji              ; anxious face with sweat                                        #  6.0  [4] (😰..😳)
-    (0x1F634, 0x1F634), //         ; Basic_Emoji              ; sleeping face                                                  #  6.1  [1] (😴)
-    (0x1F635, 0x1F640), //  ; Basic_Emoji              ; dizzy face                                                     #  6.0 [12] (😵..🙀)
-    (0x1F641, 0x1F642), //  ; Basic_Emoji              ; slightly frowning face                                         #  7.0  [2] (🙁..🙂)
-    (0x1F643, 0x1F644), //  ; Basic_Emoji              ; upside-down face                                               #  8.0  [2] (🙃..🙄)
-    (0x1F645, 0x1F64F), //  ; Basic_Emoji              ; person gesturing NO                                            #  6.0 [11] (🙅..🙏)
-    (0x1F680, 0x1F6C5), //  ; Basic_Emoji              ; rocket                                                         #  6.0 [70] (🚀..🛅)
-    (0x1F6CC, 0x1F6CC), //         ; Basic_Emoji              ; person in bed                                                  #  7.0  [1] (🛌)
-    (0x1F6D0, 0x1F6D0), //         ; Basic_Emoji              ; place of worship                                               #  8.0  [1] (🛐)
-    (0x1F6D1, 0x1F6D2), //  ; Basic_Emoji              ; stop sign                                                      #  9.0  [2] (🛑..🛒)
-    (0x1F6D5, 0x1F6D5), //         ; Basic_Emoji              ; hindu temple                                                   # 12.0  [1] (🛕)
-    (0x1F6EB, 0x1F6EC), //  ; Basic_Emoji              ; airplane departure                                             #  7.0  [2] (🛫..🛬)
-    (0x1F6F4, 0x1F6F6), //  ; Basic_Emoji              ; kick scooter                                                   #  9.0  [3] (🛴..🛶)
-    (0x1F6F7, 0x1F6F8), //  ; Basic_Emoji              ; sled                                                           # 10.0  [2] (🛷..🛸)
-    (0x1F6F9, 0x1F6F9), //         ; Basic_Emoji              ; skateboard                                                     # 11.0  [1] (🛹)
-    (0x1F6FA, 0x1F6FA), //         ; Basic_Emoji              ; auto rickshaw                                                  # 12.0  [1] (🛺)
-    (0x1F7E0, 0x1F7EB), //  ; Basic_Emoji              ; orange circle                                                  # 12.0 [12] (🟠..🟫)
-    (0x1F90D, 0x1F90F), //  ; Basic_Emoji              ; white heart                                                    # 12.0  [3] (🤍..🤏)
-    (0x1F910, 0x1F918), //  ; Basic_Emoji              ; zipper-mouth face                                              #  8.0  [9] (🤐..🤘)
-    (0x1F919, 0x1F91E), //  ; Basic_Emoji              ; call me hand                                                   #  9.0  [6] (🤙..🤞)
-    (0x1F91F, 0x1F91F), //         ; Basic_Emoji              ; love-you gesture                                               # 10.0  [1] (🤟)
-    (0x1F920, 0x1F927), //  ; Basic_Emoji              ; cowboy hat face                                                #  9.0  [8] (🤠..🤧)
-    (0x1F928, 0x1F92F), //  ; Basic_Emoji              ; face with raised eyebrow                                       # 10.0  [8] (🤨..🤯)
-    (0x1F930, 0x1F930), //         ; Basic_Emoji              ; pregnant woman                                                 #  9.0  [1] (🤰)
-    (0x1F931, 0x1F932), //  ; Basic_Emoji              ; breast-feeding                                                 # 10.0  [2] (🤱..🤲)
-    (0x1F933, 0x1F93A), //  ; Basic_Emoji              ; selfie                                                         #  9.0  [8] (🤳..🤺)
-    (0x1F93C, 0x1F93E), //  ; Basic_Emoji              ; people wrestling                                               #  9.0  [3] (🤼..🤾)
-    (0x1F93F, 0x1F93F), //         ; Basic_Emoji              ; diving mask                                                    # 12.0  [1] (🤿)
-    (0x1F940, 0x1F945), //  ; Basic_Emoji              ; wilted flower                                                  #  9.0  [6] (🥀..🥅)
-    (0x1F947, 0x1F94B), //  ; Basic_Emoji              ; 1st place medal                                                #  9.0  [5] (🥇..🥋)
-    (0x1F94C, 0x1F94C), //         ; Basic_Emoji              ; curling stone                                                  # 10.0  [1] (🥌)
-    (0x1F94D, 0x1F94F), //  ; Basic_Emoji              ; lacrosse                                                       # 11.0  [3] (🥍..🥏)
-    (0x1F950, 0x1F95E), //  ; Basic_Emoji              ; croissant                                                      #  9.0 [15] (🥐..🥞)
-    (0x1F95F, 0x1F96B), //  ; Basic_Emoji              ; dumpling                                                       # 10.0 [13] (🥟..🥫)
-    (0x1F96C, 0x1F970), //  ; Basic_Emoji              ; leafy green                                                    # 11.0  [5] (🥬..🥰)
-    (0x1F971, 0x1F971), //         ; Basic_Emoji              ; yawning face                                                   # 12.0  [1] (🥱)
-    (0x1F973, 0x1F976), //  ; Basic_Emoji              ; partying face                                                  # 11.0  [4] (🥳..🥶)
-    (0x1F97A, 0x1F97A), //         ; Basic_Emoji              ; pleading face                                                  # 11.0  [1] (🥺)
-    (0x1F97B, 0x1F97B), //         ; Basic_Emoji              ; sari                                                           # 12.0  [1] (🥻)
-    (0x1F97C, 0x1F97F), //  ; Basic_Emoji              ; lab coat                                                       # 11.0  [4] (🥼..🥿)
-    (0x1F980, 0x1F984), //  ; Basic_Emoji              ; crab                                                           #  8.0  [5] (🦀..🦄)
-    (0x1F985, 0x1F991), //  ; Basic_Emoji              ; eagle                                                          #  9.0 [13] (🦅..🦑)
-    (0x1F992, 0x1F997), //  ; Basic_Emoji              ; giraffe                                                        # 10.0  [6] (🦒..🦗)
-    (0x1F998, 0x1F9A2), //  ; Basic_Emoji              ; kangaroo                                                       # 11.0 [11] (🦘..🦢)
-    (0x1F9A5, 0x1F9AA), //  ; Basic_Emoji              ; sloth                                                          # 12.0  [6] (🦥..🦪)
-    (0x1F9AE, 0x1F9AF), //  ; Basic_Emoji              ; guide dog                                                      # 12.0  [2] (🦮..🦯)
-    (0x1F9B0, 0x1F9B9), //  ; Basic_Emoji              ; red hair                                                       # 11.0 [10] (🦰..🦹)
-    (0x1F9BA, 0x1F9BF), //  ; Basic_Emoji              ; safety vest                                                    # 12.0  [6] (🦺..🦿)
-    (0x1F9C0, 0x1F9C0), //         ; Basic_Emoji              ; cheese wedge                                                   #  8.0  [1] (🧀)
-    (0x1F9C1, 0x1F9C2), //  ; Basic_Emoji              ; cupcake                                                        # 11.0  [2] (🧁..🧂)
-    (0x1F9C3, 0x1F9CA), //  ; Basic_Emoji              ; beverage box                                                   # 12.0  [8] (🧃..🧊)
-    (0x1F9CD, 0x1F9CF), //  ; Basic_Emoji              ; person standing                                                # 12.0  [3] (🧍..🧏)
-    (0x1F9D0, 0x1F9E6), //  ; Basic_Emoji              ; face with monocle                                              # 10.0 [23] (🧐..🧦)
-    (0x1F9E7, 0x1F9FF), //  ; Basic_Emoji              ; red envelope                                                   # 11.0 [25] (🧧..🧿)
-    (0x1FA70, 0x1FA73), //  ; Basic_Emoji              ; ballet shoes                                                   # 12.0  [4] (🩰..🩳)
-    (0x1FA78, 0x1FA7A), //  ; Basic_Emoji              ; drop of blood                                                  # 12.0  [3] (🩸..🩺)
-    (0x1FA80, 0x1FA82), //  ; Basic_Emoji              ; yo-yo                                                          # 12.0  [3] (🪀..🪂)
-    (0x1FA90, 0x1FA95), //  ; Basic_Emoji              ; ringed planet                                                  # 12.0  [6] (🪐..🪕)
-];
-/*
-00A9 FE0F     ; Basic_Emoji              ; copyright                                                      #  3.2  [1] (©️)
-00AE FE0F     ; Basic_Emoji              ; registered                                                     #  3.2  [1] (®️)
-203C FE0F     ; Basic_Emoji              ; double exclamation mark                                        #  3.2  [1] (‼️)
-2049 FE0F     ; Basic_Emoji              ; exclamation question mark                                      #  3.2  [1] (⁉️)
-2122 FE0F     ; Basic_Emoji              ; trade mark                                                     #  3.2  [1] (™️)
-2139 FE0F     ; Basic_Emoji              ; information                                                    #  3.2  [1] (ℹ️)
-2194 FE0F     ; Basic_Emoji              ; left-right arrow                                               #  3.2  [1] (↔️)
-2195 FE0F     ; Basic_Emoji              ; up-down arrow                                                  #  3.2  [1] (↕️)
-2196 FE0F     ; Basic_Emoji              ; up-left arrow                                                  #  3.2  [1] (↖️)
-2197 FE0F     ; Basic_Emoji              ; up-right arrow                                                 #  3.2  [1] (↗️)
-2198 FE0F     ; Basic_Emoji              ; down-right arrow                                               #  3.2  [1] (↘️)
-2199 FE0F     ; Basic_Emoji              ; down-left arrow                                                #  3.2  [1] (↙️)
-21A9 FE0F     ; Basic_Emoji              ; right arrow curving left                                       #  3.2  [1] (↩️)
-21AA FE0F     ; Basic_Emoji              ; left arrow curving right                                       #  3.2  [1] (↪️)
-2328 FE0F     ; Basic_Emoji              ; keyboard                                                       #  3.2  [1] (⌨️)
-23CF FE0F     ; Basic_Emoji              ; eject button                                                   #  4.0  [1] (⏏️)
-23ED FE0F     ; Basic_Emoji              ; next track button                                              #  6.0  [1] (⏭️)
-23EE FE0F     ; Basic_Emoji              ; last track button                                              #  6.0  [1] (⏮️)
-23EF FE0F     ; Basic_Emoji              ; play or pause button                                           #  6.0  [1] (⏯️)
-23F1 FE0F     ; Basic_Emoji              ; stopwatch                                                      #  6.0  [1] (⏱️)
-23F2 FE0F     ; Basic_Emoji              ; timer clock                                                    #  6.0  [1] (⏲️)
-23F8 FE0F     ; Basic_Emoji              ; pause button                                                   #  7.0  [1] (⏸️)
-23F9 FE0F     ; Basic_Emoji              ; stop button                                                    #  7.0  [1] (⏹️)
-23FA FE0F     ; Basic_Emoji              ; record button                                                  #  7.0  [1] (⏺️)
-24C2 FE0F     ; Basic_Emoji              ; circled M                                                      #  3.2  [1] (Ⓜ️)
-25AA FE0F     ; Basic_Emoji              ; black small square                                             #  3.2  [1] (▪️)
-25AB FE0F     ; Basic_Emoji              ; white small square                                             #  3.2  [1] (▫️)
-25B6 FE0F     ; Basic_Emoji              ; play button                                                    #  3.2  [1] (▶️)
-25C0 FE0F     ; Basic_Emoji              ; reverse button                                                 #  3.2  [1] (◀️)
-25FB FE0F     ; Basic_Emoji              ; white medium square                                            #  3.2  [1] (◻️)
-25FC FE0F     ; Basic_Emoji              ; black medium square                                            #  3.2  [1] (◼️)
-2600 FE0F     ; Basic_Emoji              ; sun                                                            #  3.2  [1] (☀️)
-2601 FE0F     ; Basic_Emoji              ; cloud                                                          #  3.2  [1] (☁️)
-2602 FE0F     ; Basic_Emoji              ; umbrella                                                       #  3.2  [1] (☂️)
-2603 FE0F     ; Basic_Emoji              ; snowman                                                        #  3.2  [1] (☃️)
-2604 FE0F     ; Basic_Emoji              ; comet                                                          #  3.2  [1] (☄️)
-260E FE0F     ; Basic_Emoji              ; telephone                                                      #  3.2  [1] (☎️)
-2611 FE0F     ; Basic_Emoji              ; check box with check                                           #  3.2  [1] (☑️)
-2618 FE0F     ; Basic_Emoji              ; shamrock                                                       #  4.1  [1] (☘️)
-261D FE0F     ; Basic_Emoji              ; index pointing up                                              #  3.2  [1] (☝️)
-2620 FE0F     ; Basic_Emoji              ; skull and crossbones                                           #  3.2  [1] (☠️)
-2622 FE0F     ; Basic_Emoji              ; radioactive                                                    #  3.2  [1] (☢️)
-2623 FE0F     ; Basic_Emoji              ; biohazard                                                      #  3.2  [1] (☣️)
-2626 FE0F     ; Basic_Emoji              ; orthodox cross                                                 #  3.2  [1] (☦️)
-262A FE0F     ; Basic_Emoji              ; star and crescent                                              #  3.2  [1] (☪️)
-262E FE0F     ; Basic_Emoji              ; peace symbol                                                   #  3.2  [1] (☮️)
-262F FE0F     ; Basic_Emoji              ; yin yang                                                       #  3.2  [1] (☯️)
-2638 FE0F     ; Basic_Emoji              ; wheel of dharma                                                #  3.2  [1] (☸️)
-2639 FE0F     ; Basic_Emoji              ; frowning face                                                  #  3.2  [1] (☹️)
-263A FE0F     ; Basic_Emoji              ; smiling face                                                   #  3.2  [1] (☺️)
-2640 FE0F     ; Basic_Emoji              ; female sign                                                    #  3.2  [1] (♀️)
-2642 FE0F     ; Basic_Emoji              ; male sign                                                      #  3.2  [1] (♂️)
-265F FE0F     ; Basic_Emoji              ; chess pawn                                                     #  3.2  [1] (♟️)
-2660 FE0F     ; Basic_Emoji              ; spade suit                                                     #  3.2  [1] (♠️)
-2663 FE0F     ; Basic_Emoji              ; club suit                                                      #  3.2  [1] (♣️)
-2665 FE0F     ; Basic_Emoji              ; heart suit                                                     #  3.2  [1] (♥️)
-2666 FE0F     ; Basic_Emoji              ; diamond suit                                                   #  3.2  [1] (♦️)
-2668 FE0F     ; Basic_Emoji              ; hot springs                                                    #  3.2  [1] (♨️)
-267B FE0F     ; Basic_Emoji              ; recycling symbol                                               #  3.2  [1] (♻️)
-267E FE0F     ; Basic_Emoji              ; infinity                                                       #  4.1  [1] (♾️)
-2692 FE0F     ; Basic_Emoji              ; hammer and pick                                                #  4.1  [1] (⚒️)
-2694 FE0F     ; Basic_Emoji              ; crossed swords                                                 #  4.1  [1] (⚔️)
-2695 FE0F     ; Basic_Emoji              ; medical symbol                                                 #  4.1  [1] (⚕️)
-2696 FE0F     ; Basic_Emoji              ; balance scale                                                  #  4.1  [1] (⚖️)
-2697 FE0F     ; Basic_Emoji              ; alembic                                                        #  4.1  [1] (⚗️)
-2699 FE0F     ; Basic_Emoji              ; gear                                                           #  4.1  [1] (⚙️)
-269B FE0F     ; Basic_Emoji              ; atom symbol                                                    #  4.1  [1] (⚛️)
-269C FE0F     ; Basic_Emoji              ; fleur-de-lis                                                   #  4.1  [1] (⚜️)
-26A0 FE0F     ; Basic_Emoji              ; warning                                                        #  4.0  [1] (⚠️)
-26B0 FE0F     ; Basic_Emoji              ; coffin                                                         #  4.1  [1] (⚰️)
-26B1 FE0F     ; Basic_Emoji              ; funeral urn                                                    #  4.1  [1] (⚱️)
-26C8 FE0F     ; Basic_Emoji              ; cloud with lightning and rain                                  #  5.2  [1] (⛈️)
-26CF FE0F     ; Basic_Emoji              ; pick                                                           #  5.2  [1] (⛏️)
-26D1 FE0F     ; Basic_Emoji              ; rescue worker’s helmet                                         #  5.2  [1] (⛑️)
-26D3 FE0F     ; Basic_Emoji              ; chains                                                         #  5.2  [1] (⛓️)
-26E9 FE0F     ; Basic_Emoji              ; shinto shrine                                                  #  5.2  [1] (⛩️)
-26F0 FE0F     ; Basic_Emoji              ; mountain                                                       #  5.2  [1] (⛰️)
-26F1 FE0F     ; Basic_Emoji              ; umbrella on ground                                             #  5.2  [1] (⛱️)
-26F4 FE0F     ; Basic_Emoji              ; ferry                                                          #  5.2  [1] (⛴️)
-26F7 FE0F     ; Basic_Emoji              ; skier                                                          #  5.2  [1] (⛷️)
-26F8 FE0F     ; Basic_Emoji              ; ice skate                                                      #  5.2  [1] (⛸️)
-26F9 FE0F     ; Basic_Emoji              ; person bouncing ball                                           #  5.2  [1] (⛹️)
-2702 FE0F     ; Basic_Emoji              ; scissors                                                       #  3.2  [1] (✂️)
-2708 FE0F     ; Basic_Emoji              ; airplane                                                       #  3.2  [1] (✈️)
-2709 FE0F     ; Basic_Emoji              ; envelope                                                       #  3.2  [1] (✉️)
-270C FE0F     ; Basic_Emoji              ; victory hand                                                   #  3.2  [1] (✌️)
-270D FE0F     ; Basic_Emoji              ; writing hand                                                   #  3.2  [1] (✍️)
-270F FE0F     ; Basic_Emoji              ; pencil                                                         #  3.2  [1] (✏️)
-2712 FE0F     ; Basic_Emoji              ; black nib                                                      #  3.2  [1] (✒️)
-2714 FE0F     ; Basic_Emoji              ; check mark                                                     #  3.2  [1] (✔️)
-2716 FE0F     ; Basic_Emoji              ; multiplication sign                                            #  3.2  [1] (✖️)
-271D FE0F     ; Basic_Emoji              ; latin cross                                                    #  3.2  [1] (✝️)
-2721 FE0F     ; Basic_Emoji              ; star of David                                                  #  3.2  [1] (✡️)
-2733 FE0F     ; Basic_Emoji              ; eight-spoked asterisk                                          #  3.2  [1] (✳️)
-2734 FE0F     ; Basic_Emoji              ; eight-pointed star                                             #  3.2  [1] (✴️)
-2744 FE0F     ; Basic_Emoji              ; snowflake                                                      #  3.2  [1] (❄️)
-2747 FE0F     ; Basic_Emoji              ; sparkle                                                        #  3.2  [1] (❇️)
-2763 FE0F     ; Basic_Emoji              ; heart exclamation                                              #  3.2  [1] (❣️)
-2764 FE0F     ; Basic_Emoji              ; red heart                                                      #  3.2  [1] (❤️)
-27A1 FE0F     ; Basic_Emoji              ; right arrow                                                    #  3.2  [1] (➡️)
-2934 FE0F     ; Basic_Emoji              ; right arrow curving up                                         #  3.2  [1] (⤴️)
-2935 FE0F     ; Basic_Emoji              ; right arrow curving down                                       #  3.2  [1] (⤵️)
-2B05 FE0F     ; Basic_Emoji              ; left arrow                                                     #  4.0  [1] (⬅️)
-2B06 FE0F     ; Basic_Emoji              ; up arrow                                                       #  4.0  [1] (⬆️)
-2B07 FE0F     ; Basic_Emoji              ; down arrow                                                     #  4.0  [1] (⬇️)
-3030 FE0F     ; Basic_Emoji              ; wavy dash                                                      #  3.2  [1] (〰️)
-303D FE0F     ; Basic_Emoji              ; part alternation mark                                          #  3.2  [1] (〽️)
-3297 FE0F     ; Basic_Emoji              ; Japanese “congratulations” button                              #  3.2  [1] (㊗️)
-3299 FE0F     ; Basic_Emoji              ; Japanese “secret” button                                       #  3.2  [1] (㊙️)
-1F170 FE0F    ; Basic_Emoji              ; A button (blood type)                                          #  6.0  [1] (🅰️)
-1F171 FE0F    ; Basic_Emoji              ; B button (blood type)                                          #  6.0  [1] (🅱️)
-1F17E FE0F    ; Basic_Emoji              ; O button (blood type)                                          #  6.0  [1] (🅾️)
-1F17F FE0F    ; Basic_Emoji              ; P button                                                       #  5.2  [1] (🅿️)
-1F202 FE0F    ; Basic_Emoji              ; Japanese “service charge” button                               #  6.0  [1] (🈂️)
-1F237 FE0F    ; Basic_Emoji              ; Japanese “monthly amount” button                               #  6.0  [1] (🈷️)
-1F321 FE0F    ; Basic_Emoji              ; thermometer                                                    #  7.0  [1] (🌡️)
-1F324 FE0F    ; Basic_Emoji              ; sun behind small cloud                                         #  7.0  [1] (🌤️)
-1F325 FE0F    ; Basic_Emoji              ; sun behind large cloud                                         #  7.0  [1] (🌥️)
-1F326 FE0F    ; Basic_Emoji              ; sun behind rain cloud                                          #  7.0  [1] (🌦️)
-1F327 FE0F    ; Basic_Emoji              ; cloud with rain                                                #  7.0  [1] (🌧️)
-1F328 FE0F    ; Basic_Emoji              ; cloud with snow                                                #  7.0  [1] (🌨️)
-1F329 FE0F    ; Basic_Emoji              ; cloud with lightning                                           #  7.0  [1] (🌩️)
-1F32A FE0F    ; Basic_Emoji              ; tornado                                                        #  7.0  [1] (🌪️)
-1F32B FE0F    ; Basic_Emoji              ; fog                                                            #  7.0  [1] (🌫️)
-1F32C FE0F    ; Basic_Emoji              ; wind face                                                      #  7.0  [1] (🌬️)
-1F336 FE0F    ; Basic_Emoji              ; hot pepper                                                     #  7.0  [1] (🌶️)
-1F37D FE0F    ; Basic_Emoji              ; fork and knife with plate                                      #  7.0  [1] (🍽️)
-1F396 FE0F    ; Basic_Emoji              ; military medal                                                 #  7.0  [1] (🎖️)
-1F397 FE0F    ; Basic_Emoji              ; reminder ribbon                                                #  7.0  [1] (🎗️)
-1F399 FE0F    ; Basic_Emoji              ; studio microphone                                              #  7.0  [1] (🎙️)
-1F39A FE0F    ; Basic_Emoji              ; level slider                                                   #  7.0  [1] (🎚️)
-1F39B FE0F    ; Basic_Emoji              ; control knobs                                                  #  7.0  [1] (🎛️)
-1F39E FE0F    ; Basic_Emoji              ; film frames                                                    #  7.0  [1] (🎞️)
-1F39F FE0F    ; Basic_Emoji              ; admission tickets                                              #  7.0  [1] (🎟️)
-1F3CB FE0F    ; Basic_Emoji              ; person lifting weights                                         #  7.0  [1] (🏋️)
-1F3CC FE0F    ; Basic_Emoji              ; person golfing                                                 #  7.0  [1] (🏌️)
-1F3CD FE0F    ; Basic_Emoji              ; motorcycle                                                     #  7.0  [1] (🏍️)
-1F3CE FE0F    ; Basic_Emoji              ; racing car                                                     #  7.0  [1] (🏎️)
-1F3D4 FE0F    ; Basic_Emoji              ; snow-capped mountain                                           #  7.0  [1] (🏔️)
-1F3D5 FE0F    ; Basic_Emoji              ; camping                                                        #  7.0  [1] (🏕️)
-1F3D6 FE0F    ; Basic_Emoji              ; beach with umbrella                                            #  7.0  [1] (🏖️)
-1F3D7 FE0F    ; Basic_Emoji              ; building construction                                          #  7.0  [1] (🏗️)
-1F3D8 FE0F    ; Basic_Emoji              ; houses                                                         #  7.0  [1] (🏘️)
-1F3D9 FE0F    ; Basic_Emoji              ; cityscape                                                      #  7.0  [1] (🏙️)
-1F3DA FE0F    ; Basic_Emoji              ; derelict house                                                 #  7.0  [1] (🏚️)
-1F3DB FE0F    ; Basic_Emoji              ; classical building                                             #  7.0  [1] (🏛️)
-1F3DC FE0F    ; Basic_Emoji              ; desert                                                         #  7.0  [1] (🏜️)
-1F3DD FE0F    ; Basic_Emoji              ; desert island                                                  #  7.0  [1] (🏝️)
-1F3DE FE0F    ; Basic_Emoji              ; national park                                                  #  7.0  [1] (🏞️)
-1F3DF FE0F    ; Basic_Emoji              ; stadium                                                        #  7.0  [1] (🏟️)
-1F3F3 FE0F    ; Basic_Emoji              ; white flag                                                     #  7.0  [1] (🏳️)
-1F3F5 FE0F    ; Basic_Emoji              ; rosette                                                        #  7.0  [1] (🏵️)
-1F3F7 FE0F    ; Basic_Emoji              ; label                                                          #  7.0  [1] (🏷️)
-1F43F FE0F    ; Basic_Emoji              ; chipmunk                                                       #  7.0  [1] (🐿️)
-1F441 FE0F    ; Basic_Emoji              ; eye                                                            #  7.0  [1] (👁️)
-1F4FD FE0F    ; Basic_Emoji              ; film projector                                                 #  7.0  [1] (📽️)
-1F549 FE0F    ; Basic_Emoji              ; om                                                             #  7.0  [1] (🕉️)
-1F54A FE0F    ; Basic_Emoji              ; dove                                                           #  7.0  [1] (🕊️)
-1F56F FE0F    ; Basic_Emoji              ; candle                                                         #  7.0  [1] (🕯️)
-1F570 FE0F    ; Basic_Emoji              ; mantelpiece clock                                              #  7.0  [1] (🕰️)
-1F573 FE0F    ; Basic_Emoji              ; hole                                                           #  7.0  [1] (🕳️)
-1F574 FE0F    ; Basic_Emoji              ; man in suit levitating                                         #  7.0  [1] (🕴️)
-1F575 FE0F    ; Basic_Emoji              ; detective                                                      #  7.0  [1] (🕵️)
-1F576 FE0F    ; Basic_Emoji              ; sunglasses                                                     #  7.0  [1] (🕶️)
-1F577 FE0F    ; Basic_Emoji              ; spider                                                         #  7.0  [1] (🕷️)
-1F578 FE0F    ; Basic_Emoji              ; spider web                                                     #  7.0  [1] (🕸️)
-1F579 FE0F    ; Basic_Emoji              ; joystick                                                       #  7.0  [1] (🕹️)
-1F587 FE0F    ; Basic_Emoji              ; linked paperclips                                              #  7.0  [1] (🖇️)
-1F58A FE0F    ; Basic_Emoji              ; pen                                                            #  7.0  [1] (🖊️)
-1F58B FE0F    ; Basic_Emoji              ; fountain pen                                                   #  7.0  [1] (🖋️)
-1F58C FE0F    ; Basic_Emoji              ; paintbrush                                                     #  7.0  [1] (🖌️)
-1F58D FE0F    ; Basic_Emoji              ; crayon                                                         #  7.0  [1] (🖍️)
-1F590 FE0F    ; Basic_Emoji              ; hand with fingers splayed                                      #  7.0  [1] (🖐️)
-1F5A5 FE0F    ; Basic_Emoji              ; desktop computer                                               #  7.0  [1] (🖥️)
-1F5A8 FE0F    ; Basic_Emoji              ; printer                                                        #  7.0  [1] (🖨️)
-1F5B1 FE0F    ; Basic_Emoji              ; computer mouse                                                 #  7.0  [1] (🖱️)
-1F5B2 FE0F    ; Basic_Emoji              ; trackball                                                      #  7.0  [1] (🖲️)
-1F5BC FE0F    ; Basic_Emoji              ; framed picture                                                 #  7.0  [1] (🖼️)
-1F5C2 FE0F    ; Basic_Emoji              ; card index dividers                                            #  7.0  [1] (🗂️)
-1F5C3 FE0F    ; Basic_Emoji              ; card file box                                                  #  7.0  [1] (🗃️)
-1F5C4 FE0F    ; Basic_Emoji              ; file cabinet                                                   #  7.0  [1] (🗄️)
-1F5D1 FE0F    ; Basic_Emoji              ; wastebasket                                                    #  7.0  [1] (🗑️)
-1F5D2 FE0F    ; Basic_Emoji              ; spiral notepad                                                 #  7.0  [1] (🗒️)
-1F5D3 FE0F    ; Basic_Emoji              ; spiral calendar                                                #  7.0  [1] (🗓️)
-1F5DC FE0F    ; Basic_Emoji              ; clamp                                                          #  7.0  [1] (🗜️)
-1F5DD FE0F    ; Basic_Emoji              ; old key                                                        #  7.0  [1] (🗝️)
-1F5DE FE0F    ; Basic_Emoji              ; rolled-up newspaper                                            #  7.0  [1] (🗞️)
-1F5E1 FE0F    ; Basic_Emoji              ; dagger                                                         #  7.0  [1] (🗡️)
-1F5E3 FE0F    ; Basic_Emoji              ; speaking head                                                  #  7.0  [1] (🗣️)
-1F5E8 FE0F    ; Basic_Emoji              ; left speech bubble                                             #  7.0  [1] (🗨️)
-1F5EF FE0F    ; Basic_Emoji              ; right anger bubble                                             #  7.0  [1] (🗯️)
-1F5F3 FE0F    ; Basic_Emoji              ; ballot box with ballot                                         #  7.0  [1] (🗳️)
-1F5FA FE0F    ; Basic_Emoji              ; world map                                                      #  7.0  [1] (🗺️)
-1F6CB FE0F    ; Basic_Emoji              ; couch and lamp                                                 #  7.0  [1] (🛋️)
-1F6CD FE0F    ; Basic_Emoji              ; shopping bags                                                  #  7.0  [1] (🛍️)
-1F6CE FE0F    ; Basic_Emoji              ; bellhop bell                                                   #  7.0  [1] (🛎️)
-1F6CF FE0F    ; Basic_Emoji              ; bed                                                            #  7.0  [1] (🛏️)
-1F6E0 FE0F    ; Basic_Emoji              ; hammer and wrench                                              #  7.0  [1] (🛠️)
-1F6E1 FE0F    ; Basic_Emoji              ; shield                                                         #  7.0  [1] (🛡️)
-1F6E2 FE0F    ; Basic_Emoji              ; oil drum                                                       #  7.0  [1] (🛢️)
-1F6E3 FE0F    ; Basic_Emoji              ; motorway                                                       #  7.0  [1] (🛣️)
-1F6E4 FE0F    ; Basic_Emoji              ; railway track                                                  #  7.0  [1] (🛤️)
-1F6E5 FE0F    ; Basic_Emoji              ; motor boat                                                     #  7.0  [1] (🛥️)
-1F6E9 FE0F    ; Basic_Emoji              ; small airplane                                                 #  7.0  [1] (🛩️)
-1F6F0 FE0F    ; Basic_Emoji              ; satellite                                                      #  7.0  [1] (🛰️)
-1F6F3 FE0F    ; Basic_Emoji              ; passenger ship                                                 #  7.0  [1] (🛳️)
-*/

melib/src/thread.rs

@@ -33,6 +33,7 @@
  */
 
 use crate::datetime::UnixTimestamp;
+use crate::email::address::StrBuild;
 use crate::email::parser::BytesExt;
 use crate::email::*;
 
@@ -43,7 +44,6 @@ pub use iterators::*;
 use crate::text_processing::grapheme_clusters::*;
 use uuid::Uuid;
 
-use std::cell::RefCell;
 use std::cmp::Ordering;
 use std::collections::{HashMap, HashSet};
 use std::fmt;
@@ -130,7 +130,7 @@ macro_rules! make {
                 e.parent = Some($p);
             });
             let old_group = std::mem::replace($threads.groups.entry(old_group_hash).or_default(), ThreadGroup::Node {
-                parent: RefCell::new(parent_group_hash),
+                parent: Arc::new(RwLock::new(parent_group_hash)),
             });
             $threads.thread_nodes.entry($c).and_modify(|e| {
                 e.group = parent_group_hash;
@@ -291,7 +291,7 @@ pub struct Thread {
 #[derive(Clone, Debug, Deserialize, Serialize)]
 pub enum ThreadGroup {
     Root(Thread),
-    Node { parent: RefCell<ThreadHash> },
+    Node { parent: Arc<RwLock<ThreadHash>> },
 }
 
 impl Default for ThreadGroup {
@@ -337,6 +337,7 @@ impl Thread {
 #[derive(Clone, Debug, Deserialize, Serialize)]
 pub struct ThreadNode {
     pub message: Option<EnvelopeHash>,
+    pub other_mailbox: bool,
     pub parent: Option<ThreadNodeHash>,
     pub children: Vec<ThreadNodeHash>,
     pub date: UnixTimestamp,
@@ -350,6 +351,7 @@ impl Default for ThreadNode {
         ThreadNode {
             message: None,
             parent: None,
+            other_mailbox: false,
             children: Vec::new(),
             date: UnixTimestamp::default(),
             show_subject: true,
@@ -408,16 +410,16 @@ impl ThreadNode {
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 pub struct Threads {
     pub thread_nodes: HashMap<ThreadNodeHash, ThreadNode>,
-    root_set: RefCell<Vec<ThreadNodeHash>>,
-    tree_index: RefCell<Vec<ThreadNodeHash>>,
+    root_set: Arc<RwLock<Vec<ThreadNodeHash>>>,
+    tree_index: Arc<RwLock<Vec<ThreadNodeHash>>>,
     pub groups: HashMap<ThreadHash, ThreadGroup>,
 
     message_ids: HashMap<Vec<u8>, ThreadNodeHash>,
     pub message_ids_set: HashSet<Vec<u8>>,
     pub missing_message_ids: HashSet<Vec<u8>>,
     pub hash_set: HashSet<EnvelopeHash>,
-    sort: RefCell<(SortField, SortOrder)>,
-    subsort: RefCell<(SortField, SortOrder)>,
+    sort: Arc<RwLock<(SortField, SortOrder)>>,
+    subsort: Arc<RwLock<(SortField, SortOrder)>>,
 }
 
 impl PartialEq for ThreadNode {
@@ -451,13 +453,13 @@ impl Threads {
     pub fn find_group(&self, h: ThreadHash) -> ThreadHash {
         let p = match self.groups[&h] {
             ThreadGroup::Root(_) => return h,
-            ThreadGroup::Node { ref parent } => *parent.borrow(),
+            ThreadGroup::Node { ref parent } => *parent.read().unwrap(),
         };
 
         let parent_group = self.find_group(p);
         match self.groups[&h] {
             ThreadGroup::Node { ref parent } => {
-                *parent.borrow_mut() = parent_group;
+                *parent.write().unwrap() = parent_group;
             }
             _ => unreachable!(),
         }
@@ -488,8 +490,8 @@ impl Threads {
             message_ids_set,
             missing_message_ids,
             hash_set,
-            sort: RefCell::new((SortField::Date, SortOrder::Desc)),
-            subsort: RefCell::new((SortField::Subject, SortOrder::Desc)),
+            sort: Arc::new(RwLock::new((SortField::Date, SortOrder::Desc))),
+            subsort: Arc::new(RwLock::new((SortField::Subject, SortOrder::Desc))),
 
             ..Default::default()
         }
@@ -570,7 +572,7 @@ impl Threads {
         };
 
         if self.thread_nodes[&t_id].parent.is_none() {
-            let mut tree_index = self.tree_index.borrow_mut();
+            let mut tree_index = self.tree_index.write().unwrap();
             if let Some(i) = tree_index.iter().position(|t| *t == t_id) {
                 tree_index.remove(i);
             }
@@ -652,17 +654,36 @@ impl Threads {
         env_hash: EnvelopeHash,
         other_mailbox: bool,
     ) -> bool {
+        {
+            let envelopes_lck = envelopes.read().unwrap();
+            let message_id = envelopes_lck[&env_hash].message_id().raw();
+            if self.message_ids.contains_key(message_id)
+                && !self.missing_message_ids.contains(message_id)
+            {
+                let thread_hash = self.message_ids[message_id];
+                let node = self.thread_nodes.entry(thread_hash).or_default();
+                drop(message_id);
+                drop(envelopes_lck);
+                envelopes
+                    .write()
+                    .unwrap()
+                    .get_mut(&env_hash)
+                    .unwrap()
+                    .set_thread(thread_hash);
+
+                /* If thread node currently has a message from a foreign mailbox and env_hash is
+                 * from current mailbox we want to update it, otherwise return */
+                if !(node.other_mailbox && !other_mailbox) {
+                    return false;
+                }
+            }
+        }
         let envelopes_lck = envelopes.read().unwrap();
         let reply_to_id: Option<ThreadNodeHash> = envelopes_lck[&env_hash]
             .in_reply_to()
-            .map(crate::email::StrBuild::raw)
+            .map(StrBuild::raw)
             .and_then(|r| self.message_ids.get(r).cloned());
         let message_id = envelopes_lck[&env_hash].message_id().raw();
-        if self.message_ids_set.contains(message_id)
-            && !self.missing_message_ids.contains(message_id)
-        {
-            return false;
-        }
 
         if other_mailbox
             && reply_to_id.is_none()
@@ -686,13 +707,14 @@ impl Threads {
                     None
                 },
             )
-            .unwrap_or_else(ThreadNodeHash::new);
+            .unwrap_or_else(|| ThreadNodeHash::from(message_id));
         {
             let mut node = self.thread_nodes.entry(new_id).or_default();
             node.message = Some(env_hash);
             if node.parent.is_none() {
                 node.parent = reply_to_id;
             }
+            node.other_mailbox = other_mailbox;
             node.date = envelopes_lck[&env_hash].date();
             node.unseen = !envelopes_lck[&env_hash].is_seen();
         }
@@ -739,11 +761,8 @@ impl Threads {
         self.hash_set.insert(env_hash);
         if let Some(reply_to_id) = reply_to_id {
             make!((reply_to_id) parent of (new_id), self);
-        } else if let Some(r) = envelopes_lck[&env_hash]
-            .in_reply_to()
-            .map(crate::email::StrBuild::raw)
-        {
-            let reply_to_id = ThreadNodeHash::new();
+        } else if let Some(r) = envelopes_lck[&env_hash].in_reply_to().map(StrBuild::raw) {
+            let reply_to_id = ThreadNodeHash::from(r);
             self.thread_nodes.insert(
                 reply_to_id,
                 ThreadNode {
@@ -787,7 +806,7 @@ impl Threads {
                     make!((id) parent of (current_descendant_id), self);
                     current_descendant_id = id;
                 } else {
-                    let id = ThreadNodeHash::new();
+                    let id = ThreadNodeHash::from(reference.raw());
                     self.thread_nodes.insert(
                         id,
                         ThreadNode {
@@ -825,7 +844,7 @@ impl Threads {
 
         /*
         save_graph(
-            &self.tree_index.borrow(),
+            &self.tree_index.read().unwrap(),
             &self.thread_nodes,
             &self
                 .message_ids
@@ -851,7 +870,7 @@ impl Threads {
             ref thread_nodes,
             ..
         } = self;
-        let tree = &mut tree_index.borrow_mut();
+        let tree = &mut tree_index.write().unwrap();
         for t in tree.iter_mut() {
             thread_nodes[t].children.sort_by(|a, b| match subsort {
                 (SortField::Date, SortOrder::Desc) => {
@@ -1070,7 +1089,7 @@ impl Threads {
         });
     }
     fn inner_sort_by(&self, sort: (SortField, SortOrder), envelopes: &Envelopes) {
-        let tree = &mut self.tree_index.borrow_mut();
+        let tree = &mut self.tree_index.write().unwrap();
         let envelopes = envelopes.read().unwrap();
         tree.sort_by(|a, b| match sort {
             (SortField::Date, SortOrder::Desc) => {
@@ -1152,13 +1171,13 @@ impl Threads {
         subsort: (SortField, SortOrder),
         envelopes: &Envelopes,
     ) {
-        if *self.sort.borrow() != sort {
+        if *self.sort.read().unwrap() != sort {
             self.inner_sort_by(sort, envelopes);
-            *self.sort.borrow_mut() = sort;
+            *self.sort.write().unwrap() = sort;
         }
-        if *self.subsort.borrow() != subsort {
+        if *self.subsort.read().unwrap() != subsort {
             self.inner_subsort_by(subsort, envelopes);
-            *self.subsort.borrow_mut() = subsort;
+            *self.subsort.write().unwrap() = subsort;
         }
     }
 
@@ -1176,11 +1195,11 @@ impl Threads {
     }
 
     pub fn root_len(&self) -> usize {
-        self.tree_index.borrow().len()
+        self.tree_index.read().unwrap().len()
     }
 
     pub fn root_set(&self, idx: usize) -> ThreadNodeHash {
-        self.tree_index.borrow()[idx]
+        self.tree_index.read().unwrap()[idx]
     }
 
     pub fn roots(&self) -> SmallVec<[ThreadHash; 1024]> {

src/bin.rs

@@ -49,9 +49,6 @@ static GLOBAL: System = System;
 extern crate melib;
 use melib::*;
 
-mod unix;
-use unix::*;
-
 #[macro_use]
 pub mod types;
 use crate::types::*;
@@ -79,7 +76,6 @@ pub mod sqlite3;
 
 pub mod jobs;
 pub mod mailcap;
-pub mod plugins;
 
 use std::os::raw::c_int;
 
@@ -105,7 +101,6 @@ fn notify(
         nix::fcntl::FcntlArg::F_SETFL(nix::fcntl::OFlag::O_NONBLOCK),
     );
     std::thread::spawn(move || {
-        let mut buf = [0; 1];
         let mut ctr = 0;
         loop {
             ctr %= 3;
@@ -118,16 +113,6 @@ fn notify(
             for signal in signals.pending() {
                 let _ = s.send_timeout(signal, Duration::from_millis(500)).ok();
             }
-            while nix::unistd::read(alarm_pipe_r, buf.as_mut())
-                .map(|s| s > 0)
-                .unwrap_or(false)
-            {
-                let value = buf[0];
-                let _ = sender.send_timeout(
-                    ThreadEvent::UIEvent(UIEvent::Timer(value)),
-                    Duration::from_millis(2000),
-                );
-            }
 
             std::thread::sleep(std::time::Duration::from_millis(100));
             ctr += 1;
@@ -150,7 +135,7 @@ fn parse_manpage(src: &str) -> Result<ManPages> {
 
 use structopt::StructOpt;
 
-#[derive(Debug)]
+#[derive(Copy, Clone, Debug)]
 /// Choose manpage
 enum ManPages {
     /// meli(1)
@@ -206,6 +191,9 @@ enum SubCommand {
 struct ManOpt {
     #[structopt(default_value = "meli", possible_values=&["meli", "conf", "themes"], value_name="PAGE", parse(try_from_str = parse_manpage))]
     page: ManPages,
+    /// If true, output text in stdout instead of spawning $PAGER.
+    #[structopt(long = "no-raw", alias = "no-raw", value_name = "bool")]
+    no_raw: Option<Option<bool>>,
 }
 
 fn main() {
@@ -251,13 +239,51 @@ fn run_app(opt: Opt) -> Result<()> {
         }
         #[cfg(feature = "cli-docs")]
         Some(SubCommand::Man(manopt)) => {
-            let _page = manopt.page;
-            const MANPAGES: [&'static str; 3] = [
-                include_str!(concat!(env!("OUT_DIR"), "/meli.txt")),
-                include_str!(concat!(env!("OUT_DIR"), "/meli.conf.txt")),
-                include_str!(concat!(env!("OUT_DIR"), "/meli-themes.txt")),
+            let ManOpt { page, no_raw } = manopt;
+            const MANPAGES: [&'static [u8]; 3] = [
+                include_bytes!(concat!(env!("OUT_DIR"), "/meli.txt.gz")),
+                include_bytes!(concat!(env!("OUT_DIR"), "/meli.conf.txt.gz")),
+                include_bytes!(concat!(env!("OUT_DIR"), "/meli-themes.txt.gz")),
             ];
-            println!("{}", MANPAGES[_page as usize]);
+            use flate2::bufread::GzDecoder;
+            use std::io::prelude::*;
+            let mut gz = GzDecoder::new(MANPAGES[page as usize]);
+            let mut v = String::with_capacity(
+                str::parse::<usize>(unsafe {
+                    std::str::from_utf8_unchecked(gz.header().unwrap().comment().unwrap())
+                })
+                .expect(&format!(
+                    "{:?} was not compressed with size comment header",
+                    page
+                )),
+            );
+            gz.read_to_string(&mut v)?;
+
+            if let Some(no_raw) = no_raw {
+                match no_raw {
+                    Some(true) => {}
+                    None if (unsafe { libc::isatty(libc::STDOUT_FILENO) == 1 }) => {}
+                    Some(false) | None => {
+                        println!("{}", &v);
+                        return Ok(());
+                    }
+                }
+            } else {
+                if unsafe { libc::isatty(libc::STDOUT_FILENO) != 1 } {
+                    println!("{}", &v);
+                    return Ok(());
+                }
+            }
+
+            use std::process::{Command, Stdio};
+            let mut handle = Command::new(std::env::var("PAGER").unwrap_or("more".to_string()))
+                .stdin(Stdio::piped())
+                .stdout(Stdio::inherit())
+                .stderr(Stdio::inherit())
+                .spawn()?;
+            handle.stdin.take().unwrap().write_all(v.as_bytes())?;
+            handle.wait()?;
+
             return Ok(());
         }
         #[cfg(not(feature = "cli-docs"))]
@@ -324,20 +350,18 @@ fn run_app(opt: Opt) -> Result<()> {
             vec![
                 Box::new(listing::Listing::new(&mut state.context)),
                 Box::new(ContactList::new(&state.context)),
-                Box::new(StatusPanel::new(crate::conf::value(
-                    &state.context,
-                    "theme_default",
-                ))),
             ],
             &state.context,
         ));
 
-        let status_bar = Box::new(StatusBar::new(window));
+        let status_bar = Box::new(StatusBar::new(&state.context, window));
         state.register_component(status_bar);
 
-        #[cfg(feature = "dbus-notifications")]
+        #[cfg(all(target_os = "linux", feature = "dbus-notifications"))]
         {
-            let dbus_notifications = Box::new(components::notifications::DbusNotifications::new());
+            let dbus_notifications = Box::new(components::notifications::DbusNotifications::new(
+                &state.context,
+            ));
             state.register_component(dbus_notifications);
         }
         state.register_component(Box::new(
@@ -351,6 +375,7 @@ fn run_app(opt: Opt) -> Result<()> {
         .general
         .enter_command_mode
         .clone();
+    let quit_key: Key = state.context.settings.shortcuts.general.quit.clone();
 
     /* Keep track of the input mode. See UIMode for details */
     'main: loop {
@@ -397,7 +422,7 @@ fn run_app(opt: Opt) -> Result<()> {
                             match state.mode {
                                 UIMode::Normal => {
                                     match k {
-                                        Key::Char('q') | Key::Char('Q') => {
+                                        _ if k == quit_key => {
                                             if state.can_quit_cleanly() {
                                                 drop(state);
                                                 break 'main;
@@ -418,8 +443,7 @@ fn run_app(opt: Opt) -> Result<()> {
                                 },
                                 UIMode::Insert => {
                                     match k {
-                                        Key::Char('\n') | Key::Esc => {
-                                            state.mode = UIMode::Normal;
+                                        Key::Esc => {
                                             state.rcv_event(UIEvent::ChangeMode(UIMode::Normal));
                                             state.redraw();
                                         },

src/command.rs

@@ -57,7 +57,7 @@ macro_rules! to_stream {
     };
     ($($tokens:expr),*) => {
         TokenStream {
-            tokens: &[$($token),*],
+            tokens: &[$($tokens),*],
         }
     };
 }
@@ -303,6 +303,21 @@ define_commands!([
                        }
                    )
                  },
+                { tags: ["import "],
+                  desc: "import FILESYSTEM_PATH MAILBOX_PATH",
+                  tokens: &[One(Literal("import")), One(Filepath), One(MailboxPath)],
+                  parser:(
+                      fn import(input: &[u8]) -> IResult<&[u8], Action> {
+                          let (input, _) = tag("import")(input.trim())?;
+                          let (input, _) = is_a(" ")(input)?;
+                          let (input, file) = quoted_argument(input)?;
+                          let (input, _) = is_a(" ")(input)?;
+                          let (input, mailbox_path) = quoted_argument(input)?;
+                          let (input, _) = eof(input)?;
+                          Ok((input, Listing(Import(file.to_string().into(), mailbox_path.to_string()))))
+                      }
+                  )
+                },
                  { tags: ["close"],
                    desc: "close non-sticky tabs",
                    tokens: &[One(Literal("close"))],
@@ -365,14 +380,16 @@ define_commands!([
                       }
                   )
                 },
-                { tags: ["toggle_thread_snooze"],
+                { tags: ["toggle thread_snooze"],
                   desc: "turn off new notifications for this thread",
-                  tokens: &[One(Literal("toggle_thread_snooze"))],
+                  tokens: &[One(Literal("toggle thread_snooze"))],
                   parser: (
                       fn toggle_thread_snooze(input: &[u8]) -> IResult<&[u8], Action> {
-                          let (input, _) = tag("toggle_thread_snooze")(input.trim())?;
+                          let (input, _) = tag("toggle")(input)?;
+                          let (input, _) = is_a(" ")(input)?;
+                          let (input, _) = tag("thread_snooze")(input)?;
                           let (input, _) = eof(input)?;
-                          Ok((input, ToggleThreadSnooze))
+                          Ok((input, Listing(ToggleThreadSnooze)))
                       }
                   )
                 },
@@ -402,6 +419,19 @@ define_commands!([
                       }
                   )
                 },
+                { tags: ["export-mbox "],
+                  desc: "export-mbox PATH",
+                  tokens: &[One(Literal("export-mbox")), One(Filepath)],
+                  parser:(
+                      fn export_mbox(input: &[u8]) -> IResult<&[u8], Action> {
+                          let (input, _) = tag("export-mbox")(input.trim())?;
+                          let (input, _) = is_a(" ")(input)?;
+                          let (input, path) = quoted_argument(input.trim())?;
+                          let (input, _) = eof(input)?;
+                          Ok((input, Listing(ExportMbox(Some(melib::backends::mbox::MboxFormat::MboxCl2), path.to_string().into()))))
+                      }
+                  )
+                },
                 { tags: ["list-archive", "list-post", "list-unsubscribe", "list-"],
                   desc: "list-[unsubscribe/post/archive]",
                   tokens: &[One(Alternatives(&[to_stream!(One(Literal("list-archive"))), to_stream!(One(Literal("list-post"))), to_stream!(One(Literal("list-unsubscribe")))]))],
@@ -480,9 +510,10 @@ define_commands!([
                       }
                   )
                 },
-                { tags: ["add-attachment "],
+                { tags: ["add-attachment ", "add-attachment-file-picker "],
                   desc: "add-attachment PATH",
-                  tokens: &[One(Literal("add-attachment")), One(Filepath)],
+                  tokens: &[One(
+Alternatives(&[to_stream!(One(Literal("add-attachment")), One(Filepath)), to_stream!(One(Literal("add-attachment-file-picker")))]))],
                   parser:(
                       fn add_attachment<'a>(input: &'a [u8]) -> IResult<&'a [u8], Action> {
                           alt((
@@ -500,6 +531,18 @@ define_commands!([
                                       let (input, path) = quoted_argument(input)?;
                                       let (input, _) = eof(input)?;
                                       Ok((input, Compose(AddAttachment(path.to_string()))))
+                                  }, |input: &'a [u8]| -> IResult<&'a [u8], Action> {
+                                      let (input, _) = tag("add-attachment-file-picker")(input.trim())?;
+                                      let (input, _) = eof(input)?;
+                                      Ok((input, Compose(AddAttachmentFilePicker(None))))
+                                  }, |input: &'a [u8]| -> IResult<&'a [u8], Action> {
+                                      let (input, _) = tag("add-attachment-file-picker")(input.trim())?;
+                                      let (input, _) = is_a(" ")(input)?;
+                                      let (input, _) = tag("<")(input.trim())?;
+                                      let (input, _) = is_a(" ")(input)?;
+                                      let (input, shell) = map_res(not_line_ending, std::str::from_utf8)(input)?;
+                                      let (input, _) = eof(input)?;
+                                      Ok((input, Compose(AddAttachmentFilePicker(Some(shell.to_string())))))
                                   }
                               ))(input)
                       }
@@ -542,6 +585,19 @@ define_commands!([
                       }
                   )
                 },
+                { tags: ["toggle encrypt"],
+                  desc: "toggle encryption for this draft",
+                  tokens: &[One(Literal("toggle")), One(Literal("encrypt"))],
+                  parser:(
+                      fn toggle_encrypt(input: &[u8]) -> IResult<&[u8], Action> {
+                          let (input, _) = tag("toggle")(input)?;
+                          let (input, _) = is_a(" ")(input)?;
+                          let (input, _) = tag("encrypt")(input)?;
+                          let (input, _) = eof(input)?;
+                          Ok((input, Compose(ToggleEncrypt)))
+                      }
+                  )
+                },
                 { tags: ["create-mailbox "],
                   desc: "create-mailbox ACCOUNT MAILBOX_PATH",
                   tokens: &[One(Literal("create-mailbox")), One(AccountName), One(MailboxPath)],
@@ -658,6 +714,19 @@ define_commands!([
                       }
                   )
                 },
+                { tags: ["export-mail "],
+                  desc: "export-mail PATH",
+                  tokens: &[One(Literal("export-mail")), One(Filepath)],
+                  parser:(
+                      fn export_mail(input: &[u8]) -> IResult<&[u8], Action> {
+                          let (input, _) = tag("export-mail")(input.trim())?;
+                          let (input, _) = is_a(" ")(input)?;
+                          let (input, path) = quoted_argument(input.trim())?;
+                          let (input, _) = eof(input)?;
+                          Ok((input, View(ExportMail(path.to_string()))))
+                      }
+                  )
+                },
                 { tags: ["tag", "tag add", "tag remove"],
                    desc: "tag [add/remove], edits message's tags.",
                    tokens: &[One(Literal("tag")), One(Alternatives(&[to_stream!(One(Literal("add"))), to_stream!(One(Literal("remove")))]))],
@@ -710,6 +779,41 @@ define_commands!([
                           Ok((input, PrintSetting(setting.to_string())))
                       }
                   )
+                },
+                { tags: ["toggle mouse"],
+                  desc: "toggle mouse support",
+                  tokens: &[One(Literal("toggle")), One(Literal("mouse"))],
+                  parser:(
+                      fn toggle_mouse(input: &[u8]) -> IResult<&[u8], Action> {
+                          let (input, _) = tag("toggle")(input)?;
+                          let (input, _) = is_a(" ")(input)?;
+                          let (input, _) = tag("mouse")(input)?;
+                          let (input, _) = eof(input)?;
+                          Ok((input, ToggleMouse))
+                      }
+                  )
+                },
+                { tags: ["quit"],
+                  desc: "quit meli",
+                  tokens: &[One(Literal("quit"))],
+                  parser:(
+                      fn quit(input: &[u8]) -> IResult<&[u8], Action> {
+                          let (input, _) = tag("quit")(input.trim())?;
+                          let (input, _) = eof(input.trim())?;
+                          Ok((input, Quit))
+                      }
+                  )
+                },
+                { tags: ["reload-config"],
+                  desc: "reload configuration file",
+                  tokens: &[One(Literal("reload-config"))],
+                  parser:(
+                      fn reload_config(input: &[u8]) -> IResult<&[u8], Action> {
+                          let (input, _) = tag("reload-config")(input.trim())?;
+                          let (input, _) = eof(input.trim())?;
+                          Ok((input, ReloadConfiguration))
+                      }
+                  )
                 }
 ]);
 
@@ -756,16 +860,24 @@ fn listing_action(input: &[u8]) -> IResult<&[u8], Action> {
         seen_flag,
         delete_message,
         copymove,
+        import,
         search,
         select,
         toggle_thread_snooze,
         open_in_new_tab,
+        export_mbox,
         _tag,
     ))(input)
 }
 
 fn compose_action(input: &[u8]) -> IResult<&[u8], Action> {
-    alt((add_attachment, remove_attachment, toggle_sign, save_draft))(input)
+    alt((
+        add_attachment,
+        remove_attachment,
+        toggle_sign,
+        toggle_encrypt,
+        save_draft,
+    ))(input)
 }
 
 fn account_action(input: &[u8]) -> IResult<&[u8], Action> {
@@ -773,7 +885,7 @@ fn account_action(input: &[u8]) -> IResult<&[u8], Action> {
 }
 
 fn view(input: &[u8]) -> IResult<&[u8], Action> {
-    alt((pipe, save_attachment))(input)
+    alt((pipe, save_attachment, export_mail))(input)
 }
 
 pub fn parse_command(input: &[u8]) -> Result<Action, MeliError> {
@@ -795,6 +907,9 @@ pub fn parse_command(input: &[u8]) -> Result<Action, MeliError> {
         rename_mailbox,
         account_action,
         print_setting,
+        toggle_mouse,
+        reload_config,
+        quit,
     ))(input)
     .map(|(_, v)| v)
     .map_err(|err| err.into())

src/command/actions.rs

@@ -25,6 +25,7 @@
 
 use crate::components::Component;
 pub use melib::thread::{SortField, SortOrder};
+use std::path::PathBuf;
 
 extern crate uuid;
 use uuid::Uuid;
@@ -49,9 +50,12 @@ pub enum ListingAction {
     CopyToOtherAccount(AccountName, MailboxPath),
     MoveTo(MailboxPath),
     MoveToOtherAccount(AccountName, MailboxPath),
+    Import(PathBuf, MailboxPath),
+    ExportMbox(Option<melib::backends::mbox::MboxFormat>, PathBuf),
     Delete,
     OpenInNewTab,
     Tag(TagAction),
+    ToggleThreadSnooze,
 }
 
 #[derive(Debug)]
@@ -72,15 +76,18 @@ pub enum MailingListAction {
 pub enum ViewAction {
     Pipe(String, Vec<String>),
     SaveAttachment(usize, String),
+    ExportMail(String),
 }
 
 #[derive(Debug)]
 pub enum ComposeAction {
     AddAttachment(String),
+    AddAttachmentFilePicker(Option<String>),
     AddAttachmentPipe(String),
     RemoveAttachment(usize),
     SaveDraft,
     ToggleSign,
+    ToggleEncrypt,
 }
 
 #[derive(Debug)]
@@ -107,7 +114,6 @@ pub enum Action {
     Sort(SortField, SortOrder),
     SubSort(SortField, SortOrder),
     Tab(TabAction),
-    ToggleThreadSnooze,
     MailingListAction(MailingListAction),
     View(ViewAction),
     SetEnv(String, String),
@@ -116,6 +122,9 @@ pub enum Action {
     Mailbox(AccountName, MailboxOperation),
     AccountAction(AccountName, AccountAction),
     PrintSetting(String),
+    ReloadConfiguration,
+    ToggleMouse,
+    Quit,
 }
 
 impl Action {
@@ -126,7 +135,6 @@ impl Action {
             Action::Sort(_, _) => false,
             Action::SubSort(_, _) => false,
             Action::Tab(_) => false,
-            Action::ToggleThreadSnooze => false,
             Action::MailingListAction(_) => true,
             Action::View(_) => false,
             Action::SetEnv(_, _) => false,
@@ -135,6 +143,9 @@ impl Action {
             Action::Mailbox(_, _) => true,
             Action::AccountAction(_, _) => false,
             Action::PrintSetting(_) => false,
+            Action::ToggleMouse => false,
+            Action::Quit => true,
+            Action::ReloadConfiguration => false,
         }
     }
 }

src/components.rs

@@ -54,6 +54,34 @@ pub type ComponentId = Uuid;
 pub type ShortcutMap = IndexMap<&'static str, Key>;
 pub type ShortcutMaps = IndexMap<&'static str, ShortcutMap>;
 
+#[derive(Debug, Clone, Copy)]
+pub enum PageMovement {
+    Up(usize),
+    Right(usize),
+    Left(usize),
+    Down(usize),
+    PageUp(usize),
+    PageDown(usize),
+    Home,
+    End,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct ScrollContext {
+    shown_lines: usize,
+    total_lines: usize,
+    has_more_lines: bool,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum ScrollUpdate {
+    End(ComponentId),
+    Update {
+        id: ComponentId,
+        context: ScrollContext,
+    },
+}
+
 /// Types implementing this Trait can draw on the terminal and receive events.
 /// If a type wants to skip drawing if it has not changed anything, it can hold some flag in its
 /// fields (eg self.dirty = false) and act upon that in their `draw` implementation.

src/components/contacts.rs

@@ -40,7 +40,7 @@ pub struct ContactManager {
     parent_id: ComponentId,
     pub card: Card,
     mode: ViewMode,
-    form: FormWidget,
+    form: FormWidget<bool>,
     account_pos: usize,
     content: CellBuffer,
     theme_default: ThemeAttribute,
@@ -59,13 +59,6 @@ impl fmt::Display for ContactManager {
 impl ContactManager {
     fn new(context: &Context) -> Self {
         let theme_default: ThemeAttribute = crate::conf::value(context, "theme_default");
-        let default_cell = {
-            let mut ret = Cell::with_char(' ');
-            ret.set_fg(theme_default.fg)
-                .set_bg(theme_default.bg)
-                .set_attrs(theme_default.attrs);
-            ret
-        };
         ContactManager {
             id: Uuid::nil(),
             parent_id: Uuid::nil(),
@@ -73,7 +66,7 @@ impl ContactManager {
             mode: ViewMode::Edit,
             form: FormWidget::default(),
             account_pos: 0,
-            content: CellBuffer::new(100, 1, default_cell),
+            content: CellBuffer::new_with_context(100, 1, None, context),
             theme_default,
             dirty: true,
             has_changes: false,
@@ -105,8 +98,7 @@ impl ContactManager {
 
         if self.card.external_resource() {
             self.mode = ViewMode::ReadOnly;
-            self.content
-                .resize(self.content.size().0, 2, Cell::default());
+            let _ = self.content.resize(self.content.size().0, 2, None);
             write_string_to_grid(
                 "This contact's origin is external and cannot be edited within meli.",
                 &mut self.content,
@@ -118,24 +110,30 @@ impl ContactManager {
             );
         }
 
-        self.form = FormWidget::new("Save".into());
+        self.form = FormWidget::new(("Save".into(), true));
         self.form.add_button(("Cancel(Esc)".into(), false));
         self.form
-            .push(("NAME".into(), self.card.name().to_string()));
+            .push(("NAME".into(), self.card.name().to_string().into()));
         self.form.push((
             "ADDITIONAL NAME".into(),
-            self.card.additionalname().to_string(),
+            self.card.additionalname().to_string().into(),
+        ));
+        self.form.push((
+            "NAME PREFIX".into(),
+            self.card.name_prefix().to_string().into(),
+        ));
+        self.form.push((
+            "NAME SUFFIX".into(),
+            self.card.name_suffix().to_string().into(),
         ));
         self.form
-            .push(("NAME PREFIX".into(), self.card.name_prefix().to_string()));
+            .push(("E-MAIL".into(), self.card.email().to_string().into()));
         self.form
-            .push(("NAME SUFFIX".into(), self.card.name_suffix().to_string()));
+            .push(("URL".into(), self.card.url().to_string().into()));
         self.form
-            .push(("E-MAIL".into(), self.card.email().to_string()));
-        self.form.push(("URL".into(), self.card.url().to_string()));
-        self.form.push(("KEY".into(), self.card.key().to_string()));
+            .push(("KEY".into(), self.card.key().to_string().into()));
         for (k, v) in self.card.extra_properties() {
-            self.form.push((k.to_string(), v.to_string()));
+            self.form.push((k.to_string().into(), v.to_string().into()));
         }
     }
 
@@ -173,7 +171,7 @@ impl Component for ContactManager {
         match self.mode {
             ViewMode::Discard(ref mut selector) => {
                 /* Let user choose whether to quit with/without saving or cancel */
-                selector.draw(grid, center_area(area, selector.content.size()), context);
+                selector.draw(grid, area, context);
             }
             _ => {}
         }
@@ -182,6 +180,15 @@ impl Component for ContactManager {
     }
 
     fn process_event(&mut self, event: &mut UIEvent, context: &mut Context) -> bool {
+        match event {
+            UIEvent::ConfigReload { old_settings: _ } => {
+                self.theme_default = crate::conf::value(context, "theme_default");
+                self.content = CellBuffer::new_with_context(100, 1, None, context);
+                self.initialized = false;
+                self.set_dirty(true);
+            }
+            _ => {}
+        }
         match self.mode {
             ViewMode::Discard(ref mut selector) => {
                 if selector.process_event(event, context) {
@@ -209,10 +216,10 @@ impl Component for ContactManager {
                                 .into_iter()
                                 .map(|(s, v)| {
                                     (
-                                        s,
+                                        s.to_string(),
                                         match v {
                                             Field::Text(v, _) => v.as_str().to_string(),
-                                            Field::Choice(mut v, c) => v.remove(c),
+                                            Field::Choice(mut v, c) => v.remove(c).to_string(),
                                         },
                                     )
                                 })

src/components/contacts/contact_list.rs

@@ -54,7 +54,10 @@ pub struct ContactList {
 
     mode: ViewMode,
     dirty: bool,
-    show_divider: bool,
+
+    sidebar_divider: char,
+    sidebar_divider_theme: ThemeAttribute,
+
     menu_visibility: bool,
     movement: Option<PageMovement>,
     cmd_buf: String,
@@ -98,7 +101,8 @@ impl ContactList {
             cmd_buf: String::with_capacity(8),
             view: None,
             ratio: 90,
-            show_divider: false,
+            sidebar_divider: context.settings.listing.sidebar_divider,
+            sidebar_divider_theme: conf::value(context, "mail.sidebar_divider"),
             menu_visibility: true,
             id: ComponentId::new_v4(),
         }
@@ -132,25 +136,18 @@ impl ContactList {
             min_width.2 = cmp::max(min_width.2, c.url().split_graphemes().len());
         }
 
-        let default_cell = {
-            let mut ret = Cell::with_char(' ');
-            ret.set_fg(self.theme_default.fg)
-                .set_bg(self.theme_default.bg)
-                .set_attrs(self.theme_default.attrs);
-            ret
-        };
         /* name column */
         self.data_columns.columns[0] =
-            CellBuffer::new_with_context(min_width.0, self.length, default_cell, context);
+            CellBuffer::new_with_context(min_width.0, self.length, None, context);
         /* email column */
         self.data_columns.columns[1] =
-            CellBuffer::new_with_context(min_width.1, self.length, default_cell, context);
+            CellBuffer::new_with_context(min_width.1, self.length, None, context);
         /* url column */
         self.data_columns.columns[2] =
-            CellBuffer::new_with_context(min_width.2, self.length, default_cell, context);
+            CellBuffer::new_with_context(min_width.2, self.length, None, context);
         /* source column */
         self.data_columns.columns[3] =
-            CellBuffer::new_with_context("external".len(), self.length, default_cell, context);
+            CellBuffer::new_with_context("external".len(), self.length, None, context);
 
         let account = &context.accounts[self.account_pos];
         let book = &account.address_book;
@@ -205,16 +202,9 @@ impl ContactList {
         }
 
         if self.length == 0 {
-            let default_cell = {
-                let mut ret = Cell::with_char(' ');
-                ret.set_fg(self.theme_default.fg)
-                    .set_bg(self.theme_default.bg)
-                    .set_attrs(self.theme_default.attrs);
-                ret
-            };
             let message = "Address book is empty.".to_string();
             self.data_columns.columns[0] =
-                CellBuffer::new_with_context(message.len(), self.length, default_cell, context);
+                CellBuffer::new_with_context(message.len(), self.length, None, context);
             write_string_to_grid(
                 &message,
                 &mut self.data_columns.columns[0],
@@ -239,13 +229,11 @@ impl ContactList {
         change_colors(grid, area, fg_color, bg_color);
     }
 
-    fn draw_menu(&mut self, grid: &mut CellBuffer, mut area: Area, context: &mut Context) {
+    fn draw_menu(&mut self, grid: &mut CellBuffer, area: Area, context: &mut Context) {
         if !self.is_dirty() {
             return;
         }
         clear_area(grid, area, self.theme_default);
-        /* visually divide menu and listing */
-        area = (area.0, pos_dec(area.1, (1, 0)));
         let upper_left = upper_left!(area);
         let bottom_right = bottom_right!(area);
         self.dirty = false;
@@ -273,28 +261,35 @@ impl ContactList {
 
         let width = width!(area);
         let must_highlight_account: bool = self.account_pos == a.index;
-        let (fg_color, bg_color) = if must_highlight_account {
-            if self.account_pos == a.index {
-                (Color::Byte(233), Color::Byte(15))
-            } else {
-                (Color::Byte(15), Color::Byte(233))
+        let account_attrs = if must_highlight_account {
+            let mut v = crate::conf::value(context, "mail.sidebar_highlighted");
+            if !context.settings.terminal.use_color() {
+                v.attrs |= Attr::REVERSE;
             }
+            v
         } else {
-            (self.theme_default.fg, self.theme_default.bg)
+            crate::conf::value(context, "mail.sidebar_account_name")
         };
 
         let s = format!(" [{}]", context.accounts[a.index].address_book.len());
 
         if a.name.grapheme_len() + s.len() > width + 1 {
             /* Print account name */
-            let (x, y) =
-                write_string_to_grid(&a.name, grid, fg_color, bg_color, Attr::BOLD, area, None);
+            let (x, y) = write_string_to_grid(
+                &a.name,
+                grid,
+                account_attrs.fg,
+                account_attrs.bg,
+                account_attrs.attrs,
+                area,
+                None,
+            );
             write_string_to_grid(
                 &s,
                 grid,
-                fg_color,
-                bg_color,
-                Attr::BOLD,
+                account_attrs.fg,
+                account_attrs.bg,
+                account_attrs.attrs,
                 (
                     pos_dec(
                         (get_x(bottom_right!(area)), get_y(upper_left!(area))),
@@ -307,9 +302,9 @@ impl ContactList {
             write_string_to_grid(
                 "…",
                 grid,
-                fg_color,
-                bg_color,
-                Attr::BOLD,
+                account_attrs.fg,
+                account_attrs.bg,
+                account_attrs.attrs,
                 (
                     pos_dec(
                         (get_x(bottom_right!(area)), get_y(upper_left!(area))),
@@ -321,20 +316,29 @@ impl ContactList {
             );
 
             for x in x..=get_x(bottom_right!(area)) {
-                grid[(x, y)].set_fg(fg_color);
-                grid[(x, y)].set_bg(bg_color);
+                grid[(x, y)]
+                    .set_fg(account_attrs.fg)
+                    .set_bg(account_attrs.bg)
+                    .set_attrs(account_attrs.attrs);
             }
         } else {
             /* Print account name */
 
-            let (x, y) =
-                write_string_to_grid(&a.name, grid, fg_color, bg_color, Attr::BOLD, area, None);
+            let (x, y) = write_string_to_grid(
+                &a.name,
+                grid,
+                account_attrs.fg,
+                account_attrs.bg,
+                account_attrs.attrs,
+                area,
+                None,
+            );
             write_string_to_grid(
                 &s,
                 grid,
-                fg_color,
-                bg_color,
-                Attr::BOLD,
+                account_attrs.fg,
+                account_attrs.bg,
+                account_attrs.attrs,
                 (
                     pos_dec(
                         (get_x(bottom_right!(area)), get_y(upper_left!(area))),
@@ -345,8 +349,10 @@ impl ContactList {
                 None,
             );
             for x in x..=get_x(bottom_right!(area)) {
-                grid[(x, y)].set_fg(fg_color);
-                grid[(x, y)].set_bg(bg_color);
+                grid[(x, y)]
+                    .set_fg(account_attrs.fg)
+                    .set_bg(account_attrs.bg)
+                    .set_attrs(account_attrs.attrs);
             }
         }
     }
@@ -408,6 +414,27 @@ impl ContactList {
 
         let top_idx = page_no * rows;
 
+        if self.length >= rows {
+            context
+                .replies
+                .push_back(UIEvent::StatusEvent(StatusEvent::ScrollUpdate(
+                    ScrollUpdate::Update {
+                        id: self.id,
+                        context: ScrollContext {
+                            shown_lines: top_idx + rows,
+                            total_lines: self.length,
+                            has_more_lines: false,
+                        },
+                    },
+                )));
+        } else {
+            context
+                .replies
+                .push_back(UIEvent::StatusEvent(StatusEvent::ScrollUpdate(
+                    ScrollUpdate::End(self.id),
+                )));
+        }
+
         /* If cursor position has changed, remove the highlight from the previous position and
          * apply it in the new one. */
         if self.cursor_pos != self.new_cursor_pos && prev_page_no == page_no {
@@ -562,19 +589,12 @@ impl Component for ContactList {
         };
         let mid = get_x(bottom_right) - right_component_width;
         if self.dirty && mid != get_x(upper_left) {
-            if self.show_divider {
-                for i in get_y(upper_left)..=get_y(bottom_right) {
-                    grid[(mid, i)]
-                        .set_ch(VERT_BOUNDARY)
-                        .set_fg(self.theme_default.fg)
-                        .set_bg(self.theme_default.bg);
-                }
-            } else {
-                for i in get_y(upper_left)..=get_y(bottom_right) {
-                    grid[(mid, i)]
-                        .set_fg(self.theme_default.fg)
-                        .set_bg(self.theme_default.bg);
-                }
+            for i in get_y(upper_left)..=get_y(bottom_right) {
+                grid[(mid, i)]
+                    .set_ch(self.sidebar_divider)
+                    .set_fg(self.sidebar_divider_theme.fg)
+                    .set_bg(self.sidebar_divider_theme.bg)
+                    .set_attrs(self.sidebar_divider_theme.attrs);
             }
             context
                 .dirty_areas
@@ -586,13 +606,25 @@ impl Component for ContactList {
         } else if right_component_width == 0 {
             self.draw_menu(grid, area, context);
         } else {
-            self.draw_menu(grid, (upper_left, (mid, get_y(bottom_right))), context);
+            self.draw_menu(
+                grid,
+                (upper_left, (mid.saturating_sub(1), get_y(bottom_right))),
+                context,
+            );
             self.draw_list(grid, (set_x(upper_left, mid + 1), bottom_right), context);
         }
         self.dirty = false;
     }
 
     fn process_event(&mut self, event: &mut UIEvent, context: &mut Context) -> bool {
+        if let UIEvent::ConfigReload { old_settings: _ } = event {
+            self.theme_default = crate::conf::value(context, "theme_default");
+            self.initialized = false;
+            self.sidebar_divider = context.settings.listing.sidebar_divider;
+            self.sidebar_divider_theme = conf::value(context, "mail.sidebar_divider");
+            self.set_dirty(true);
+        }
+
         if let Some(ref mut v) = self.view {
             if v.process_event(event, context) {
                 return true;
@@ -610,6 +642,11 @@ impl Component for ContactList {
 
                     self.mode = ViewMode::View(manager.id());
                     self.view = Some(manager);
+                    context
+                        .replies
+                        .push_back(UIEvent::StatusEvent(StatusEvent::ScrollUpdate(
+                            ScrollUpdate::End(self.id),
+                        )));
 
                     return true;
                 }
@@ -628,6 +665,11 @@ impl Component for ContactList {
 
                     self.mode = ViewMode::View(manager.id());
                     self.view = Some(manager);
+                    context
+                        .replies
+                        .push_back(UIEvent::StatusEvent(StatusEvent::ScrollUpdate(
+                            ScrollUpdate::End(self.id),
+                        )));
 
                     return true;
                 }
@@ -642,7 +684,7 @@ impl Component for ContactList {
                     let mut draft: Draft = Draft::default();
                     *draft.headers_mut().get_mut("To").unwrap() =
                         format!("{} <{}>", &card.name(), &card.email());
-                    let mut composer = Composer::new(account_hash, context);
+                    let mut composer = Composer::with_account(account_hash, context);
                     composer.set_draft(draft);
                     context
                         .replies

src/components/mail.rs

@@ -33,6 +33,7 @@ pub use crate::view::*;
 mod compose;
 pub use self::compose::*;
 
+#[cfg(feature = "gpgme")]
 pub mod pgp;
 
 mod status;

src/components/mail/compose/edit_attachments.rs

@@ -0,0 +1,312 @@
+/*
+ * meli -
+ *
+ * Copyright  Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+
+#[derive(Debug, Copy, Clone, PartialEq)]
+pub enum EditAttachmentCursor {
+    AttachmentNo(usize),
+    Buttons,
+}
+
+impl Default for EditAttachmentCursor {
+    fn default() -> Self {
+        EditAttachmentCursor::Buttons
+    }
+}
+
+#[derive(Debug)]
+pub enum EditAttachmentMode {
+    Overview,
+    Edit {
+        inner: FormWidget<FormButtonActions>,
+        no: usize,
+    },
+}
+
+#[derive(Debug)]
+pub struct EditAttachments {
+    pub mode: EditAttachmentMode,
+    pub buttons: ButtonWidget<FormButtonActions>,
+    pub cursor: EditAttachmentCursor,
+    pub dirty: bool,
+    pub id: ComponentId,
+}
+
+impl EditAttachments {
+    pub fn new() -> Self {
+        let mut buttons = ButtonWidget::new(("Add".into(), FormButtonActions::Other("add")));
+        buttons.push(("Go Back".into(), FormButtonActions::Cancel));
+        buttons.set_focus(true);
+        buttons.set_cursor(1);
+        EditAttachments {
+            mode: EditAttachmentMode::Overview,
+            buttons,
+            cursor: EditAttachmentCursor::Buttons,
+            dirty: true,
+            id: ComponentId::new_v4(),
+        }
+    }
+}
+
+impl EditAttachmentsRefMut<'_, '_> {
+    fn new_edit_widget(&self, no: usize) -> Option<FormWidget<FormButtonActions>> {
+        if no >= self.draft.attachments().len() {
+            return None;
+        }
+        let filename = self.draft.attachments()[no].content_type().name();
+        let mime_type = self.draft.attachments()[no].content_type();
+        let mut ret = FormWidget::new(("Save".into(), FormButtonActions::Accept));
+
+        ret.add_button(("Reset".into(), FormButtonActions::Reset));
+        ret.add_button(("Cancel".into(), FormButtonActions::Cancel));
+        ret.push(("Filename".into(), filename.unwrap_or_default().to_string()));
+        ret.push(("Mime type".into(), mime_type.to_string()));
+        Some(ret)
+    }
+}
+
+#[derive(Debug)]
+pub struct EditAttachmentsRefMut<'a, 'b> {
+    pub inner: &'a mut EditAttachments,
+    pub draft: &'b mut Draft,
+}
+
+impl std::fmt::Display for EditAttachmentsRefMut<'_, '_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "edit attachments")
+    }
+}
+
+impl Component for EditAttachmentsRefMut<'_, '_> {
+    fn draw(&mut self, grid: &mut CellBuffer, area: Area, context: &mut Context) {
+        if let EditAttachmentMode::Edit {
+            ref mut inner,
+            no: _,
+        } = self.inner.mode
+        {
+            inner.draw(grid, area, context);
+        } else if self.is_dirty() {
+            let attachments_no = self.draft.attachments().len();
+            let theme_default = crate::conf::value(context, "theme_default");
+            clear_area(grid, area, theme_default);
+            if attachments_no == 0 {
+                write_string_to_grid(
+                    "no attachments",
+                    grid,
+                    theme_default.fg,
+                    theme_default.bg,
+                    theme_default.attrs,
+                    area,
+                    None,
+                );
+            } else {
+                write_string_to_grid(
+                    &format!("{} attachments ", attachments_no),
+                    grid,
+                    theme_default.fg,
+                    theme_default.bg,
+                    theme_default.attrs,
+                    area,
+                    None,
+                );
+                for (i, a) in self.draft.attachments().iter().enumerate() {
+                    let bg = if let EditAttachmentCursor::AttachmentNo(u) = self.inner.cursor {
+                        if u == i {
+                            Color::Byte(237)
+                        } else {
+                            theme_default.bg
+                        }
+                    } else {
+                        theme_default.bg
+                    };
+                    if let Some(name) = a.content_type().name() {
+                        write_string_to_grid(
+                            &format!(
+                                "[{}] \"{}\", {} {}",
+                                i,
+                                name,
+                                a.content_type(),
+                                melib::Bytes(a.raw.len())
+                            ),
+                            grid,
+                            theme_default.fg,
+                            bg,
+                            theme_default.attrs,
+                            (pos_inc(upper_left!(area), (0, 1 + i)), bottom_right!(area)),
+                            None,
+                        );
+                    } else {
+                        write_string_to_grid(
+                            &format!("[{}] {} {}", i, a.content_type(), melib::Bytes(a.raw.len())),
+                            grid,
+                            theme_default.fg,
+                            bg,
+                            theme_default.attrs,
+                            (pos_inc(upper_left!(area), (0, 1 + i)), bottom_right!(area)),
+                            None,
+                        );
+                    }
+                }
+            }
+            self.inner.buttons.draw(
+                grid,
+                (
+                    pos_inc(upper_left!(area), (0, 1 + self.draft.attachments().len())),
+                    bottom_right!(area),
+                ),
+                context,
+            );
+            self.set_dirty(false);
+            context.dirty_areas.push_back(area);
+        }
+    }
+
+    fn process_event(&mut self, event: &mut UIEvent, context: &mut Context) -> bool {
+        if let EditAttachmentMode::Edit {
+            ref mut inner,
+            ref no,
+        } = self.inner.mode
+        {
+            if inner.process_event(event, context) {
+                match inner.buttons_result() {
+                    Some(FormButtonActions::Accept) | Some(FormButtonActions::Cancel) => {
+                        self.inner.mode = EditAttachmentMode::Overview;
+                    }
+                    Some(FormButtonActions::Reset) => {
+                        let no = *no;
+                        if let Some(inner) = self.new_edit_widget(no) {
+                            self.inner.mode = EditAttachmentMode::Edit { inner, no };
+                        }
+                    }
+                    Some(_) | None => {}
+                }
+                return true;
+            }
+        } else {
+            match event {
+                UIEvent::Input(Key::Up) => {
+                    self.set_dirty(true);
+                    match self.inner.cursor {
+                        EditAttachmentCursor::AttachmentNo(ref mut n) => {
+                            if self.draft.attachments().is_empty() {
+                                self.inner.cursor = EditAttachmentCursor::Buttons;
+                                self.inner.buttons.set_focus(true);
+                                self.inner.buttons.process_event(event, context);
+                                return true;
+                            }
+                            *n = n.saturating_sub(1);
+                        }
+                        EditAttachmentCursor::Buttons => {
+                            if !self.inner.buttons.process_event(event, context) {
+                                self.inner.buttons.set_focus(false);
+                                if self.draft.attachments().is_empty() {
+                                    return true;
+                                }
+                                self.inner.cursor = EditAttachmentCursor::AttachmentNo(
+                                    self.draft.attachments().len() - 1,
+                                );
+                            }
+                        }
+                    }
+                    return true;
+                }
+                UIEvent::Input(Key::Down) => {
+                    self.set_dirty(true);
+                    match self.inner.cursor {
+                        EditAttachmentCursor::AttachmentNo(ref mut n) => {
+                            if *n + 1 == self.draft.attachments().len() {
+                                self.inner.cursor = EditAttachmentCursor::Buttons;
+                                self.inner.buttons.set_focus(true);
+                                self.inner.buttons.process_event(event, context);
+                                return true;
+                            }
+                            *n += 1;
+                        }
+                        EditAttachmentCursor::Buttons => {
+                            self.inner.buttons.set_focus(true);
+                            self.inner.buttons.process_event(event, context);
+                        }
+                    }
+                    return true;
+                }
+                UIEvent::Input(Key::Char('\n')) => {
+                    match self.inner.cursor {
+                        EditAttachmentCursor::AttachmentNo(ref no) => {
+                            if let Some(inner) = self.new_edit_widget(*no) {
+                                self.inner.mode = EditAttachmentMode::Edit { inner, no: *no };
+                            }
+                            self.set_dirty(true);
+                        }
+                        EditAttachmentCursor::Buttons => {
+                            self.inner.buttons.process_event(event, context);
+                        }
+                    }
+                    return true;
+                }
+                _ => {
+                    if self.inner.cursor == EditAttachmentCursor::Buttons
+                        && self.inner.buttons.process_event(event, context)
+                    {
+                        return true;
+                    }
+                }
+            }
+        }
+        false
+    }
+
+    fn is_dirty(&self) -> bool {
+        self.inner.dirty
+            || self.inner.buttons.is_dirty()
+            || if let EditAttachmentMode::Edit { ref inner, no: _ } = self.inner.mode {
+                inner.is_dirty()
+            } else {
+                false
+            }
+    }
+
+    fn set_dirty(&mut self, value: bool) {
+        self.inner.dirty = value;
+        self.inner.buttons.set_dirty(value);
+        if let EditAttachmentMode::Edit {
+            ref mut inner,
+            no: _,
+        } = self.inner.mode
+        {
+            inner.set_dirty(value);
+        }
+    }
+
+    fn kill(&mut self, _uuid: Uuid, _context: &mut Context) {}
+
+    fn get_shortcuts(&self, _context: &Context) -> ShortcutMaps {
+        ShortcutMaps::default()
+    }
+
+    fn id(&self) -> ComponentId {
+        self.inner.id
+    }
+
+    fn set_id(&mut self, new_id: ComponentId) {
+        self.inner.id = new_id;
+    }
+}

src/components/mail/compose/gpg.rs

@@ -0,0 +1,288 @@
+/*
+ * meli
+ *
+ * Copyright 2020  Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+
+#[derive(Debug)]
+pub enum KeySelection {
+    LoadingKeys {
+        handle: JoinHandle<Result<Vec<melib::gpgme::Key>>>,
+        progress_spinner: ProgressSpinner,
+        secret: bool,
+        local: bool,
+        pattern: String,
+        allow_remote_lookup: ToggleFlag,
+    },
+    Error {
+        id: ComponentId,
+        err: MeliError,
+    },
+    Loaded {
+        widget: UIDialog<melib::gpgme::Key>,
+        keys: Vec<melib::gpgme::Key>,
+    },
+}
+
+impl std::fmt::Display for KeySelection {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "select pgp keys")
+    }
+}
+
+impl KeySelection {
+    pub fn new(
+        secret: bool,
+        local: bool,
+        pattern: String,
+        allow_remote_lookup: ToggleFlag,
+        context: &mut Context,
+    ) -> Result<Self> {
+        use melib::gpgme::*;
+        debug!("KeySelection::new");
+        debug!(&secret);
+        debug!(&local);
+        debug!(&pattern);
+        debug!(&allow_remote_lookup);
+        let mut ctx = Context::new()?;
+        if local {
+            ctx.set_auto_key_locate(LocateKey::LOCAL)?;
+        } else {
+            ctx.set_auto_key_locate(LocateKey::WKD | LocateKey::LOCAL)?;
+        }
+        let job = ctx.keylist(secret, Some(pattern.clone()))?;
+        let handle = context.job_executor.spawn_specialized(job);
+        let mut progress_spinner = ProgressSpinner::new(8, context);
+        progress_spinner.start();
+        Ok(KeySelection::LoadingKeys {
+            handle,
+            secret,
+            local,
+            pattern,
+            allow_remote_lookup,
+            progress_spinner,
+        })
+    }
+}
+
+impl Component for KeySelection {
+    fn draw(&mut self, grid: &mut CellBuffer, area: Area, context: &mut Context) {
+        match self {
+            KeySelection::LoadingKeys {
+                ref mut progress_spinner,
+                ..
+            } => progress_spinner.draw(grid, center_area(area, (2, 2)), context),
+            KeySelection::Error { ref err, .. } => {
+                let theme_default = crate::conf::value(context, "theme_default");
+                write_string_to_grid(
+                    &err.to_string(),
+                    grid,
+                    theme_default.fg,
+                    theme_default.bg,
+                    theme_default.attrs,
+                    center_area(area, (15, 2)),
+                    Some(0),
+                );
+            }
+            KeySelection::Loaded { ref mut widget, .. } => widget.draw(grid, area, context),
+        }
+    }
+
+    fn process_event(&mut self, event: &mut UIEvent, context: &mut Context) -> bool {
+        debug!(&self);
+        debug!(&event);
+        match self {
+            KeySelection::LoadingKeys {
+                ref mut progress_spinner,
+                ref mut handle,
+                secret,
+                local,
+                ref mut pattern,
+                allow_remote_lookup,
+                ..
+            } => match event {
+                UIEvent::StatusEvent(StatusEvent::JobFinished(ref id)) if *id == handle.job_id => {
+                    match handle.chan.try_recv() {
+                        Err(_) => { /* Job was canceled */ }
+                        Ok(None) => { /* something happened, perhaps a worker thread panicked */ }
+                        Ok(Some(Ok(keys))) => {
+                            if keys.is_empty() {
+                                let id = progress_spinner.id();
+                                if allow_remote_lookup.is_true() {
+                                    match Self::new(
+                                        *secret,
+                                        *local,
+                                        std::mem::replace(pattern, String::new()),
+                                        *allow_remote_lookup,
+                                        context,
+                                    ) {
+                                        Ok(w) => {
+                                            *self = w;
+                                        }
+                                        Err(err) => *self = KeySelection::Error { err, id },
+                                    }
+                                } else if !*local && allow_remote_lookup.is_ask() {
+                                    *self = KeySelection::Error {
+                                        err: MeliError::new(format!(
+                                            "No keys found for {}, perform remote lookup?",
+                                            pattern
+                                        )),
+                                        id,
+                                    }
+                                } else {
+                                    *self = KeySelection::Error {
+                                        err: MeliError::new(format!(
+                                            "No keys found for {}.",
+                                            pattern
+                                        )),
+                                        id,
+                                    }
+                                }
+                                if let KeySelection::Error { ref err, .. } = self {
+                                    context.replies.push_back(UIEvent::StatusEvent(
+                                        StatusEvent::DisplayMessage(err.to_string()),
+                                    ));
+                                    let res: Option<melib::gpgme::Key> = None;
+                                    context
+                                        .replies
+                                        .push_back(UIEvent::FinishedUIDialog(id, Box::new(res)));
+                                }
+                                return false;
+                            }
+                            let mut widget = UIDialog::new(
+                                "select key",
+                                keys.iter()
+                                    .map(|k| {
+                                        (
+                                            k.clone(),
+                                            if let Some(primary_uid) = k.primary_uid() {
+                                                format!("{} {}", k.fingerprint(), primary_uid)
+                                            } else {
+                                                k.fingerprint().to_string()
+                                            },
+                                        )
+                                    })
+                                    .collect::<Vec<(melib::gpgme::Key, String)>>(),
+                                true,
+                                Some(Box::new(
+                                    move |id: ComponentId, results: &[melib::gpgme::Key]| {
+                                        Some(UIEvent::FinishedUIDialog(
+                                            id,
+                                            Box::new(results.get(0).map(|k| k.clone())),
+                                        ))
+                                    },
+                                )),
+                                context,
+                            );
+                            widget.set_dirty(true);
+                            *self = KeySelection::Loaded { widget, keys };
+                        }
+                        Ok(Some(Err(err))) => {
+                            *self = KeySelection::Error {
+                                err,
+                                id: ComponentId::new_v4(),
+                            };
+                        }
+                    }
+                    false
+                }
+                _ => progress_spinner.process_event(event, context),
+            },
+            KeySelection::Error { .. } => false,
+            KeySelection::Loaded { ref mut widget, .. } => widget.process_event(event, context),
+        }
+    }
+
+    fn is_dirty(&self) -> bool {
+        match self {
+            KeySelection::LoadingKeys {
+                ref progress_spinner,
+                ..
+            } => progress_spinner.is_dirty(),
+            KeySelection::Error { .. } => true,
+            KeySelection::Loaded { ref widget, .. } => widget.is_dirty(),
+        }
+    }
+
+    fn set_dirty(&mut self, value: bool) {
+        match self {
+            KeySelection::LoadingKeys {
+                ref mut progress_spinner,
+                ..
+            } => progress_spinner.set_dirty(value),
+            KeySelection::Error { .. } => {}
+            KeySelection::Loaded { ref mut widget, .. } => widget.set_dirty(value),
+        }
+    }
+
+    fn kill(&mut self, _uuid: Uuid, _context: &mut Context) {}
+
+    fn get_shortcuts(&self, context: &Context) -> ShortcutMaps {
+        match self {
+            KeySelection::LoadingKeys { .. } | KeySelection::Error { .. } => {
+                ShortcutMaps::default()
+            }
+            KeySelection::Loaded { ref widget, .. } => widget.get_shortcuts(context),
+        }
+    }
+
+    fn id(&self) -> ComponentId {
+        match self {
+            KeySelection::LoadingKeys {
+                ref progress_spinner,
+                ..
+            } => progress_spinner.id(),
+            KeySelection::Error { ref id, .. } => *id,
+            KeySelection::Loaded { ref widget, .. } => widget.id(),
+        }
+    }
+
+    fn set_id(&mut self, new_id: ComponentId) {
+        match self {
+            KeySelection::LoadingKeys {
+                ref mut progress_spinner,
+                ..
+            } => progress_spinner.set_id(new_id),
+            KeySelection::Error { ref mut id, .. } => *id = new_id,
+            KeySelection::Loaded { ref mut widget, .. } => widget.set_id(new_id),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct GpgComposeState {
+    pub sign_mail: ToggleFlag,
+    pub encrypt_mail: ToggleFlag,
+    pub encrypt_keys: Vec<melib::gpgme::Key>,
+    pub encrypt_for_self: bool,
+    pub sign_keys: Vec<melib::gpgme::Key>,
+}
+
+impl GpgComposeState {
+    pub fn new() -> Self {
+        GpgComposeState {
+            sign_mail: ToggleFlag::Unset,
+            encrypt_mail: ToggleFlag::Unset,
+            encrypt_keys: vec![],
+            encrypt_for_self: true,
+            sign_keys: vec![],
+        }
+    }
+}

src/components/mail/listing/compact.rs

@@ -19,10 +19,9 @@
  * along with meli. If not, see <http://www.gnu.org/licenses/>.
  */
 
-use super::EntryStrings;
 use super::*;
-use crate::components::utilities::PageMovement;
-use crate::jobs::{oneshot, JobId};
+use crate::components::PageMovement;
+use crate::jobs::JoinHandle;
 use std::cmp;
 use std::convert::TryInto;
 use std::iter::FromIterator;
@@ -136,16 +135,8 @@ pub struct CompactListing {
     rows_drawn: SegmentTree,
     rows: Vec<((usize, (ThreadHash, EnvelopeHash)), EntryStrings)>,
 
-    search_job: Option<(
-        String,
-        oneshot::Receiver<Result<SmallVec<[EnvelopeHash; 512]>>>,
-        JobId,
-    )>,
-    select_job: Option<(
-        String,
-        oneshot::Receiver<Result<SmallVec<[EnvelopeHash; 512]>>>,
-        JobId,
-    )>,
+    search_job: Option<(String, JoinHandle<Result<SmallVec<[EnvelopeHash; 512]>>>)>,
+    select_job: Option<(String, JoinHandle<Result<SmallVec<[EnvelopeHash; 512]>>>)>,
     filter_term: String,
     filtered_selection: Vec<ThreadHash>,
     filtered_order: HashMap<ThreadHash, usize>,
@@ -161,7 +152,7 @@ pub struct CompactListing {
 
     movement: Option<PageMovement>,
     modifier_active: bool,
-    modifier_command: Option<char>,
+    modifier_command: Option<Modifier>,
     id: ComponentId,
 }
 
@@ -236,17 +227,10 @@ impl MailListingTrait for CompactListing {
         match context.accounts[&self.cursor_pos.0].load(self.cursor_pos.1) {
             Ok(()) => {}
             Err(_) => {
-                let default_cell = {
-                    let mut ret = Cell::with_char(' ');
-                    ret.set_fg(self.color_cache.theme_default.fg)
-                        .set_bg(self.color_cache.theme_default.bg)
-                        .set_attrs(self.color_cache.theme_default.attrs);
-                    ret
-                };
                 let message: String =
                     context.accounts[&self.cursor_pos.0][&self.cursor_pos.1].status();
                 self.data_columns.columns[0] =
-                    CellBuffer::new_with_context(message.len(), 1, default_cell, context);
+                    CellBuffer::new_with_context(message.len(), 1, None, context);
                 self.length = 0;
                 write_string_to_grid(
                     message.as_str(),
@@ -401,30 +385,23 @@ impl MailListingTrait for CompactListing {
 
         min_width.0 = self.length.saturating_sub(1).to_string().len();
 
-        let default_cell = {
-            let mut ret = Cell::with_char(' ');
-            ret.set_fg(self.color_cache.theme_default.fg)
-                .set_bg(self.color_cache.theme_default.bg)
-                .set_attrs(self.color_cache.theme_default.attrs);
-            ret
-        };
         /* index column */
         self.data_columns.columns[0] =
-            CellBuffer::new_with_context(min_width.0, rows.len(), default_cell, context);
+            CellBuffer::new_with_context(min_width.0, rows.len(), None, context);
 
         /* date column */
         self.data_columns.columns[1] =
-            CellBuffer::new_with_context(min_width.1, rows.len(), default_cell, context);
+            CellBuffer::new_with_context(min_width.1, rows.len(), None, context);
         /* from column */
         self.data_columns.columns[2] =
-            CellBuffer::new_with_context(min_width.2, rows.len(), default_cell, context);
+            CellBuffer::new_with_context(min_width.2, rows.len(), None, context);
         self.data_columns.segment_tree[2] = row_widths.2.into();
         /* flags column */
         self.data_columns.columns[3] =
-            CellBuffer::new_with_context(min_width.3, rows.len(), default_cell, context);
+            CellBuffer::new_with_context(min_width.3, rows.len(), None, context);
         /* subject column */
         self.data_columns.columns[4] =
-            CellBuffer::new_with_context(min_width.4, rows.len(), default_cell, context);
+            CellBuffer::new_with_context(min_width.4, rows.len(), None, context);
         self.data_columns.segment_tree[4] = row_widths.4.into();
 
         self.rows = rows;
@@ -442,7 +419,7 @@ impl MailListingTrait for CompactListing {
         if self.length == 0 && self.filter_term.is_empty() {
             let message: String = account[&self.cursor_pos.1].status();
             self.data_columns.columns[0] =
-                CellBuffer::new_with_context(message.len(), self.length + 1, default_cell, context);
+                CellBuffer::new_with_context(message.len(), self.length + 1, None, context);
             write_string_to_grid(
                 &message,
                 &mut self.data_columns.columns[0],
@@ -564,7 +541,7 @@ impl ListingTrait for CompactListing {
                     } else if self.new_cursor_pos.2 + rows * multiplier > self.length {
                         self.new_cursor_pos.2 = self.length - 1;
                     } else {
-                        self.new_cursor_pos.2 = (self.length / rows) * rows;
+                        self.new_cursor_pos.2 = (self.length.saturating_sub(1) / rows) * rows;
                     }
                 }
                 PageMovement::Right(_) | PageMovement::Left(_) => {}
@@ -603,7 +580,9 @@ impl ListingTrait for CompactListing {
                 self.highlight_line(grid, new_area, *idx, context);
                 context.dirty_areas.push_back(new_area);
             }
-            return;
+            if !self.force_draw {
+                return;
+            }
         } else if self.cursor_pos != self.new_cursor_pos {
             self.cursor_pos = self.new_cursor_pos;
         }
@@ -811,15 +790,8 @@ impl ListingTrait for CompactListing {
                     self.new_cursor_pos.2 =
                         std::cmp::min(self.filtered_selection.len() - 1, self.cursor_pos.2);
                 } else {
-                    let default_cell = {
-                        let mut ret = Cell::with_char(' ');
-                        ret.set_fg(self.color_cache.theme_default.fg)
-                            .set_bg(self.color_cache.theme_default.bg)
-                            .set_attrs(self.color_cache.theme_default.attrs);
-                        ret
-                    };
                     self.data_columns.columns[0] =
-                        CellBuffer::new_with_context(0, 0, default_cell, context);
+                        CellBuffer::new_with_context(0, 0, None, context);
                 }
                 self.redraw_threads_list(
                     context,
@@ -838,15 +810,8 @@ impl ListingTrait for CompactListing {
                     format!("Failed to search for term {}: {}", &self.filter_term, e),
                     ERROR,
                 );
-                let default_cell = {
-                    let mut ret = Cell::with_char(' ');
-                    ret.set_fg(self.color_cache.theme_default.fg)
-                        .set_bg(self.color_cache.theme_default.bg)
-                        .set_attrs(self.color_cache.theme_default.attrs);
-                    ret
-                };
                 self.data_columns.columns[0] =
-                    CellBuffer::new_with_context(message.len(), 1, default_cell, context);
+                    CellBuffer::new_with_context(message.len(), 1, None, context);
                 write_string_to_grid(
                     &message,
                     &mut self.data_columns.columns[0],
@@ -860,8 +825,20 @@ impl ListingTrait for CompactListing {
         }
     }
 
-    fn set_command_modifier(&mut self, is_active: bool) {
-        self.modifier_active = is_active;
+    fn unfocused(&self) -> bool {
+        self.unfocused
+    }
+
+    fn set_modifier_active(&mut self, new_val: bool) {
+        self.modifier_active = new_val;
+    }
+
+    fn set_modifier_command(&mut self, new_val: Option<Modifier>) {
+        self.modifier_command = new_val;
+    }
+
+    fn modifier_command(&self) -> Option<Modifier> {
+        self.modifier_command
     }
 
     fn set_movement(&mut self, mvm: PageMovement) {
@@ -918,9 +895,9 @@ impl CompactListing {
         let thread = threads.thread_ref(hash);
         let mut tags = String::new();
         let mut colors: SmallVec<[_; 8]> = SmallVec::new();
-        let backend_lck = context.accounts[&self.cursor_pos.0].backend.read().unwrap();
-        if let Some(t) = backend_lck.tags() {
-            let tags_lck = t.read().unwrap();
+        let account = &context.accounts[&self.cursor_pos.0];
+        if account.backend_capabilities.supports_tags {
+            let tags_lck = account.collection.tag_index.read().unwrap();
             for t in e.labels().iter() {
                 if mailbox_settings!(
                     context[self.cursor_pos.0][&self.cursor_pos.1]
@@ -1414,8 +1391,7 @@ impl Component for CompactListing {
             let (upper_left, bottom_right) = area;
             let rows = get_y(bottom_right) - get_y(upper_left) + 1;
 
-            if let Some('s') = self.modifier_command.take() {
-                self.set_command_modifier(false);
+            if let Some(modifier) = self.modifier_command.take() {
                 if let Some(mvm) = self.movement.as_ref() {
                     match mvm {
                         PageMovement::Up(amount) => {
@@ -1423,16 +1399,47 @@ impl Component for CompactListing {
                                 ..=self.new_cursor_pos.2
                             {
                                 let thread = self.get_thread_under_cursor(c);
-                                self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                match modifier {
+                                    Modifier::SymmetricDifference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                    }
+                                    Modifier::Union => {
+                                        self.selection.entry(thread).and_modify(|e| *e = true);
+                                    }
+                                    Modifier::Difference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = false);
+                                    }
+                                    Modifier::Intersection => {}
+                                }
                                 self.row_updates.push(thread);
                             }
+                            if modifier == Modifier::Intersection {
+                                for c in (0..self.new_cursor_pos.2.saturating_sub(*amount))
+                                    .chain((self.new_cursor_pos.2 + 2)..self.length)
+                                {
+                                    let thread = self.get_thread_under_cursor(c);
+                                    self.selection.entry(thread).and_modify(|e| *e = false);
+                                    self.row_updates.push(thread);
+                                }
+                            }
                         }
                         PageMovement::PageUp(multiplier) => {
                             for c in self.new_cursor_pos.2.saturating_sub(rows * multiplier)
                                 ..=self.new_cursor_pos.2
                             {
                                 let thread = self.get_thread_under_cursor(c);
-                                self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                match modifier {
+                                    Modifier::SymmetricDifference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                    }
+                                    Modifier::Union => {
+                                        self.selection.entry(thread).and_modify(|e| *e = true);
+                                    }
+                                    Modifier::Difference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = false);
+                                    }
+                                    Modifier::Intersection => {}
+                                }
                                 self.row_updates.push(thread);
                             }
                         }
@@ -1441,9 +1448,30 @@ impl Component for CompactListing {
                                 ..std::cmp::min(self.length, self.new_cursor_pos.2 + amount + 1)
                             {
                                 let thread = self.get_thread_under_cursor(c);
-                                self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                match modifier {
+                                    Modifier::SymmetricDifference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                    }
+                                    Modifier::Union => {
+                                        self.selection.entry(thread).and_modify(|e| *e = true);
+                                    }
+                                    Modifier::Difference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = false);
+                                    }
+                                    Modifier::Intersection => {}
+                                }
                                 self.row_updates.push(thread);
                             }
+                            if modifier == Modifier::Intersection {
+                                for c in (0..self.new_cursor_pos.2).chain(
+                                    (std::cmp::min(self.length, self.new_cursor_pos.2 + amount + 1)
+                                        + 1)..self.length,
+                                ) {
+                                    let thread = self.get_thread_under_cursor(c);
+                                    self.selection.entry(thread).and_modify(|e| *e = false);
+                                    self.row_updates.push(thread);
+                                }
+                            }
                         }
                         PageMovement::PageDown(multiplier) => {
                             for c in self.new_cursor_pos.2
@@ -1453,27 +1481,87 @@ impl Component for CompactListing {
                                 )
                             {
                                 let thread = self.get_thread_under_cursor(c);
-                                self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                match modifier {
+                                    Modifier::SymmetricDifference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                    }
+                                    Modifier::Union => {
+                                        self.selection.entry(thread).and_modify(|e| *e = true);
+                                    }
+                                    Modifier::Difference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = false);
+                                    }
+                                    Modifier::Intersection => {}
+                                }
                                 self.row_updates.push(thread);
                             }
+                            if modifier == Modifier::Intersection {
+                                for c in (0..self.new_cursor_pos.2).chain(
+                                    (std::cmp::min(
+                                        self.new_cursor_pos.2 + rows * multiplier + 1,
+                                        self.length,
+                                    ) + 1)..self.length,
+                                ) {
+                                    let thread = self.get_thread_under_cursor(c);
+                                    self.selection.entry(thread).and_modify(|e| *e = false);
+                                    self.row_updates.push(thread);
+                                }
+                            }
                         }
                         PageMovement::Right(_) | PageMovement::Left(_) => {}
                         PageMovement::Home => {
                             for c in 0..=self.new_cursor_pos.2 {
                                 let thread = self.get_thread_under_cursor(c);
-                                self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                match modifier {
+                                    Modifier::SymmetricDifference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                    }
+                                    Modifier::Union => {
+                                        self.selection.entry(thread).and_modify(|e| *e = true);
+                                    }
+                                    Modifier::Difference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = false);
+                                    }
+                                    Modifier::Intersection => {}
+                                }
                                 self.row_updates.push(thread);
                             }
+                            if modifier == Modifier::Intersection {
+                                for c in (self.new_cursor_pos.2 + 1)..self.length {
+                                    let thread = self.get_thread_under_cursor(c);
+                                    self.selection.entry(thread).and_modify(|e| *e = false);
+                                    self.row_updates.push(thread);
+                                }
+                            }
                         }
                         PageMovement::End => {
                             for c in self.new_cursor_pos.2..self.length {
                                 let thread = self.get_thread_under_cursor(c);
-                                self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                match modifier {
+                                    Modifier::SymmetricDifference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                    }
+                                    Modifier::Union => {
+                                        self.selection.entry(thread).and_modify(|e| *e = true);
+                                    }
+                                    Modifier::Difference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = false);
+                                    }
+                                    Modifier::Intersection => {}
+                                }
                                 self.row_updates.push(thread);
                             }
+                            if modifier == Modifier::Intersection {
+                                for c in 0..self.new_cursor_pos.2 {
+                                    let thread = self.get_thread_under_cursor(c);
+                                    self.selection.entry(thread).and_modify(|e| *e = false);
+                                    self.row_updates.push(thread);
+                                }
+                            }
                         }
                     }
                 }
+                self.force_draw = true;
             }
 
             if !self.row_updates.is_empty() {
@@ -1539,6 +1627,8 @@ impl Component for CompactListing {
                         ) =>
                 {
                     self.unfocused = false;
+                    self.view
+                        .process_event(&mut UIEvent::VisibilityChange(false), context);
                     self.dirty = true;
                     /* If self.row_updates is not empty and we exit a thread, the row_update events
                      * will be performed but the list will not be drawn. So force a draw in any case.
@@ -1548,12 +1638,10 @@ impl Component for CompactListing {
                 }
                 UIEvent::Input(ref key)
                     if !self.unfocused
-                        && shortcut!(
-                            key == shortcuts[CompactListing::DESCRIPTION]["select_entry"]
-                        ) =>
+                        && shortcut!(key == shortcuts[Listing::DESCRIPTION]["select_entry"]) =>
                 {
-                    if self.modifier_active {
-                        self.modifier_command = Some('s');
+                    if self.modifier_active && self.modifier_command.is_none() {
+                        self.modifier_command = Some(Modifier::default());
                     } else {
                         let thread_hash = self.get_thread_under_cursor(self.cursor_pos.2);
                         self.selection.entry(thread_hash).and_modify(|e| *e = !*e);
@@ -1580,7 +1668,7 @@ impl Component for CompactListing {
                             // FIXME: perform subsort.
                             return true;
                         }
-                        Action::ToggleThreadSnooze if !self.unfocused => {
+                        Action::Listing(ToggleThreadSnooze) if !self.unfocused => {
                             let thread = self.get_thread_under_cursor(self.cursor_pos.2);
                             let account = &mut context.accounts[&self.cursor_pos.0];
                             account
@@ -1605,6 +1693,43 @@ impl Component for CompactListing {
             }
         }
         match *event {
+            UIEvent::ConfigReload { old_settings: _ } => {
+                self.color_cache = ColorCache {
+                    even_unseen: crate::conf::value(context, "mail.listing.compact.even_unseen"),
+                    even_selected: crate::conf::value(
+                        context,
+                        "mail.listing.compact.even_selected",
+                    ),
+                    even_highlighted: crate::conf::value(
+                        context,
+                        "mail.listing.compact.even_highlighted",
+                    ),
+                    odd_unseen: crate::conf::value(context, "mail.listing.compact.odd_unseen"),
+                    odd_selected: crate::conf::value(context, "mail.listing.compact.odd_selected"),
+                    odd_highlighted: crate::conf::value(
+                        context,
+                        "mail.listing.compact.odd_highlighted",
+                    ),
+                    even: crate::conf::value(context, "mail.listing.compact.even"),
+                    odd: crate::conf::value(context, "mail.listing.compact.odd"),
+                    attachment_flag: crate::conf::value(context, "mail.listing.attachment_flag"),
+                    thread_snooze_flag: crate::conf::value(
+                        context,
+                        "mail.listing.thread_snooze_flag",
+                    ),
+                    tag_default: crate::conf::value(context, "mail.listing.tag_default"),
+                    theme_default: crate::conf::value(context, "theme_default"),
+                    ..self.color_cache
+                };
+                if !context.settings.terminal.use_color() {
+                    self.color_cache.highlighted.attrs |= Attr::REVERSE;
+                    self.color_cache.tag_default.attrs |= Attr::REVERSE;
+                    self.color_cache.even_highlighted.attrs |= Attr::REVERSE;
+                    self.color_cache.odd_highlighted.attrs |= Attr::REVERSE;
+                }
+                self.refresh_mailbox(context, true);
+                self.set_dirty(true);
+            }
             UIEvent::MailboxUpdate((ref idxa, ref idxf))
                 if (*idxa, *idxf) == (self.new_cursor_pos.0, self.cursor_pos.1) =>
             {
@@ -1698,13 +1823,10 @@ impl Component for CompactListing {
                     self.cursor_pos.1,
                 ) {
                     Ok(job) => {
-                        let (chan, handle, job_id) = context.accounts[&self.cursor_pos.0]
+                        let handle = context.accounts[&self.cursor_pos.0]
                             .job_executor
                             .spawn_specialized(job);
-                        context.accounts[&self.cursor_pos.0]
-                            .active_jobs
-                            .insert(job_id, crate::conf::accounts::JobRequest::Search(handle));
-                        self.search_job = Some((filter_term.to_string(), chan, job_id));
+                        self.search_job = Some((filter_term.to_string(), handle));
                     }
                     Err(err) => {
                         context.replies.push_back(UIEvent::Notification(
@@ -1723,16 +1845,13 @@ impl Component for CompactListing {
                     self.cursor_pos.1,
                 ) {
                     Ok(job) => {
-                        let (mut chan, handle, job_id) = context.accounts[&self.cursor_pos.0]
+                        let mut handle = context.accounts[&self.cursor_pos.0]
                             .job_executor
                             .spawn_specialized(job);
-                        if let Ok(Some(search_result)) = try_recv_timeout!(&mut chan) {
+                        if let Ok(Some(search_result)) = try_recv_timeout!(&mut handle.chan) {
                             self.select(search_term, search_result, context);
                         } else {
-                            context.accounts[&self.cursor_pos.0]
-                                .active_jobs
-                                .insert(job_id, crate::conf::accounts::JobRequest::Search(handle));
-                            self.select_job = Some((search_term.to_string(), chan, job_id));
+                            self.select_job = Some((search_term.to_string(), handle));
                         }
                     }
                     Err(err) => {
@@ -1749,24 +1868,30 @@ impl Component for CompactListing {
                 if self
                     .search_job
                     .as_ref()
-                    .map(|(_, _, j)| j == job_id)
+                    .map(|(_, j)| j == job_id)
                     .unwrap_or(false) =>
             {
-                let (filter_term, mut rcvr, _job_id) = self.search_job.take().unwrap();
-                let results = rcvr.try_recv().unwrap().unwrap();
-                self.filter(filter_term, results, context);
+                let (filter_term, mut handle) = self.search_job.take().unwrap();
+                match handle.chan.try_recv() {
+                    Err(_) => { /* search was canceled */ }
+                    Ok(None) => { /* something happened, perhaps a worker thread panicked */ }
+                    Ok(Some(results)) => self.filter(filter_term, results, context),
+                }
                 self.set_dirty(true);
             }
             UIEvent::StatusEvent(StatusEvent::JobFinished(ref job_id))
                 if self
                     .select_job
                     .as_ref()
-                    .map(|(_, _, j)| j == job_id)
+                    .map(|(_, j)| j == job_id)
                     .unwrap_or(false) =>
             {
-                let (search_term, mut rcvr, _job_id) = self.select_job.take().unwrap();
-                let results = rcvr.try_recv().unwrap().unwrap();
-                self.select(&search_term, results, context);
+                let (search_term, mut handle) = self.select_job.take().unwrap();
+                match handle.chan.try_recv() {
+                    Err(_) => { /* search was canceled */ }
+                    Ok(None) => { /* something happened, perhaps a worker thread panicked */ }
+                    Ok(Some(results)) => self.select(&search_term, results, context),
+                }
                 self.set_dirty(true);
             }
             _ => {}
@@ -1797,6 +1922,8 @@ impl Component for CompactListing {
 
         let config_map = context.settings.shortcuts.compact_listing.key_values();
         map.insert(CompactListing::DESCRIPTION, config_map);
+        let config_map = context.settings.shortcuts.listing.key_values();
+        map.insert(Listing::DESCRIPTION, config_map);
 
         map
     }

src/components/mail/listing/conversations.rs

@@ -20,8 +20,8 @@
  */
 
 use super::*;
-use crate::components::utilities::PageMovement;
-use crate::jobs::{oneshot, JobId};
+use crate::components::PageMovement;
+use crate::jobs::JoinHandle;
 use std::iter::FromIterator;
 
 macro_rules! row_attr {
@@ -104,11 +104,7 @@ pub struct ConversationsListing {
     /// Cache current view.
     content: CellBuffer,
 
-    search_job: Option<(
-        String,
-        oneshot::Receiver<Result<SmallVec<[EnvelopeHash; 512]>>>,
-        JobId,
-    )>,
+    search_job: Option<(String, JoinHandle<Result<SmallVec<[EnvelopeHash; 512]>>>)>,
     filter_term: String,
     filtered_selection: Vec<ThreadHash>,
     filtered_order: HashMap<ThreadHash, usize>,
@@ -124,7 +120,7 @@ pub struct ConversationsListing {
 
     movement: Option<PageMovement>,
     modifier_active: bool,
-    modifier_command: Option<char>,
+    modifier_command: Option<Modifier>,
     id: ComponentId,
 }
 
@@ -199,17 +195,9 @@ impl MailListingTrait for ConversationsListing {
         match context.accounts[&self.cursor_pos.0].load(self.cursor_pos.1) {
             Ok(()) => {}
             Err(_) => {
-                let default_cell = {
-                    let mut ret = Cell::with_char(' ');
-                    ret.set_fg(self.color_cache.theme_default.fg)
-                        .set_bg(self.color_cache.theme_default.bg)
-                        .set_attrs(self.color_cache.theme_default.attrs);
-                    ret
-                };
                 let message: String =
                     context.accounts[&self.cursor_pos.0][&self.cursor_pos.1].status();
-                self.content =
-                    CellBuffer::new_with_context(message.len(), 1, default_cell, context);
+                self.content = CellBuffer::new_with_context(message.len(), 1, None, context);
                 self.length = 0;
                 write_string_to_grid(
                     message.as_str(),
@@ -299,17 +287,20 @@ impl MailListingTrait for ConversationsListing {
             }
             from_address_list.clear();
             from_address_set.clear();
-            for (_, h) in threads.thread_group_iter(thread) {
-                let env_hash = threads.thread_nodes()[&h].message().unwrap();
-
-                let envelope: &EnvelopeRef = &context.accounts[&self.cursor_pos.0]
-                    .collection
-                    .get_env(env_hash);
+            for envelope in threads
+                .thread_group_iter(thread)
+                .filter_map(|(_, h)| threads.thread_nodes()[&h].message())
+                .map(|env_hash| {
+                    context.accounts[&self.cursor_pos.0]
+                        .collection
+                        .get_env(env_hash)
+                })
+            {
                 for addr in envelope.from().iter() {
-                    if from_address_set.contains(addr.raw()) {
+                    if from_address_set.contains(addr.address_spec_raw()) {
                         continue;
                     }
-                    from_address_set.insert(addr.raw().to_vec());
+                    from_address_set.insert(addr.address_spec_raw().to_vec());
                     from_address_list.push(addr.clone());
                 }
             }
@@ -357,8 +348,7 @@ impl MailListingTrait for ConversationsListing {
         }
 
         let width = max_entry_columns;
-        self.content =
-            CellBuffer::new_with_context(width, 4 * rows.len(), Cell::with_char(' '), context);
+        self.content = CellBuffer::new_with_context(width, 4 * rows.len(), None, context);
 
         let padding_fg = self.color_cache.padding.fg;
 
@@ -488,15 +478,8 @@ impl MailListingTrait for ConversationsListing {
             }
         }
         if self.length == 0 && self.filter_term.is_empty() {
-            let default_cell = {
-                let mut ret = Cell::with_char(' ');
-                ret.set_fg(self.color_cache.theme_default.fg)
-                    .set_bg(self.color_cache.theme_default.bg)
-                    .set_attrs(self.color_cache.theme_default.attrs);
-                ret
-            };
             let message: String = account[&self.cursor_pos.1].status();
-            self.content = CellBuffer::new_with_context(message.len(), 1, default_cell, context);
+            self.content = CellBuffer::new_with_context(message.len(), 1, None, context);
             write_string_to_grid(
                 &message,
                 &mut self.content,
@@ -642,7 +625,7 @@ impl ListingTrait for ConversationsListing {
                     } else if self.new_cursor_pos.2 + rows * multiplier > self.length {
                         self.new_cursor_pos.2 = self.length - 1;
                     } else {
-                        self.new_cursor_pos.2 = (self.length / rows) * rows;
+                        self.new_cursor_pos.2 = (self.length.saturating_sub(1) / rows) * rows;
                     }
                 }
                 PageMovement::Right(_) | PageMovement::Left(_) => {}
@@ -650,7 +633,7 @@ impl ListingTrait for ConversationsListing {
                     self.new_cursor_pos.2 = 0;
                 }
                 PageMovement::End => {
-                    self.new_cursor_pos.2 = self.length - 1;
+                    self.new_cursor_pos.2 = self.length.saturating_sub(1);
                 }
             }
         }
@@ -824,14 +807,7 @@ impl ListingTrait for ConversationsListing {
                     self.new_cursor_pos.2 =
                         std::cmp::min(self.filtered_selection.len() - 1, self.cursor_pos.2);
                 } else {
-                    let default_cell = {
-                        let mut ret = Cell::with_char(' ');
-                        ret.set_fg(self.color_cache.theme_default.fg)
-                            .set_bg(self.color_cache.theme_default.bg)
-                            .set_attrs(self.color_cache.theme_default.attrs);
-                        ret
-                    };
-                    self.content = CellBuffer::new_with_context(0, 0, default_cell, context);
+                    self.content = CellBuffer::new_with_context(0, 0, None, context);
                 }
                 self.redraw_threads_list(
                     context,
@@ -850,15 +826,7 @@ impl ListingTrait for ConversationsListing {
                     format!("Failed to search for term {}: {}", self.filter_term, e),
                     ERROR,
                 );
-                let default_cell = {
-                    let mut ret = Cell::with_char(' ');
-                    ret.set_fg(self.color_cache.theme_default.fg)
-                        .set_bg(self.color_cache.theme_default.bg)
-                        .set_attrs(self.color_cache.theme_default.attrs);
-                    ret
-                };
-                self.content =
-                    CellBuffer::new_with_context(message.len(), 1, default_cell, context);
+                self.content = CellBuffer::new_with_context(message.len(), 1, None, context);
                 write_string_to_grid(
                     &message,
                     &mut self.content,
@@ -872,8 +840,20 @@ impl ListingTrait for ConversationsListing {
         }
     }
 
-    fn set_command_modifier(&mut self, is_active: bool) {
-        self.modifier_active = is_active;
+    fn unfocused(&self) -> bool {
+        self.unfocused
+    }
+
+    fn set_modifier_active(&mut self, new_val: bool) {
+        self.modifier_active = new_val;
+    }
+
+    fn set_modifier_command(&mut self, new_val: Option<Modifier>) {
+        self.modifier_command = new_val;
+    }
+
+    fn modifier_command(&self) -> Option<Modifier> {
+        self.modifier_command
     }
 
     fn set_movement(&mut self, mvm: PageMovement) {
@@ -928,9 +908,9 @@ impl ConversationsListing {
         let thread = threads.thread_ref(hash);
         let mut tags = String::new();
         let mut colors = SmallVec::new();
-        let backend_lck = context.accounts[&self.cursor_pos.0].backend.read().unwrap();
-        if let Some(t) = backend_lck.tags() {
-            let tags_lck = t.read().unwrap();
+        let account = &context.accounts[&self.cursor_pos.0];
+        if account.backend_capabilities.supports_tags {
+            let tags_lck = account.collection.tag_index.read().unwrap();
             for t in e.labels().iter() {
                 if mailbox_settings!(
                     context[self.cursor_pos.0][&self.cursor_pos.1]
@@ -1022,6 +1002,7 @@ impl ConversationsListing {
                     .as_ref()
                     .map(String::as_str)
                     .or(Some("%Y-%m-%d %T")),
+                false,
             ),
         }
     }
@@ -1068,17 +1049,20 @@ impl ConversationsListing {
         let mut from_address_list = Vec::new();
         let mut from_address_set: std::collections::HashSet<Vec<u8>> =
             std::collections::HashSet::new();
-        for (_, h) in threads.thread_group_iter(thread_hash) {
-            let env_hash = threads.thread_nodes()[&h].message().unwrap();
-
-            let envelope: &EnvelopeRef = &context.accounts[&self.cursor_pos.0]
-                .collection
-                .get_env(env_hash);
+        for envelope in threads
+            .thread_group_iter(thread_hash)
+            .filter_map(|(_, h)| threads.thread_nodes()[&h].message())
+            .map(|env_hash| {
+                context.accounts[&self.cursor_pos.0]
+                    .collection
+                    .get_env(env_hash)
+            })
+        {
             for addr in envelope.from().iter() {
-                if from_address_set.contains(addr.raw()) {
+                if from_address_set.contains(addr.address_spec_raw()) {
                     continue;
                 }
-                from_address_set.insert(addr.raw().to_vec());
+                from_address_set.insert(addr.address_spec_raw().to_vec());
                 from_address_list.push(addr.clone());
             }
         }
@@ -1264,8 +1248,7 @@ impl Component for ConversationsListing {
             }
             let (upper_left, bottom_right) = area;
             let rows = (get_y(bottom_right) - get_y(upper_left) + 1) / 3;
-            if let Some('s') = self.modifier_command.take() {
-                self.set_command_modifier(false);
+            if let Some(modifier) = self.modifier_command.take() {
                 if let Some(mvm) = self.movement.as_ref() {
                     match mvm {
                         PageMovement::Up(amount) => {
@@ -1273,16 +1256,47 @@ impl Component for ConversationsListing {
                                 ..=self.new_cursor_pos.2
                             {
                                 let thread = self.get_thread_under_cursor(c);
-                                self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                match modifier {
+                                    Modifier::SymmetricDifference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                    }
+                                    Modifier::Union => {
+                                        self.selection.entry(thread).and_modify(|e| *e = true);
+                                    }
+                                    Modifier::Difference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = false);
+                                    }
+                                    Modifier::Intersection => {}
+                                }
                                 self.row_updates.push(thread);
                             }
+                            if modifier == Modifier::Intersection {
+                                for c in (0..self.new_cursor_pos.2.saturating_sub(*amount))
+                                    .chain((self.new_cursor_pos.2 + 2)..self.length)
+                                {
+                                    let thread = self.get_thread_under_cursor(c);
+                                    self.selection.entry(thread).and_modify(|e| *e = false);
+                                    self.row_updates.push(thread);
+                                }
+                            }
                         }
                         PageMovement::PageUp(multiplier) => {
                             for c in self.new_cursor_pos.2.saturating_sub(rows * multiplier)
                                 ..=self.new_cursor_pos.2
                             {
                                 let thread = self.get_thread_under_cursor(c);
-                                self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                match modifier {
+                                    Modifier::SymmetricDifference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                    }
+                                    Modifier::Union => {
+                                        self.selection.entry(thread).and_modify(|e| *e = true);
+                                    }
+                                    Modifier::Difference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = false);
+                                    }
+                                    Modifier::Intersection => {}
+                                }
                                 self.row_updates.push(thread);
                             }
                         }
@@ -1291,9 +1305,30 @@ impl Component for ConversationsListing {
                                 ..std::cmp::min(self.length, self.new_cursor_pos.2 + amount + 1)
                             {
                                 let thread = self.get_thread_under_cursor(c);
-                                self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                match modifier {
+                                    Modifier::SymmetricDifference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                    }
+                                    Modifier::Union => {
+                                        self.selection.entry(thread).and_modify(|e| *e = true);
+                                    }
+                                    Modifier::Difference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = false);
+                                    }
+                                    Modifier::Intersection => {}
+                                }
                                 self.row_updates.push(thread);
                             }
+                            if modifier == Modifier::Intersection {
+                                for c in (0..self.new_cursor_pos.2).chain(
+                                    (std::cmp::min(self.length, self.new_cursor_pos.2 + amount + 1)
+                                        + 1)..self.length,
+                                ) {
+                                    let thread = self.get_thread_under_cursor(c);
+                                    self.selection.entry(thread).and_modify(|e| *e = false);
+                                    self.row_updates.push(thread);
+                                }
+                            }
                         }
                         PageMovement::PageDown(multiplier) => {
                             for c in self.new_cursor_pos.2
@@ -1303,24 +1338,83 @@ impl Component for ConversationsListing {
                                 )
                             {
                                 let thread = self.get_thread_under_cursor(c);
-                                self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                match modifier {
+                                    Modifier::SymmetricDifference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                    }
+                                    Modifier::Union => {
+                                        self.selection.entry(thread).and_modify(|e| *e = true);
+                                    }
+                                    Modifier::Difference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = false);
+                                    }
+                                    Modifier::Intersection => {}
+                                }
                                 self.row_updates.push(thread);
                             }
+                            if modifier == Modifier::Intersection {
+                                for c in (0..self.new_cursor_pos.2).chain(
+                                    (std::cmp::min(
+                                        self.new_cursor_pos.2 + rows * multiplier + 1,
+                                        self.length,
+                                    ) + 1)..self.length,
+                                ) {
+                                    let thread = self.get_thread_under_cursor(c);
+                                    self.selection.entry(thread).and_modify(|e| *e = false);
+                                    self.row_updates.push(thread);
+                                }
+                            }
                         }
                         PageMovement::Right(_) | PageMovement::Left(_) => {}
                         PageMovement::Home => {
                             for c in 0..=self.new_cursor_pos.2 {
                                 let thread = self.get_thread_under_cursor(c);
-                                self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                match modifier {
+                                    Modifier::SymmetricDifference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                    }
+                                    Modifier::Union => {
+                                        self.selection.entry(thread).and_modify(|e| *e = true);
+                                    }
+                                    Modifier::Difference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = false);
+                                    }
+                                    Modifier::Intersection => {}
+                                }
                                 self.row_updates.push(thread);
                             }
+                            if modifier == Modifier::Intersection {
+                                for c in (self.new_cursor_pos.2 + 1)..self.length {
+                                    let thread = self.get_thread_under_cursor(c);
+                                    self.selection.entry(thread).and_modify(|e| *e = false);
+                                    self.row_updates.push(thread);
+                                }
+                            }
                         }
                         PageMovement::End => {
                             for c in self.new_cursor_pos.2..self.length {
                                 let thread = self.get_thread_under_cursor(c);
-                                self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                match modifier {
+                                    Modifier::SymmetricDifference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = !*e);
+                                    }
+                                    Modifier::Union => {
+                                        self.selection.entry(thread).and_modify(|e| *e = true);
+                                    }
+                                    Modifier::Difference => {
+                                        self.selection.entry(thread).and_modify(|e| *e = false);
+                                    }
+                                    Modifier::Intersection => {}
+                                }
                                 self.row_updates.push(thread);
                             }
+                            if modifier == Modifier::Intersection {
+                                for c in 0..self.new_cursor_pos.2 {
+                                    let thread = self.get_thread_under_cursor(c);
+                                    self.selection.entry(thread).and_modify(|e| *e = false);
+                                    self.row_updates.push(thread);
+                                }
+                            }
                         }
                     }
                 }
@@ -1397,6 +1491,8 @@ impl Component for ConversationsListing {
                         ) =>
                 {
                     self.unfocused = false;
+                    self.view
+                        .process_event(&mut UIEvent::VisibilityChange(false), context);
                     self.dirty = true;
                     /* If self.row_updates is not empty and we exit a thread, the row_update events
                      * will be performed but the list will not be drawn. So force a draw in any case.
@@ -1406,12 +1502,10 @@ impl Component for ConversationsListing {
                 }
                 UIEvent::Input(ref key)
                     if !self.unfocused
-                        && shortcut!(
-                            key == shortcuts[ConversationsListing::DESCRIPTION]["select_entry"]
-                        ) =>
+                        && shortcut!(key == shortcuts[Listing::DESCRIPTION]["select_entry"]) =>
                 {
-                    if self.modifier_active {
-                        self.modifier_command = Some('s');
+                    if self.modifier_active && self.modifier_command.is_none() {
+                        self.modifier_command = Some(Modifier::default());
                     } else {
                         let thread_hash = self.get_thread_under_cursor(self.cursor_pos.2);
                         self.selection.entry(thread_hash).and_modify(|e| *e = !*e);
@@ -1508,7 +1602,7 @@ impl Component for ConversationsListing {
                             */
                         return true;
                     }
-                    Action::ToggleThreadSnooze if !self.unfocused => {
+                    Action::Listing(ToggleThreadSnooze) if !self.unfocused => {
                         let thread = self.get_thread_under_cursor(self.cursor_pos.2);
                         let account = &mut context.accounts[&self.cursor_pos.0];
                         account
@@ -1531,6 +1625,34 @@ impl Component for ConversationsListing {
             }
         }
         match *event {
+            UIEvent::ConfigReload { old_settings: _ } => {
+                self.color_cache = ColorCache {
+                    theme_default: crate::conf::value(context, "mail.listing.conversations"),
+                    subject: crate::conf::value(context, "mail.listing.conversations.subject"),
+                    from: crate::conf::value(context, "mail.listing.conversations.from"),
+                    date: crate::conf::value(context, "mail.listing.conversations.date"),
+                    selected: crate::conf::value(context, "mail.listing.conversations.selected"),
+                    unseen: crate::conf::value(context, "mail.listing.conversations.unseen"),
+                    highlighted: crate::conf::value(
+                        context,
+                        "mail.listing.conversations.highlighted",
+                    ),
+                    attachment_flag: crate::conf::value(context, "mail.listing.attachment_flag"),
+                    thread_snooze_flag: crate::conf::value(
+                        context,
+                        "mail.listing.thread_snooze_flag",
+                    ),
+                    tag_default: crate::conf::value(context, "mail.listing.tag_default"),
+                    ..self.color_cache
+                };
+
+                if !context.settings.terminal.use_color() {
+                    self.color_cache.highlighted.attrs |= Attr::REVERSE;
+                    self.color_cache.tag_default.attrs |= Attr::REVERSE;
+                }
+                self.refresh_mailbox(context, true);
+                self.set_dirty(true);
+            }
             UIEvent::MailboxUpdate((ref idxa, ref idxf))
                 if (*idxa, *idxf) == (self.new_cursor_pos.0, self.cursor_pos.1) =>
             {
@@ -1555,13 +1677,10 @@ impl Component for ConversationsListing {
                         self.cursor_pos.1,
                     ) {
                         Ok(job) => {
-                            let (chan, handle, job_id) = context.accounts[&self.cursor_pos.0]
+                            let handle = context.accounts[&self.cursor_pos.0]
                                 .job_executor
                                 .spawn_specialized(job);
-                            context.accounts[&self.cursor_pos.0]
-                                .active_jobs
-                                .insert(job_id, crate::conf::accounts::JobRequest::Search(handle));
-                            self.search_job = Some((filter_term.to_string(), chan, job_id));
+                            self.search_job = Some((filter_term.to_string(), handle));
                         }
                         Err(err) => {
                             context.replies.push_back(UIEvent::Notification(
@@ -1601,12 +1720,15 @@ impl Component for ConversationsListing {
                 if self
                     .search_job
                     .as_ref()
-                    .map(|(_, _, j)| j == job_id)
+                    .map(|(_, j)| j == job_id)
                     .unwrap_or(false) =>
             {
-                let (filter_term, mut rcvr, _job_id) = self.search_job.take().unwrap();
-                let results = rcvr.try_recv().unwrap().unwrap();
-                self.filter(filter_term, results, context);
+                let (filter_term, mut handle) = self.search_job.take().unwrap();
+                match handle.chan.try_recv() {
+                    Err(_) => { /* search was canceled */ }
+                    Ok(None) => { /* something happened, perhaps a worker thread panicked */ }
+                    Ok(Some(results)) => self.filter(filter_term, results, context),
+                }
                 self.set_dirty(true);
             }
             _ => {}
@@ -1638,6 +1760,8 @@ impl Component for ConversationsListing {
 
         let config_map = context.settings.shortcuts.compact_listing.key_values();
         map.insert(ConversationsListing::DESCRIPTION, config_map);
+        let config_map = context.settings.shortcuts.listing.key_values();
+        map.insert(Listing::DESCRIPTION, config_map);
 
         map
     }

src/components/mail/listing/offline.rs

@@ -20,7 +20,7 @@
  */
 
 use super::*;
-use crate::components::utilities::PageMovement;
+use crate::components::PageMovement;
 
 #[derive(Debug)]
 pub struct OfflineListing {
@@ -79,6 +79,10 @@ impl ListingTrait for OfflineListing {
 
     fn draw_list(&mut self, _: &mut CellBuffer, _: Area, _: &mut Context) {}
 
+    fn unfocused(&self) -> bool {
+        false
+    }
+
     fn set_movement(&mut self, _: PageMovement) {}
 }