Cargo.lock: Update lexical-core version

Fixes compilation on macos 10.15.3, rustc 1.53.0

.gdbinit

@@ -1,42 +1,5 @@
 set language rust
 source ~/.gdbinit
 break rust_panic
-break core::option::expect_failed::h4927e1fef06c4878
-break core::panicking::panic
-break libcore/panicking.rs:58
-break libcore/result.rs:945
 set auto-load python-scripts
-break melib/src/mailbox/thread.rs:1010
 set print thread-events off
-
-#python
-#import os
-#import sys
-#
-#sys.path.insert(0, os.getcwd() + '/scripts/gdb_meli/')
-#import gdb
-#import gdb_meli
-#
-#print(gdb_meli.__file__)
-#
-#help(gdb_meli)
-##from gdb_meli import build_pretty_printer
-##print(gdb.objfiles()[0].filename)
-##gdb_meli.register_pretty_printer(gdb)
-##gdb.printing.register_pretty_printer(
-##    gdb.current_objfile(),
-##    gdb_meli.build_pretty_printer())
-#end
-python
-
-import sys, os
-
-sys.path.insert(0, os.getcwd() + '/scripts/gdb_meli/')
-
-
-import gdb_meli, gdb
-#gdb_meli.register_meli_printers(gdb)
-#gdb.printing.register_pretty_printer(
-#	 gdb.current_objfile(),
-#	 gdb_meli.build_meli_printer())
-end

.gitignore

@@ -6,3 +6,12 @@ target/
 **/*.rs.bk
 .gdb_history
 *.log
+
+__pycache__/
+*.py[cod]
+
+debian/.debhelper/
+debian/debhelper-build-stamp
+debian/files
+debian/meli.substvars
+debian/meli/

CHANGELOG.md

@@ -0,0 +1,111 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- Add import command to import email from files into accounts
+- Add add-attachment-file-picker command and `file_picker_command` setting to
+  use external commands to choose files when composing new mail
+
+## [alpha-0.6.2] - 2020-09-24
+
+### Added
+- Add customizable mailbox tree in sidebar
+- Make `dbus` dependency opt-out (feature is `dbus-notifications`)
+- Implemented JMAP async, search, tagging, syncing
+- Preserve account order from configuration file
+- Implemented IMAP `CONDSTORE` support for IMAP cache
+- Add `timeout` setting for IMAP
+- Implement TCP keepalive for IMAP
+- Rewrote email address parsers.
+- Implement `copy_messages` for maildir
+- Implement selection with motions
+
+### Fixed
+- Fixed various problems with IMAP cache
+- Fixed various problems with IMAP message counts
+- Fixed various problems with IMAP connection hanging
+- Fixed IMAP not reconnecting on dropped IDLE connections
+- Fixed various problems with notmuch backend
+
+## [alpha-0.6.1] - 2020-08-02
+
+### Added
+
+- added experimental NNTP backend
+- added server extension support and use in account status tab
+
+### Fixed
+
+- imap: fixed IDLE connection getting stuck when using DEFLATE
+
+## [alpha-0.6.0] - 2020-07-29
+
+### Added
+
+- Add `select` command to select threads that match search query
+- Add support for mass copying/deleting/flagging/moving of messages
+- IMAP: add support for COMPRESS=DEFLATE and others
+  Extension use can be configured with individual flags such as `use_deflate`
+- Rename EXECUTE mode to COMMAND
+- add async IMAP backend
+- add in-app SMTP support
+- ui: Show decoded source by default when viewing an Envelope's source
+- ui: Add search in pagers
+- Add managesieve REPL binary for managesieve script management
+- imap: `add server_password_command`
+- configuration: Add per-folder and per-account configuration overrides.
+  e.g. `accounts."imap.domain.tld".mailboxes."INBOX".index_style = "plain"`
+
+  The selection is done for a specific field as follows:
+
+  ```text
+  if per-folder override is defined, return per-folder override
+    else if per-account override is defined, return per-account override
+      else return global setting field value.
+  ```
+- themes: Add Italics, Blink, Dim and Hidden text attributes
+- ui: recognize readline shortcuts in Execute mode
+- ui: hopefully smarter auto-completion in Execute mode
+- demo NNTP python plugin
+- ui: add `auto_choose_multipart_alternative`: Choose `text/html` alternative if `text/plain` is empty in `multipart/alternative` attachments.
+- ui: custom date format strings
+- ui: manual refresh for mailbox view
+- ui: create mailbox command
+- fs autocomplete
+- ui: add support for [`NO_COLOR`](https://no-color.org/)
+- enhanced, portable Makefile
+- added Debian packaging
+- added `default_header_values`: default header values used when creating a new draft
+- ui: switch between sidebar and mailbox view with {left,right} keys for more intuitive navigation
+- ui: add optional filter query for each mailbox view to view only the matching subset of messages (for example, you can hide all seen envelopes with `filter = "not flags:seen"`
+
+### Changed
+
+- Replace any use of 'folder' with 'mailbox' in user configuration
+- Load libnotmuch dynamically
+- Launch all user shell commands with `sh -c "..."`
+
+### Fixed
+
+- notmuch: add support for multiple accounts on same notmuch db
+
+## [alpha-0.5.1] - 2020-02-09
+
+### Added
+
+- Added in-terminal floating notifications with history
+- Added mailbox creation/deletion commands in IMAP accounts
+- Added cli-docs compile time feature: Optionally build manpages to text with mandoc and print them from the command line.
+- Added new theme keys
+
+[unreleased]: #
+[alpha-0.5.1]: https://github.com/meli/meli/releases/tag/alpha-0.5.1
+[alpha-0.6.0]: https://github.com/meli/meli/releases/tag/alpha-0.6.0
+[alpha-0.6.1]: https://github.com/meli/meli/releases/tag/alpha-0.6.1
+[alpha-0.6.2]: https://github.com/meli/meli/releases/tag/alpha-0.6.2

Cargo.toml

@@ -1,31 +1,90 @@
 [package]
 name = "meli"
-version = "0.3.2"
+version = "0.6.2"
 authors = ["Manos Pitsidianakis <el13635@mail.ntua.gr>"]
 edition = "2018"
 
+license = "GPL-3.0-or-later"
+readme = "README.md"
+description = "terminal mail client"
+homepage = "https://meli.delivery"
+repository = "https://git.meli.delivery/meli/meli.git"
+keywords = ["mail", "mua", "maildir", "terminal", "imap"]
+categories = ["command-line-utilities", "email"]
+default-run = "meli"
+
 [[bin]]
 name = "meli"
 path = "src/bin.rs"
 
+#[[bin]]
+#name = "managesieve-meli"
+#path = "src/managesieve.rs"
+
+#[[bin]]
+#name = "async"
+#path = "src/async.rs"
+
 [dependencies]
 xdg = "2.1.0"
 crossbeam = "0.7.2"
-signal-hook = "0.1.10"
-nix = "*"
-melib = { path = "melib", version = "*" }
-ui = { path = "ui", version = "*" }
+signal-hook = "0.1.12"
+signal-hook-registry = "1.2.0"
+nix = "0.17.0"
+melib = { path = "melib", version = "0.6.2" }
+
+serde = "1.0.71"
+serde_derive = "1.0.71"
+serde_json = "1.0"
+toml = { version = "0.5.6", features = ["preserve_order", ] }
+indexmap = { version = "^1.6", features = ["serde-1", ] }
+linkify = "0.4.0"
+notify = "4.0.1" # >:c
+termion = "1.5.1"
+bincode = "^1.3.0"
+uuid = { version = "0.8.1", features = ["serde", "v4"] }
+unicode-segmentation = "1.2.1" # >:c
+libc = {version = "0.2.59", features = ["extra_traits",]}
+smallvec = { version = "^1.5.0", features = ["serde", ] }
+bitflags = "1.0"
+pcre2 = { version = "0.2.3", optional = true }
+structopt = { version = "0.3.14", default-features = false }
+svg_crate = { version = "0.8.0", optional = true, package = "svg" }
+futures = "0.3.5"
+async-task = "3.0.0"
+num_cpus = "1.12.0"
+flate2 = { version = "1.0.16", optional = true }
+ 
+[target.'cfg(target_os="linux")'.dependencies]
+notify-rust = { version = "^4", optional = true }
+
+[build-dependencies]
+syn = { version = "1.0.31", features = [] }
+quote = "^1.0"
+proc-macro2 = "1.0.18"
+flate2 = { version = "1.0.16", optional = true }
 
 [profile.release]
-lto = true
+lto = "fat"
+codegen-units = 1
+opt-level = "s"
 debug = false
 
 [workspace]
-members = ["melib", "ui", "debug_printer", "testing", "text_processing"]
+members = ["melib", "tools", ]
 
 [features]
-default = []
+default = ["sqlite3", "notmuch", "regexp", "smtp", "dbus-notifications", "gpgme"]
+notmuch = ["melib/notmuch_backend", ]
+jmap = ["melib/jmap_backend",]
+sqlite3 = ["melib/sqlite3"]
+smtp = ["melib/smtp"]
+regexp = ["pcre2"]
+dbus-notifications = ["notify-rust",]
+cli-docs = ["flate2"]
+svgscreenshot = ["svg_crate"]
+gpgme = ["melib/gpgme"]
 
 # Print tracing logs as meli runs in stderr
 # enable for debug tracing logs: build with --features=debug-tracing
-debug-tracing = ["melib/debug-tracing", "ui/debug-tracing"]
+debug-tracing = ["melib/debug-tracing", ]

Makefile

@@ -1,24 +1,171 @@
+# meli - Makefile
+#
+# Copyright 2017-2020 Manos Pitsidianakis
+#
+# This file is part of meli.
+#
+# meli is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# meli is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with meli. If not, see <http://www.gnu.org/licenses/>.
+
+# Options
+PREFIX ?= /usr/local
+EXPANDED_PREFIX := `cd ${PREFIX} && pwd -P`
+BINDIR ?= ${EXPANDED_PREFIX}/bin
+MANDIR ?= ${EXPANDED_PREFIX}/share/man
+
+CARGO_TARGET_DIR ?= target
+MIN_RUSTC ?= 1.39.0
+CARGO_BIN ?= cargo
+CARGO_ARGS ?=
+
+# Installation parameters
+DOCS_SUBDIR ?= docs/
+MANPAGES ?= meli.1 meli.conf.5 meli-themes.5
+FEATURES ?= --features "${MELI_FEATURES}"
+
+MANPATHS != ACCUM="";for m in `manpath 2> /dev/null | tr ':' ' '`; do if [ -d "$${m}" ]; then REAL_PATH=`cd $${m} && pwd` ACCUM="$${ACCUM}:$${REAL_PATH}";fi;done;echo -n $${ACCUM} | sed 's/^://'
+VERSION != sed -n "s/^version\s*=\s*\"\(.*\)\"/\1/p" Cargo.toml
+GIT_COMMIT != git show-ref -s --abbrev HEAD
+DATE != date -I
+
+# Output parameters
+BOLD ?= `[ -z $${TERM} ] && echo "" || tput bold`
+UNDERLINE ?= `[ -z $${TERM} ] && echo "" || tput smul`
+ANSI_RESET ?= `[ -z $${TERM} ] && echo "" || tput sgr0`
+CARGO_COLOR ?= `[ -z $${NO_COLOR+x} ] && echo "" || echo "--color=never "`
+RED ?= `[ -z $${NO_COLOR+x} ] && ([ -z $${TERM} ] && echo "" || tput setaf 1) || echo ""`
+GREEN ?= `[ -z $${NO_COLOR+x} ] && ([ -z $${TERM} ] && echo "" || tput setaf 2) || echo ""`
+
 .POSIX:
 .SUFFIXES:
-meli:
-	cargo build --release
+meli: check-deps
+	@${CARGO_BIN} build ${CARGO_ARGS} ${CARGO_COLOR}--target-dir="${CARGO_TARGET_DIR}" ${FEATURES} --release
+
+help:
+	@echo "For a quick start, build and install locally:\n ${BOLD}${GREEN}PREFIX=~/.local make install${ANSI_RESET}\n"
+	@echo "Available subcommands:"
+	@echo " - ${BOLD}meli${ANSI_RESET} (builds meli with optimizations in \$$CARGO_TARGET_DIR)"
+	@echo " - ${BOLD}install${ANSI_RESET} (installs binary in \$$BINDIR and documentation to \$$MANDIR)"
+	@echo " - ${BOLD}uninstall${ANSI_RESET}"
+	@echo "Secondary subcommands:"
+	@echo " - ${BOLD}clean${ANSI_RESET} (cleans build artifacts)"
+	@echo " - ${BOLD}check-deps${ANSI_RESET} (checks dependencies)"
+	@echo " - ${BOLD}install-bin${ANSI_RESET} (installs binary to \$$BINDIR)"
+	@echo " - ${BOLD}install-doc${ANSI_RESET} (installs manpages to \$$MANDIR)"
+	@echo " - ${BOLD}help${ANSI_RESET} (prints this information)"
+
+	@echo " - ${BOLD}dist${ANSI_RESET} (creates release tarball named meli-"${VERSION}".tar.gz in this directory)"
+	@echo " - ${BOLD}deb-dist${ANSI_RESET} (builds debian package in the parent directory)"
+	@echo " - ${BOLD}distclean${ANSI_RESET} (cleans distribution build artifacts)"
+	@echo " - ${BOLD}build-rustdoc${ANSI_RESET} (builds rustdoc documentation for all packages in \$$CARGO_TARGET_DIR)"
+	@echo "\nENVIRONMENT variables of interest:"
+	@echo "* PREFIX = ${UNDERLINE}${EXPANDED_PREFIX}${ANSI_RESET}"
+	@echo -n "* MELI_FEATURES = ${UNDERLINE}"
+	@[ -z $${MELI_FEATURES+x} ] && echo -n "unset" || echo -n ${MELI_FEATURES}
+	@echo ${ANSI_RESET}
+	@echo "* BINDIR = ${UNDERLINE}${BINDIR}${ANSI_RESET}"
+	@echo "* MANDIR = ${UNDERLINE}${MANDIR}${ANSI_RESET}"
+	@echo -n "* MANPATH = ${UNDERLINE}"
+	@[ $${MANPATH+x} ] && echo -n $${MANPATH} || echo -n "unset"
+	@echo ${ANSI_RESET}
+	@echo "* (cleaned) output of manpath(1) = ${UNDERLINE}${MANPATHS}${ANSI_RESET}"
+	@echo -n "* NO_MAN ${UNDERLINE}"
+	@[ $${NO_MAN+x} ] && echo -n "set" || echo -n "unset"
+	@echo ${ANSI_RESET}
+	@echo -n "* NO_COLOR ${UNDERLINE}"
+	@[ $${NO_COLOR+x} ] && echo -n "set" || echo -n "unset"
+	@echo ${ANSI_RESET}
+	@echo "* CARGO_BIN = ${UNDERLINE}${CARGO_BIN}${ANSI_RESET}"
+	@echo "* CARGO_ARGS = ${UNDERLINE}${CARGO_ARGS}${ANSI_RESET}"
+	@echo "* MIN_RUSTC = ${UNDERLINE}${MIN_RUSTC}${ANSI_RESET}"
+	@#@echo "* CARGO_COLOR = ${CARGO_COLOR}"
+
+.PHONY: check
+check:
+	@${CARGO_BIN} test ${CARGO_ARGS} ${CARGO_COLOR}--target-dir="${CARGO_TARGET_DIR}" --workspace
+
+.PHONY: check-deps
+check-deps:
+	@(if ! echo ${MIN_RUSTC}\\n`${CARGO_BIN} --version | grep ^cargo | cut -d ' ' -f 2` | sort -CV; then echo "rust version >= ${RED}${MIN_RUSTC}${ANSI_RESET} required, found: `which ${CARGO_BIN}` `${CARGO_BIN} --version | cut -d ' ' -f 2`" \
+		"\nYour options:\n - Set CARGO_BIN to a supported version\n - Install a supported version from your distribution's package manager\n - Install a supported version from ${UNDERLINE}https://rustup.rs/${ANSI_RESET}" ; exit 1; fi)
 
-PREFIX=/usr/local
 
 .PHONY: clean
-clean: rm -ri ./target/
+clean:
+	-rm -rf ./${CARGO_TARGET_DIR}/
 
+.PHONY: distclean
+distclean: clean
+	@rm -f meli-${VERSION}.tar.gz
 
 .PHONY: uninstall
-uninstall: rm -f $(DESTDIR)$(PREFIX)/bin/meli
-	rm $(DESTDIR)$(PREFIX)/share/man/man1/meli.1.gz
-	rm $(DESTDIR)$(PREFIX)/share/man/man5/meli.conf.5.gz
+uninstall:
+	rm -f $(DESTDIR)${BINDIR}/meli
+	-rm $(DESTDIR)${MANDIR}/man1/meli.1.gz
+	-rm $(DESTDIR)${MANDIR}/man5/meli.conf.5.gz
+	-rm $(DESTDIR)${MANDIR}/man5/meli-themes.5.gz
+
+.PHONY: install-doc
+install-doc:
+	@(if [ -z $${NO_MAN+x} ]; then \
+		mkdir -p $(DESTDIR)${MANDIR}/man1 ; \
+		mkdir -p $(DESTDIR)${MANDIR}/man5 ; \
+		echo " - ${BOLD}Installing manpages to ${ANSI_RESET}${DESTDIR}${MANDIR}:" ; \
+		for MANPAGE in ${MANPAGES}; do \
+			SECTION=`echo $${MANPAGE} | rev | cut -d "." -f 1`; \
+			MANPAGEPATH=${DESTDIR}${MANDIR}/man$${SECTION}/$${MANPAGE}.gz; \
+			echo "  * installing $${MANPAGE} → ${GREEN}$${MANPAGEPATH}${ANSI_RESET}"; \
+			gzip -n < ${DOCS_SUBDIR}$${MANPAGE} > $${MANPAGEPATH} \
+    ; done ; \
+	(case ":${MANPATHS}:" in \
+  *:${DESTDIR}${MANDIR}:*) echo -n "";; \
+	*) echo "\n${RED}${BOLD}WARNING${ANSI_RESET}: ${UNDERLINE}Path ${DESTDIR}${MANDIR} is not contained in your MANPATH variable or the output of \`manpath\` command.${ANSI_RESET} \`man\` might fail finding the installed manpages. Consider adding it if necessary.\nMANPATH variable / output of \`manpath\`: ${MANPATHS}" ;; \
+	esac) ; \
+	else echo "NO_MAN is defined, so no documentation is going to be installed." ; fi)
+
+.PHONY: install-bin
+install-bin: meli
+	@mkdir -p $(DESTDIR)${BINDIR}
+	@echo " - ${BOLD}Installing binary to ${ANSI_RESET}${GREEN}${DESTDIR}${BINDIR}/meli${ANSI_RESET}"
+	@case ":${PATH}:" in \
+  *:${DESTDIR}${BINDIR}:*) echo -n "";; \
+	*) echo "\n${RED}${BOLD}WARNING${ANSI_RESET}: ${UNDERLINE}Path ${DESTDIR}${BINDIR} is not contained in your PATH variable.${ANSI_RESET} Consider adding it if necessary.\nPATH variable: ${PATH}";; \
+	esac
+	@mkdir -p $(DESTDIR)${BINDIR}
+	@rm -f  $(DESTDIR)${BINDIR}/meli
+	@cp ./${CARGO_TARGET_DIR}/release/meli $(DESTDIR)${BINDIR}/meli
+	@chmod 755 $(DESTDIR)${BINDIR}/meli
+
 
 .PHONY: install
-install: meli
-	mkdir -p $(DESTDIR)$(PREFIX)/bin
-	mkdir -p $(DESTDIR)$(PREFIX)/share/man/man1
-	mkdir -p $(DESTDIR)$(PREFIX)/share/man/man5
-	cp -f target/release/meli $(DESTDIR)$(PREFIX)/bin
-	gzip < meli.1 > $(DESTDIR)$(PREFIX)/share/man/man1/meli.1.gz
-	gzip < meli.conf.5 > $(DESTDIR)$(PREFIX)/share/man/man5/meli.conf.5.gz
+.NOTPARALLEL: yes
+install: meli install-bin install-doc
+	@(if [ -z $${NO_MAN+x} ]; then \
+	echo "\n You're ready to go. You might want to read the \"STARTING WITH meli\" section in the manpage (\`man meli\`)" ;\
+	fi)
+	@echo " - Report bugs in the mailing list or git issue tracker ${UNDERLINE}https://git.meli.delivery${ANSI_RESET}"
+	@echo " - If you have a specific feature or workflow you want to use, you can post in the mailing list or git issue tracker."
+
+.PHONY: dist
+dist:
+	@git archive --format=tar.gz --prefix=meli-${VERSION}/ HEAD >meli-${VERSION}.tar.gz
+	@echo meli-${VERSION}.tar.gz
+
+.PHONY: deb-dist
+deb-dist:
+	@dpkg-buildpackage -b -rfakeroot -us -uc
+	@echo ${BOLD}${GREEN}Generated${ANSI_RESET}  ../meli_${VERSION}-1_amd64.deb
+
+.PHONY: build-rustdoc
+build-rustdoc:
+	@RUSTDOCFLAGS="--crate-version ${VERSION}_${GIT_COMMIT}_${DATE}" ${CARGO_BIN} doc ${CARGO_ARGS} ${CARGO_COLOR}--target-dir="${CARGO_TARGET_DIR}" --all-features --no-deps --workspace --document-private-items --open

README

@@ -1,93 +0,0 @@
-    __
- __/  \__
-/  \__/  \__                       .
-\__/  \__/  \    , _ , _     ___   │   '
-/  \__   \__/    │' `│  `┒ .'   `  │   │
-\__/  \__/  \    │   │   │ |────'  │   │
-   \__/  \__/    │       / `.___, /\__ /
-      \__/
-                                    ,-.
-                                    \_/
-        terminal mail user agent   {|||)<
-                                    / \
-                                    `-'
-DOCUMENTATION
-=============
-
-After installing meli, see meli(1) and meli.conf(5) for documentation.
-
-BUILDING
-========
-
-meli requires rust 1.34 and rust's package manager, Cargo. Information on how
-to get it on your system can be found here:
-
-https://doc.rust-lang.org/cargo/getting-started/installation.html
-
-With Cargo available, the project can be built with
-
-# make
-
-The resulting binary will then be found under target/release/meli
-
-Run:
-
-# make install
-
-to install the binary and man pages. This requires root, so I suggest you override the default paths and install it in your $HOME:
-
-# make PREFIX=$HOME/.local install
-
-See meli(1) and meli.conf(5) for documentation.
-
-You can build and run meli with one command:
-
-# cargo run --release
-
-While the project is in early development, meli will only be developed for the
-linux kernel and respected linux distributions. Support for more UNIX-like OSes
-is on the roadmap.
-
-BUILDING IN DEBIAN
-==================
-
-Building with Debian's packaged cargo might require the installation of these
-two packages: librust-openssl-sys-dev and librust-libdbus-sys-dev
-
-DEVELOPMENT
-===========
-
-Development builds can be built and/or run with
-
-# cargo build
-# cargo run
-
-There is a debug/tracing log feature that can be enabled by using the flag
-`--feature debug-tracing` after uncommenting the features in `Cargo.toml`. The logs
-are printed in stderr, thus you can run meli with a redirection (i.e `2> log`)
-
-Code style follows the default rustfmt profile.
-
-CONFIG
-======
-
-meli by default looks for a configuration file in this location:
-# $XDG_CONFIG_HOME/meli/config
-
-You can run meli with arbitrary configuration files by setting the MELI_CONFIG
-environment variable to their locations, ie:
-
-# MELI_CONFIG=./test_config cargo run
-
-TESTING
-=======
-
-How to run specific tests:
-
-# cargo test -p {melib, ui, meli} (-- --nocapture) (--test test_name)
-
-PROFILING
-=========
-
-# perf record -g target/debug/bin
-# perf script | stackcollapse-perf | rust-unmangle | flamegraph > perf.svg

README.md

@@ -0,0 +1,130 @@
+# meli [![GitHub license](https://img.shields.io/github/license/meli/meli)](https://github.com/meli/meli/blob/master/COPYING) [![Crates.io](https://img.shields.io/crates/v/meli)](https://crates.io/crates/meli)
+
+**BSD/Linux terminal email client with support for multiple accounts and Maildir / mbox / notmuch / IMAP / JMAP.**
+
+Community links:
+[mailing lists](https://lists.meli.delivery/) | `#meli` on OFTC IRC | Report bugs and/or feature requests in [meli's issue tracker](https://git.meli.delivery/meli/meli/issues "meli gitea issue tracker")
+
+| | | |
+:---:|:---:|:---:
+![Main view screenshot](./docs/screenshots/main.webp "mail meli view screenshot")  |  ![Compact main view screenshot](./docs/screenshots/compact.webp "compact main view screenshot") | ![Compose with embed terminal editor screenshot](./docs/screenshots/compose.webp "composing view screenshot")
+Main view             |  Compact main view | Compose with embed terminal editor
+
+Main repository:
+* https://git.meli.delivery/meli/meli
+
+Official mirrors:
+* https://github.com/meli/meli
+
+## Install
+- Try an [online interactive web demo](https://meli.delivery/wasm2.html "online interactive web demo") powered by WebAssembly
+- [`cargo install meli`](https://crates.io/crates/meli "crates.io meli package")
+- [Download and install pre-built debian package, static linux binary](https://github.com/meli/meli/releases/ "github releases for meli"), or
+- Install with [Nix](https://search.nixos.org/packages?show=meli&query=meli&from=0&size=30&sort=relevance&channel=unstable#disabled "nixos package search results for 'meli'")
+
+## Documentation
+
+See also [Quickstart tutorial](https://meli.delivery/documentation.html#quick-start).
+
+After installing meli, see `meli(1)`, `meli.conf(5)` and `meli-themes(5)` for documentation. Sample configuration and theme files can be found in the `docs/samples/` subdirectory. Manual pages are also [hosted online](https://meli.delivery/documentation.html "meli documentation").
+
+meli by default looks for a configuration file in this location: `$XDG_CONFIG_HOME/meli/config.toml`
+
+You can run meli with arbitrary configuration files by setting the `$MELI_CONFIG`
+environment variable to their locations, i.e.:
+
+```sh
+MELI_CONFIG=./test_config cargo run
+```
+
+## Build
+For a quick start, build and install locally:
+
+```sh
+ PREFIX=~/.local make install
+```
+
+Available subcommands for `make` are listed with `make help`. The Makefile *should* be POSIX portable and not require a specific `make` version.
+
+meli requires rust 1.39 and rust's package manager, Cargo. Information on how
+to get it on your system can be found here: <https://doc.rust-lang.org/cargo/getting-started/installation.html>
+
+With Cargo available, the project can be built with `make` and the resulting binary will then be found under `target/release/meli`. Run `make install` to install the binary and man pages. This requires root, so I suggest you override the default paths and install it in your `$HOME`: `make PREFIX=$HOME/.local install`.
+
+You can build and run meli with one command: `cargo run --release`.
+
+### Build features
+
+Some functionality is held behind "feature gates", or compile-time flags. The following list explains each feature's purpose:
+
+- `gpgme` enables GPG support via `libgpgme` (on by default)
+- `dbus-notifications` enables showing notifications using `dbus` (on by default)
+- `notmuch` provides support for using a notmuch database as a mail backend (on by default)
+- `jmap` provides support for connecting to a jmap server and use it as a mail backend (off by default)
+- `sqlite3` provides support for builting fast search indexes in local sqlite3 databases (on by default)
+- `cli-docs` includes the manpage documentation compiled by either `mandoc` or `man` binary to plain text in `meli`'s command line. Embedded documentation can be viewed with the subcommand `meli man [PAGE]`
+- `svgscreenshot` provides support for taking screenshots of the current view of meli and saving it as SVG files. Its only purpose is taking screenshots for the official meli webpage. (off by default)
+- `debug-tracing` enables various trace debug logs from various places around the meli code base. The trace log is printed in `stderr`. (off by default)
+
+### Build Debian package (*deb*)
+
+Building with Debian's packaged cargo might require the installation of these
+two packages: `librust-openssl-sys-dev librust-libdbus-sys-dev`
+
+A `*.deb` package can be built with `make deb-dist`
+
+### Using notmuch
+
+To use the optional notmuch backend feature, you must have `libnotmuch5` installed in your system. In Debian-like systems, install the `libnotmuch5` packages. meli detects the library's presence on runtime.
+
+### Using GPG
+
+To use the optional gpg feature, you must have `libgpgme` installed in your system. In Debian-like systems, install the `libgpgme11` package. meli detects the library's presence on runtime.
+
+### Building with JMAP
+
+To build with JMAP support, prepend the environment variable `MELI_FEATURES='jmap'` to your make invocation:
+
+```sh
+MELI_FEATURES="jmap" make
+```
+
+or if building directly with cargo, use the flag `--features="jmap"'.
+
+# Development
+
+Development builds can be built and/or run with
+
+```
+cargo build
+cargo run
+```
+
+There is a debug/tracing log feature that can be enabled by using the flag
+`--feature debug-tracing` after uncommenting the features in `Cargo.toml`. The logs
+are printed in stderr, thus you can run meli with a redirection (i.e `2> log`)
+
+Code style follows the default rustfmt profile.
+
+## Testing
+
+How to run specific tests:
+
+```sh
+cargo test -p {melib, meli} (-- --nocapture) (--test test_name)
+```
+
+## Profiling
+
+```sh
+perf record -g target/debug/bin
+perf script | stackcollapse-perf | rust-unmangle | flamegraph > perf.svg
+```
+
+## Running fuzz targets
+
+Note: `cargo-fuzz` requires the nightly toolchain.
+
+```sh
+cargo +nightly fuzz run envelope_parse -- -dict=fuzz/envelope_tokens.dict
+```

build.rs

@@ -1,7 +1,7 @@
 /*
- * meli - bin.rs
+ * meli - build.rs
  *
- * Copyright 2019 Manos Pitsidianakis
+ * Copyright 2020  Manos Pitsidianakis
  *
  * This file is part of meli.
  *
@@ -18,60 +18,89 @@
  * You should have received a copy of the GNU General Public License
  * along with meli. If not, see <http://www.gnu.org/licenses/>.
  */
-use std::fs::File;
-use std::io::prelude::*;
-use std::io::BufWriter;
-use std::path::PathBuf;
-use std::process::Command;
 
-fn main() -> Result<(), std::io::Error> {
-    if let Err(e) = std::fs::create_dir("src/manuals") {
-        if e.kind() != std::io::ErrorKind::AlreadyExists {
-            Err(e)?;
-        }
-    }
-    let mut build_flag = false;
-    let meli_1_metadata = std::fs::metadata("meli.1")?;
-    if let Ok(metadata) = std::fs::metadata("src/manuals/meli.txt") {
-        if metadata.modified()? < meli_1_metadata.modified()? {
-            build_flag = true;
-        }
-    } else {
-        /* Doesn't exist */
-        build_flag = true;
-    }
+extern crate proc_macro;
+extern crate quote;
+extern crate syn;
+mod config_macros;
 
-    if build_flag {
-        let output = if let Ok(output) = Command::new("mandoc").args(&["meli.1"]).output() {
-            output.stdout
-        } else {
-            b"mandoc was not found on your system. It's required in order to compile the manual pages into plain text for use with the --*manual command line flags.".to_vec()
-        };
-        let man_path = PathBuf::from("src/manuals/meli.txt");
-        let file = File::create(&man_path)?;
-        BufWriter::new(file).write_all(&output)?;
-    }
+fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+    config_macros::override_derive(&[
+        ("src/conf/pager.rs", "PagerSettings"),
+        ("src/conf/listing.rs", "ListingSettings"),
+        ("src/conf/notifications.rs", "NotificationsSettings"),
+        ("src/conf/shortcuts.rs", "Shortcuts"),
+        ("src/conf/composing.rs", "ComposingSettings"),
+        ("src/conf/tags.rs", "TagsSettings"),
+        ("src/conf/pgp.rs", "PGPSettings"),
+    ]);
+    #[cfg(feature = "cli-docs")]
+    {
+        use flate2::Compression;
+        use flate2::GzBuilder;
+        const MANDOC_OPTS: &[&'static str] = &["-T", "utf8", "-I", "os=Generated by mandoc(1)"];
+        use std::env;
+        use std::fs::File;
+        use std::io::prelude::*;
+        use std::path::Path;
+        use std::process::Command;
+        let out_dir = env::var("OUT_DIR").unwrap();
+        let mut out_dir_path = Path::new(&out_dir).to_path_buf();
+        out_dir_path.push("meli.txt.gz");
 
-    let mut build_flag = false;
-    let meli_conf_5_metadata = std::fs::metadata("meli.conf.5")?;
-    if let Ok(metadata) = std::fs::metadata("src/manuals/meli_conf.txt") {
-        if metadata.modified()? < meli_conf_5_metadata.modified()? {
-            build_flag = true;
-        }
-    } else {
-        /* Doesn't exist */
-        build_flag = true;
-    }
+        let output = Command::new("mandoc")
+            .args(MANDOC_OPTS)
+            .arg("docs/meli.1")
+            .output()
+            .or_else(|_| Command::new("man").arg("-l").arg("docs/meli.1").output())
+            .unwrap();
 
-    if build_flag {
-        let output = if let Ok(output) = Command::new("mandoc").args(&["meli.conf.5"]).output() {
-            output.stdout
-        } else {
-            b"mandoc was not found on your system. It's required in order to compile the manual pages into plain text for use with the --*manual command line flags.".to_vec()
-        };
-        let man_path = PathBuf::from("src/manuals/meli_conf.txt");
-        let file = File::create(&man_path)?;
-        BufWriter::new(file).write_all(&output)?;
+        let file = File::create(&out_dir_path).unwrap();
+        let mut gz = GzBuilder::new()
+            .comment(output.stdout.len().to_string().into_bytes())
+            .write(file, Compression::default());
+        gz.write_all(&output.stdout).unwrap();
+        gz.finish().unwrap();
+        out_dir_path.pop();
+
+        out_dir_path.push("meli.conf.txt.gz");
+        let output = Command::new("mandoc")
+            .args(MANDOC_OPTS)
+            .arg("docs/meli.conf.5")
+            .output()
+            .or_else(|_| {
+                Command::new("man")
+                    .arg("-l")
+                    .arg("docs/meli.conf.5")
+                    .output()
+            })
+            .unwrap();
+        let file = File::create(&out_dir_path).unwrap();
+        let mut gz = GzBuilder::new()
+            .comment(output.stdout.len().to_string().into_bytes())
+            .write(file, Compression::default());
+        gz.write_all(&output.stdout).unwrap();
+        gz.finish().unwrap();
+        out_dir_path.pop();
+
+        out_dir_path.push("meli-themes.txt.gz");
+        let output = Command::new("mandoc")
+            .args(MANDOC_OPTS)
+            .arg("docs/meli-themes.5")
+            .output()
+            .or_else(|_| {
+                Command::new("man")
+                    .arg("-l")
+                    .arg("docs/meli-themes.5")
+                    .output()
+            })
+            .unwrap();
+        let file = File::create(&out_dir_path).unwrap();
+        let mut gz = GzBuilder::new()
+            .comment(output.stdout.len().to_string().into_bytes())
+            .write(file, Compression::default());
+        gz.write_all(&output.stdout).unwrap();
+        gz.finish().unwrap();
     }
-    Ok(())
 }

config_macros.rs

@@ -0,0 +1,218 @@
+/*
+ * meli -
+ *
+ * Copyright  Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use std::fs::File;
+use std::io::prelude::*;
+use std::process::{Command, Stdio};
+
+use quote::{format_ident, quote};
+
+// Write ConfigStructOverride to overrides.rs
+pub fn override_derive(filenames: &[(&str, &str)]) {
+    let mut output_file =
+        File::create("src/conf/overrides.rs").expect("Unable to open output file");
+    let mut output_string = r##"/*
+ * meli - conf/overrides.rs
+ *
+ * Copyright 2020 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//! This module is automatically generated by build.rs.
+use super::*;
+
+"##
+    .to_string();
+
+    'file_loop: for (filename, ident) in filenames {
+        println!("cargo:rerun-if-changed={}", filename);
+        let mut file = File::open(&filename)
+            .unwrap_or_else(|err| panic!("Unable to open file `{}` {}", filename, err));
+
+        let mut src = String::new();
+        file.read_to_string(&mut src).expect("Unable to read file");
+
+        let syntax = syn::parse_file(&src).expect("Unable to parse file");
+        if syntax.items.iter().any(|item| {
+            if let syn::Item::Struct(s) = item {
+                if s.ident.to_string().ends_with("Override") {
+                    println!("ident {} exists, skipping {}", ident, filename);
+                    return true;
+                }
+            }
+            false
+        }) {
+            continue 'file_loop;
+        }
+
+        for item in syntax.items.iter() {
+            if let syn::Item::Struct(s) = item {
+                if s.ident != ident {
+                    continue;
+                }
+                if s.ident.to_string().ends_with("Override") {
+                    unreachable!();
+                }
+                let override_ident: syn::Ident = format_ident!("{}Override", s.ident);
+                let mut field_tokentrees = vec![];
+                let mut attrs_tokens = vec![];
+                for attr in &s.attrs {
+                    if let Ok(syn::Meta::List(ml)) = attr.parse_meta() {
+                        if ml.path.get_ident().is_some() && ml.path.get_ident().unwrap() == "cfg" {
+                            attrs_tokens.push(attr);
+                        }
+                    }
+                }
+                let mut field_idents = vec![];
+                for f in &s.fields {
+                    let ident = &f.ident;
+                    let ty = &f.ty;
+                    let attrs = f
+                        .attrs
+                        .iter()
+                        .filter_map(|f| {
+                            let mut new_attr = f.clone();
+                            if let quote::__private::TokenTree::Group(g) =
+                                f.tokens.clone().into_iter().next().unwrap()
+                            {
+                                let attr_inner_value = f.tokens.to_string();
+                                if !attr_inner_value.starts_with("( default")
+                                    && !attr_inner_value.starts_with("( default =")
+                                    && !attr_inner_value.starts_with("(default")
+                                    && !attr_inner_value.starts_with("(default =")
+                                {
+                                    return Some(new_attr);
+                                }
+                                if attr_inner_value.starts_with("( default =")
+                                    || attr_inner_value.starts_with("(default =")
+                                {
+                                    let rest = g.stream().into_iter().skip(4);
+                                    new_attr.tokens = quote! { ( #(#rest)*) };
+                                    match new_attr.tokens.to_string().as_str() {
+                                        "( )" | "()" => {
+                                            return None;
+                                        }
+                                        _ => {}
+                                    }
+                                } else if attr_inner_value.starts_with("( default")
+                                    || attr_inner_value.starts_with("(default")
+                                {
+                                    let rest = g.stream().into_iter().skip(2);
+                                    new_attr.tokens = quote! { ( #(#rest)*) };
+                                    match new_attr.tokens.to_string().as_str() {
+                                        "( )" | "()" => {
+                                            return None;
+                                        }
+                                        _ => {}
+                                    }
+                                }
+                            }
+
+                            Some(new_attr)
+                        })
+                        .collect::<Vec<_>>();
+                    let t = quote! {
+                        #(#attrs)*
+                        #[serde(default)]
+                        pub #ident : Option<#ty>
+                    };
+                    field_idents.push(ident);
+                    field_tokentrees.push(t);
+                }
+                //let fields = &s.fields;
+
+                let literal_struct = quote! {
+                    #(#attrs_tokens)*
+                    #[derive(Debug, Serialize, Deserialize, Clone)]
+                    #[serde(deny_unknown_fields)]
+                    pub struct #override_ident {
+                        #(#field_tokentrees),*
+                    }
+
+
+                    #(#attrs_tokens)*
+                    impl Default for #override_ident {
+                        fn default() -> Self {
+                            #override_ident {
+                                #(#field_idents: None),*
+                            }
+                        }
+                    }
+                };
+                output_string.push_str(&literal_struct.to_string());
+                output_string.push_str("\n\n");
+            }
+        }
+    }
+
+    let rustfmt_closure = move |output_file: &mut File, output_string: &str| {
+        let mut rustfmt = Command::new("rustfmt")
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()
+            .map_err(|err| format!("failed to execute rustfmt {}", err))?;
+
+        {
+            // limited borrow of stdin
+            let stdin = rustfmt
+                .stdin
+                .as_mut()
+                .ok_or("failed to get rustfmt stdin")?;
+            stdin
+                .write_all(output_string.as_bytes())
+                .map_err(|err| format!("failed to write to rustfmt stdin {}", err))?;
+        }
+
+        let output = rustfmt
+            .wait_with_output()
+            .map_err(|err| format!("failed to wait on rustfmt child {}", err))?;
+        if !output.stderr.is_empty() {
+            return Err(format!(
+                "rustfmt invocation replied with: `{}`",
+                String::from_utf8_lossy(&output.stderr)
+            ));
+        }
+
+        output_file
+            .write_all(&output.stdout)
+            .expect("failed to write to src/conf/overrides.rs");
+        Ok(())
+    };
+    if let Err(err) = rustfmt_closure(&mut output_file, &output_string) {
+        println!("Tried rustfmt on overrides module, got error: {}", err);
+        output_file.write_all(output_string.as_bytes()).unwrap();
+    }
+}

contrib/oauth2.py

@@ -0,0 +1,348 @@
+#!/usr/bin/env python3
+#
+# Copyright 2012 Google Inc.
+# Copyright 2020 Manos Pitsidianakis
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+     # http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Performs client tasks for testing IMAP OAuth2 authentication.
+
+To use this script, you'll need to have registered with Google as an OAuth
+application and obtained an OAuth client ID and client secret.
+See https://developers.google.com/identity/protocols/OAuth2 for instructions on
+registering and for documentation of the APIs invoked by this code.
+
+This script has 3 modes of operation.
+
+1. The first mode is used to generate and authorize an OAuth2 token, the
+first step in logging in via OAuth2.
+
+  oauth2 --user=xxx@gmail.com \
+    --client_id=1038[...].apps.googleusercontent.com \
+    --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ \
+    --generate_oauth2_token
+
+The script will converse with Google and generate an oauth request
+token, then present you with a URL you should visit in your browser to
+authorize the token. Once you get the verification code from the Google
+website, enter it into the script to get your OAuth access token. The output
+from this command will contain the access token, a refresh token, and some
+metadata about the tokens. The access token can be used until it expires, and
+the refresh token lasts indefinitely, so you should record these values for
+reuse.
+
+2. The script will generate new access tokens using a refresh token.
+
+  oauth2 --user=xxx@gmail.com \
+    --client_id=1038[...].apps.googleusercontent.com \
+    --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ \
+    --refresh_token=1/Yzm6MRy4q1xi7Dx2DuWXNgT6s37OrP_DW_IoyTum4YA
+
+3. The script will generate an OAuth2 string that can be fed
+directly to IMAP or SMTP. This is triggered with the --generate_oauth2_string
+option.
+
+  oauth2 --generate_oauth2_string --user=xxx@gmail.com \
+    --access_token=ya29.AGy[...]ezLg
+
+The output of this mode will be a base64-encoded string. To use it, connect to a
+IMAPFE and pass it as the second argument to the AUTHENTICATE command.
+
+  a AUTHENTICATE XOAUTH2 a9sha9sfs[...]9dfja929dk==
+"""
+
+import base64
+import imaplib
+import json
+from optparse import OptionParser
+import smtplib
+import sys
+import urllib.request, urllib.parse, urllib.error
+
+
+def SetupOptionParser():
+  # Usage message is the module's docstring.
+  parser = OptionParser(usage=__doc__)
+  parser.add_option('--generate_oauth2_token',
+                    action='store_true',
+                    dest='generate_oauth2_token',
+                    help='generates an OAuth2 token for testing')
+  parser.add_option('--generate_oauth2_string',
+                    action='store_true',
+                    dest='generate_oauth2_string',
+                    help='generates an initial client response string for '
+                         'OAuth2')
+  parser.add_option('--client_id',
+                    default=None,
+                    help='Client ID of the application that is authenticating. '
+                         'See OAuth2 documentation for details.')
+  parser.add_option('--client_secret',
+                    default=None,
+                    help='Client secret of the application that is '
+                         'authenticating. See OAuth2 documentation for '
+                         'details.')
+  parser.add_option('--access_token',
+                    default=None,
+                    help='OAuth2 access token')
+  parser.add_option('--refresh_token',
+                    default=None,
+                    help='OAuth2 refresh token')
+  parser.add_option('--scope',
+                    default='https://mail.google.com/',
+                    help='scope for the access token. Multiple scopes can be '
+                         'listed separated by spaces with the whole argument '
+                         'quoted.')
+  parser.add_option('--test_imap_authentication',
+                    action='store_true',
+                    dest='test_imap_authentication',
+                    help='attempts to authenticate to IMAP')
+  parser.add_option('--test_smtp_authentication',
+                    action='store_true',
+                    dest='test_smtp_authentication',
+                    help='attempts to authenticate to SMTP')
+  parser.add_option('--user',
+                    default=None,
+                    help='email address of user whose account is being '
+                         'accessed')
+  parser.add_option('--quiet',
+                    action='store_true',
+                    default=False,
+                    dest='quiet',
+                    help='Omit verbose descriptions and only print '
+                         'machine-readable outputs.')
+  return parser
+
+
+# The URL root for accessing Google Accounts.
+GOOGLE_ACCOUNTS_BASE_URL = 'https://accounts.google.com'
+
+
+# Hardcoded dummy redirect URI for non-web apps.
+REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob'
+
+
+def AccountsUrl(command):
+  """Generates the Google Accounts URL.
+
+  Args:
+    command: The command to execute.
+
+  Returns:
+    A URL for the given command.
+  """
+  return '%s/%s' % (GOOGLE_ACCOUNTS_BASE_URL, command)
+
+
+def UrlEscape(text):
+  # See OAUTH 5.1 for a definition of which characters need to be escaped.
+  return urllib.parse.quote(text, safe='~-._')
+
+
+def UrlUnescape(text):
+  # See OAUTH 5.1 for a definition of which characters need to be escaped.
+  return urllib.parse.unquote(text)
+
+
+def FormatUrlParams(params):
+  """Formats parameters into a URL query string.
+
+  Args:
+    params: A key-value map.
+
+  Returns:
+    A URL query string version of the given parameters.
+  """
+  param_fragments = []
+  for param in sorted(iter(params.items()), key=lambda x: x[0]):
+    param_fragments.append('%s=%s' % (param[0], UrlEscape(param[1])))
+  return '&'.join(param_fragments)
+
+
+def GeneratePermissionUrl(client_id, scope='https://mail.google.com/'):
+  """Generates the URL for authorizing access.
+
+  This uses the "OAuth2 for Installed Applications" flow described at
+  https://developers.google.com/accounts/docs/OAuth2InstalledApp
+
+  Args:
+    client_id: Client ID obtained by registering your app.
+    scope: scope for access token, e.g. 'https://mail.google.com'
+  Returns:
+    A URL that the user should visit in their browser.
+  """
+  params = {}
+  params['client_id'] = client_id
+  params['redirect_uri'] = REDIRECT_URI
+  params['scope'] = scope
+  params['response_type'] = 'code'
+  return '%s?%s' % (AccountsUrl('o/oauth2/auth'),
+                    FormatUrlParams(params))
+
+
+def AuthorizeTokens(client_id, client_secret, authorization_code):
+  """Obtains OAuth access token and refresh token.
+
+  This uses the application portion of the "OAuth2 for Installed Applications"
+  flow at https://developers.google.com/accounts/docs/OAuth2InstalledApp#handlingtheresponse
+
+  Args:
+    client_id: Client ID obtained by registering your app.
+    client_secret: Client secret obtained by registering your app.
+    authorization_code: code generated by Google Accounts after user grants
+        permission.
+  Returns:
+    The decoded response from the Google Accounts server, as a dict. Expected
+    fields include 'access_token', 'expires_in', and 'refresh_token'.
+  """
+  params = {}
+  params['client_id'] = client_id
+  params['client_secret'] = client_secret
+  params['code'] = authorization_code
+  params['redirect_uri'] = REDIRECT_URI
+  params['grant_type'] = 'authorization_code'
+  request_url = AccountsUrl('o/oauth2/token')
+
+  response = urllib.request.urlopen(request_url, urllib.parse.urlencode(params).encode()).read()
+  return json.loads(response)
+
+
+def RefreshToken(client_id, client_secret, refresh_token):
+  """Obtains a new token given a refresh token.
+
+  See https://developers.google.com/accounts/docs/OAuth2InstalledApp#refresh
+
+  Args:
+    client_id: Client ID obtained by registering your app.
+    client_secret: Client secret obtained by registering your app.
+    refresh_token: A previously-obtained refresh token.
+  Returns:
+    The decoded response from the Google Accounts server, as a dict. Expected
+    fields include 'access_token', 'expires_in', and 'refresh_token'.
+  """
+  params = {}
+  params['client_id'] = client_id
+  params['client_secret'] = client_secret
+  params['refresh_token'] = refresh_token
+  params['grant_type'] = 'refresh_token'
+  request_url = AccountsUrl('o/oauth2/token')
+
+  response = urllib.request.urlopen(request_url, urllib.parse.urlencode(params).encode()).read()
+  return json.loads(response)
+
+
+def GenerateOAuth2String(username, access_token, base64_encode=True):
+  """Generates an IMAP OAuth2 authentication string.
+
+  See https://developers.google.com/google-apps/gmail/oauth2_overview
+
+  Args:
+    username: the username (email address) of the account to authenticate
+    access_token: An OAuth2 access token.
+    base64_encode: Whether to base64-encode the output.
+
+  Returns:
+    The SASL argument for the OAuth2 mechanism.
+  """
+  auth_string = 'user=%s\1auth=Bearer %s\1\1' % (username, access_token)
+  if base64_encode:
+    auth_string = base64.b64encode(bytes(auth_string, 'utf-8'))
+  return auth_string
+
+
+def TestImapAuthentication(user, auth_string):
+  """Authenticates to IMAP with the given auth_string.
+
+  Prints a debug trace of the attempted IMAP connection.
+
+  Args:
+    user: The Gmail username (full email address)
+    auth_string: A valid OAuth2 string, as returned by GenerateOAuth2String.
+        Must not be base64-encoded, since imaplib does its own base64-encoding.
+  """
+  print()
+  imap_conn = imaplib.IMAP4_SSL('imap.gmail.com')
+  imap_conn.debug = 4
+  imap_conn.authenticate('XOAUTH2', lambda x: auth_string)
+  imap_conn.select('INBOX')
+
+
+def TestSmtpAuthentication(user, auth_string):
+  """Authenticates to SMTP with the given auth_string.
+
+  Args:
+    user: The Gmail username (full email address)
+    auth_string: A valid OAuth2 string, not base64-encoded, as returned by
+        GenerateOAuth2String.
+  """
+  print()
+  smtp_conn = smtplib.SMTP('smtp.gmail.com', 587)
+  smtp_conn.set_debuglevel(True)
+  smtp_conn.ehlo('test')
+  smtp_conn.starttls()
+  smtp_conn.docmd('AUTH', 'XOAUTH2 ' + base64.b64encode(auth_string))
+
+
+def RequireOptions(options, *args):
+  missing = [arg for arg in args if getattr(options, arg) is None]
+  if missing:
+    print('Missing options: %s' % ' '.join(missing), file=sys.stderr)
+    sys.exit(-1)
+
+
+def main(argv):
+  options_parser = SetupOptionParser()
+  (options, args) = options_parser.parse_args()
+  if options.refresh_token:
+    RequireOptions(options, 'client_id', 'client_secret')
+    response = RefreshToken(options.client_id, options.client_secret,
+                            options.refresh_token)
+    if options.quiet:
+      print(response['access_token'])
+    else:
+      print('Access Token: %s' % response['access_token'])
+      print('Access Token Expiration Seconds: %s' % response['expires_in'])
+  elif options.generate_oauth2_string:
+    RequireOptions(options, 'user', 'access_token')
+    oauth2_string = GenerateOAuth2String(options.user, options.access_token)
+    if options.quiet:
+      print(oauth2_string.decode('utf-8'))
+    else:
+      print('OAuth2 argument:\n' + oauth2_string.decode('utf-8'))
+  elif options.generate_oauth2_token:
+    RequireOptions(options, 'client_id', 'client_secret')
+    print('To authorize token, visit this url and follow the directions:')
+    print('  %s' % GeneratePermissionUrl(options.client_id, options.scope))
+    authorization_code = input('Enter verification code: ')
+    response = AuthorizeTokens(options.client_id, options.client_secret,
+                                authorization_code)
+    print('Refresh Token: %s' % response['refresh_token'])
+    print('Access Token: %s' % response['access_token'])
+    print('Access Token Expiration Seconds: %s' % response['expires_in'])
+  elif options.test_imap_authentication:
+    RequireOptions(options, 'user', 'access_token')
+    TestImapAuthentication(options.user,
+        GenerateOAuth2String(options.user, options.access_token,
+                             base64_encode=False))
+  elif options.test_smtp_authentication:
+    RequireOptions(options, 'user', 'access_token')
+    TestSmtpAuthentication(options.user,
+        GenerateOAuth2String(options.user, options.access_token,
+                             base64_encode=False))
+  else:
+    options_parser.print_help()
+    print('Nothing to do, exiting.')
+    return
+
+
+if __name__ == '__main__':
+  main(sys.argv)

debian/changelog

@@ -0,0 +1,49 @@
+meli (0.6.2-1) buster; urgency=low
+
+  Added
+  - Add customizable mailbox tree in sidebar
+  - Make `dbus` dependency opt-out (feature is `dbus-notifications`)
+  - Implemented JMAP async, search, tagging, syncing
+  - Preserve account order from configuration file
+  - Implemented IMAP `CONDSTORE` support for IMAP cache
+  - Add `timeout` setting for IMAP
+  - Implement TCP keepalive for IMAP
+  - Rewrote email address parsers.
+  - Implement `copy_messages` for maildir
+  - Implement selection with motions
+
+  Fixed
+  - Fixed various problems with IMAP cache
+  - Fixed various problems with IMAP message counts
+  - Fixed various problems with IMAP connection hanging
+  - Fixed IMAP not reconnecting on dropped IDLE connections
+  - Fixed various problems with notmuch backend
+
+ -- Manos Pitsidianakis <epilys@nessuent.xyz>  Thu, 24 Sep 2020 18:14:00 +0200
+meli (0.6.1-1) buster; urgency=low
+
+  * added experimental NNTP backend
+  * added server extension support and use in account status tab
+  * imap: fixed IDLE connection getting stuck when using DEFLATE
+
+ -- Manos Pitsidianakis <epilys@nessuent.xyz>  Sun, 02 Aug 2020 01:09:05 +0200
+meli (0.6.0-1) buster; urgency=low
+
+  * Update to 0.6.0
+
+ -- Manos Pitsidianakis <epilys@nessuent.xyz>  Wed, 29 Jul 2020 22:24:08 +0200
+meli (0.5.1-1) buster; urgency=low
+
+  * Update to 0.5.1
+
+ -- Manos Pitsidianakis <epilys@nessuent.xyz>  Wed, 29 Jan 2020 22:24:08 +0200
+meli (0.5.0-1) buster; urgency=low
+
+  * Update to 0.5.0
+
+ -- Manos Pitsidianakis <epilys@nessuent.xyz>  Wed, 29 Jan 2020 22:24:08 +0200
+meli (0.4.1-1) buster; urgency=low
+
+  * Initial release.
+
+ -- Manos Pitsidianakis <epilys@nessuent.xyz>  Wed, 29 Jan 2020 22:24:08 +0200

debian/compat

@@ -0,0 +1 @@
+11

debian/control

@@ -0,0 +1,14 @@
+Source: meli
+Section: mail
+Priority: optional
+Maintainer: Manos Pitsidianakis <epilys@nessuent.xyz>
+Build-Depends: debhelper (>=11~), mandoc (>=1.14.4-1)
+Standards-Version: 4.1.4
+Homepage: https://meli.delivery
+
+Package: meli
+Architecture: any
+Multi-Arch: foreign
+Depends: ${misc:Depends}, ${shlibs:Depends}
+Recommends: libnotmuch, xdg-utils (>=1.1.3-1)
+Description: terminal mail client

debian/copyright

@@ -0,0 +1,685 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: meli
+Source: <https://git.meli.delivery/meli/meli>
+#
+# Please double check copyright with the licensecheck(1) command.
+
+Files:     *
+Copyright: 2017-2020 Manos Pitsidianakis
+License:   GPL-3.0+
+#----------------------------------------------------------------------------
+# License file: COPYING
+                     GNU GENERAL PUBLIC LICENSE
+                        Version 3, 29 June 2007
+ .
+  Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+  Everyone is permitted to copy and distribute verbatim copies
+  of this license document, but changing it is not allowed.
+ .
+                             Preamble
+ .
+   The GNU General Public License is a free, copyleft license for
+ software and other kinds of works.
+ .
+   The licenses for most software and other practical works are designed
+ to take away your freedom to share and change the works.  By contrast,
+ the GNU General Public License is intended to guarantee your freedom to
+ share and change all versions of a program--to make sure it remains free
+ software for all its users.  We, the Free Software Foundation, use the
+ GNU General Public License for most of our software; it applies also to
+ any other work released this way by its authors.  You can apply it to
+ your programs, too.
+ .
+   When we speak of free software, we are referring to freedom, not
+ price.  Our General Public Licenses are designed to make sure that you
+ have the freedom to distribute copies of free software (and charge for
+ them if you wish), that you receive source code or can get it if you
+ want it, that you can change the software or use pieces of it in new
+ free programs, and that you know you can do these things.
+ .
+   To protect your rights, we need to prevent others from denying you
+ these rights or asking you to surrender the rights.  Therefore, you have
+ certain responsibilities if you distribute copies of the software, or if
+ you modify it: responsibilities to respect the freedom of others.
+ .
+   For example, if you distribute copies of such a program, whether
+ gratis or for a fee, you must pass on to the recipients the same
+ freedoms that you received.  You must make sure that they, too, receive
+ or can get the source code.  And you must show them these terms so they
+ know their rights.
+ .
+   Developers that use the GNU GPL protect your rights with two steps:
+ (1) assert copyright on the software, and (2) offer you this License
+ giving you legal permission to copy, distribute and/or modify it.
+ .
+   For the developers' and authors' protection, the GPL clearly explains
+ that there is no warranty for this free software.  For both users' and
+ authors' sake, the GPL requires that modified versions be marked as
+ changed, so that their problems will not be attributed erroneously to
+ authors of previous versions.
+ .
+   Some devices are designed to deny users access to install or run
+ modified versions of the software inside them, although the manufacturer
+ can do so.  This is fundamentally incompatible with the aim of
+ protecting users' freedom to change the software.  The systematic
+ pattern of such abuse occurs in the area of products for individuals to
+ use, which is precisely where it is most unacceptable.  Therefore, we
+ have designed this version of the GPL to prohibit the practice for those
+ products.  If such problems arise substantially in other domains, we
+ stand ready to extend this provision to those domains in future versions
+ of the GPL, as needed to protect the freedom of users.
+ .
+   Finally, every program is threatened constantly by software patents.
+ States should not allow patents to restrict development and use of
+ software on general-purpose computers, but in those that do, we wish to
+ avoid the special danger that patents applied to a free program could
+ make it effectively proprietary.  To prevent this, the GPL assures that
+ patents cannot be used to render the program non-free.
+ .
+   The precise terms and conditions for copying, distribution and
+ modification follow.
+ .
+                        TERMS AND CONDITIONS
+ .
+   0. Definitions.
+ .
+   "This License" refers to version 3 of the GNU General Public License.
+ .
+   "Copyright" also means copyright-like laws that apply to other kinds of
+ works, such as semiconductor masks.
+ .
+   "The Program" refers to any copyrightable work licensed under this
+ License.  Each licensee is addressed as "you".  "Licensees" and
+ "recipients" may be individuals or organizations.
+ .
+   To "modify" a work means to copy from or adapt all or part of the work
+ in a fashion requiring copyright permission, other than the making of an
+ exact copy.  The resulting work is called a "modified version" of the
+ earlier work or a work "based on" the earlier work.
+ .
+   A "covered work" means either the unmodified Program or a work based
+ on the Program.
+ .
+   To "propagate" a work means to do anything with it that, without
+ permission, would make you directly or secondarily liable for
+ infringement under applicable copyright law, except executing it on a
+ computer or modifying a private copy.  Propagation includes copying,
+ distribution (with or without modification), making available to the
+ public, and in some countries other activities as well.
+ .
+   To "convey" a work means any kind of propagation that enables other
+ parties to make or receive copies.  Mere interaction with a user through
+ a computer network, with no transfer of a copy, is not conveying.
+ .
+   An interactive user interface displays "Appropriate Legal Notices"
+ to the extent that it includes a convenient and prominently visible
+ feature that (1) displays an appropriate copyright notice, and (2)
+ tells the user that there is no warranty for the work (except to the
+ extent that warranties are provided), that licensees may convey the
+ work under this License, and how to view a copy of this License.  If
+ the interface presents a list of user commands or options, such as a
+ menu, a prominent item in the list meets this criterion.
+ .
+   1. Source Code.
+ .
+   The "source code" for a work means the preferred form of the work
+ for making modifications to it.  "Object code" means any non-source
+ form of a work.
+ .
+   A "Standard Interface" means an interface that either is an official
+ standard defined by a recognized standards body, or, in the case of
+ interfaces specified for a particular programming language, one that
+ is widely used among developers working in that language.
+ .
+   The "System Libraries" of an executable work include anything, other
+ than the work as a whole, that (a) is included in the normal form of
+ packaging a Major Component, but which is not part of that Major
+ Component, and (b) serves only to enable use of the work with that
+ Major Component, or to implement a Standard Interface for which an
+ implementation is available to the public in source code form.  A
+ "Major Component", in this context, means a major essential component
+ (kernel, window system, and so on) of the specific operating system
+ (if any) on which the executable work runs, or a compiler used to
+ produce the work, or an object code interpreter used to run it.
+ .
+   The "Corresponding Source" for a work in object code form means all
+ the source code needed to generate, install, and (for an executable
+ work) run the object code and to modify the work, including scripts to
+ control those activities.  However, it does not include the work's
+ System Libraries, or general-purpose tools or generally available free
+ programs which are used unmodified in performing those activities but
+ which are not part of the work.  For example, Corresponding Source
+ includes interface definition files associated with source files for
+ the work, and the source code for shared libraries and dynamically
+ linked subprograms that the work is specifically designed to require,
+ such as by intimate data communication or control flow between those
+ subprograms and other parts of the work.
+ .
+   The Corresponding Source need not include anything that users
+ can regenerate automatically from other parts of the Corresponding
+ Source.
+ .
+   The Corresponding Source for a work in source code form is that
+ same work.
+ .
+   2. Basic Permissions.
+ .
+   All rights granted under this License are granted for the term of
+ copyright on the Program, and are irrevocable provided the stated
+ conditions are met.  This License explicitly affirms your unlimited
+ permission to run the unmodified Program.  The output from running a
+ covered work is covered by this License only if the output, given its
+ content, constitutes a covered work.  This License acknowledges your
+ rights of fair use or other equivalent, as provided by copyright law.
+ .
+   You may make, run and propagate covered works that you do not
+ convey, without conditions so long as your license otherwise remains
+ in force.  You may convey covered works to others for the sole purpose
+ of having them make modifications exclusively for you, or provide you
+ with facilities for running those works, provided that you comply with
+ the terms of this License in conveying all material for which you do
+ not control copyright.  Those thus making or running the covered works
+ for you must do so exclusively on your behalf, under your direction
+ and control, on terms that prohibit them from making any copies of
+ your copyrighted material outside their relationship with you.
+ .
+   Conveying under any other circumstances is permitted solely under
+ the conditions stated below.  Sublicensing is not allowed; section 10
+ makes it unnecessary.
+ .
+   3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+ .
+   No covered work shall be deemed part of an effective technological
+ measure under any applicable law fulfilling obligations under article
+ 11 of the WIPO copyright treaty adopted on 20 December 1996, or
+ similar laws prohibiting or restricting circumvention of such
+ measures.
+ .
+   When you convey a covered work, you waive any legal power to forbid
+ circumvention of technological measures to the extent such circumvention
+ is effected by exercising rights under this License with respect to
+ the covered work, and you disclaim any intention to limit operation or
+ modification of the work as a means of enforcing, against the work's
+ users, your or third parties' legal rights to forbid circumvention of
+ technological measures.
+ .
+   4. Conveying Verbatim Copies.
+ .
+   You may convey verbatim copies of the Program's source code as you
+ receive it, in any medium, provided that you conspicuously and
+ appropriately publish on each copy an appropriate copyright notice;
+ keep intact all notices stating that this License and any
+ non-permissive terms added in accord with section 7 apply to the code;
+ keep intact all notices of the absence of any warranty; and give all
+ recipients a copy of this License along with the Program.
+ .
+   You may charge any price or no price for each copy that you convey,
+ and you may offer support or warranty protection for a fee.
+ .
+   5. Conveying Modified Source Versions.
+ .
+   You may convey a work based on the Program, or the modifications to
+ produce it from the Program, in the form of source code under the
+ terms of section 4, provided that you also meet all of these conditions:
+ .
+     a) The work must carry prominent notices stating that you modified
+     it, and giving a relevant date.
+ .
+     b) The work must carry prominent notices stating that it is
+     released under this License and any conditions added under section
+     7.  This requirement modifies the requirement in section 4 to
+     "keep intact all notices".
+ .
+     c) You must license the entire work, as a whole, under this
+     License to anyone who comes into possession of a copy.  This
+     License will therefore apply, along with any applicable section 7
+     additional terms, to the whole of the work, and all its parts,
+     regardless of how they are packaged.  This License gives no
+     permission to license the work in any other way, but it does not
+     invalidate such permission if you have separately received it.
+ .
+     d) If the work has interactive user interfaces, each must display
+     Appropriate Legal Notices; however, if the Program has interactive
+     interfaces that do not display Appropriate Legal Notices, your
+     work need not make them do so.
+ .
+   A compilation of a covered work with other separate and independent
+ works, which are not by their nature extensions of the covered work,
+ and which are not combined with it such as to form a larger program,
+ in or on a volume of a storage or distribution medium, is called an
+ "aggregate" if the compilation and its resulting copyright are not
+ used to limit the access or legal rights of the compilation's users
+ beyond what the individual works permit.  Inclusion of a covered work
+ in an aggregate does not cause this License to apply to the other
+ parts of the aggregate.
+ .
+   6. Conveying Non-Source Forms.
+ .
+   You may convey a covered work in object code form under the terms
+ of sections 4 and 5, provided that you also convey the
+ machine-readable Corresponding Source under the terms of this License,
+ in one of these ways:
+ .
+     a) Convey the object code in, or embodied in, a physical product
+     (including a physical distribution medium), accompanied by the
+     Corresponding Source fixed on a durable physical medium
+     customarily used for software interchange.
+ .
+     b) Convey the object code in, or embodied in, a physical product
+     (including a physical distribution medium), accompanied by a
+     written offer, valid for at least three years and valid for as
+     long as you offer spare parts or customer support for that product
+     model, to give anyone who possesses the object code either (1) a
+     copy of the Corresponding Source for all the software in the
+     product that is covered by this License, on a durable physical
+     medium customarily used for software interchange, for a price no
+     more than your reasonable cost of physically performing this
+     conveying of source, or (2) access to copy the
+     Corresponding Source from a network server at no charge.
+ .
+     c) Convey individual copies of the object code with a copy of the
+     written offer to provide the Corresponding Source.  This
+     alternative is allowed only occasionally and noncommercially, and
+     only if you received the object code with such an offer, in accord
+     with subsection 6b.
+ .
+     d) Convey the object code by offering access from a designated
+     place (gratis or for a charge), and offer equivalent access to the
+     Corresponding Source in the same way through the same place at no
+     further charge.  You need not require recipients to copy the
+     Corresponding Source along with the object code.  If the place to
+     copy the object code is a network server, the Corresponding Source
+     may be on a different server (operated by you or a third party)
+     that supports equivalent copying facilities, provided you maintain
+     clear directions next to the object code saying where to find the
+     Corresponding Source.  Regardless of what server hosts the
+     Corresponding Source, you remain obligated to ensure that it is
+     available for as long as needed to satisfy these requirements.
+ .
+     e) Convey the object code using peer-to-peer transmission, provided
+     you inform other peers where the object code and Corresponding
+     Source of the work are being offered to the general public at no
+     charge under subsection 6d.
+ .
+   A separable portion of the object code, whose source code is excluded
+ from the Corresponding Source as a System Library, need not be
+ included in conveying the object code work.
+ .
+   A "User Product" is either (1) a "consumer product", which means any
+ tangible personal property which is normally used for personal, family,
+ or household purposes, or (2) anything designed or sold for incorporation
+ into a dwelling.  In determining whether a product is a consumer product,
+ doubtful cases shall be resolved in favor of coverage.  For a particular
+ product received by a particular user, "normally used" refers to a
+ typical or common use of that class of product, regardless of the status
+ of the particular user or of the way in which the particular user
+ actually uses, or expects or is expected to use, the product.  A product
+ is a consumer product regardless of whether the product has substantial
+ commercial, industrial or non-consumer uses, unless such uses represent
+ the only significant mode of use of the product.
+ .
+   "Installation Information" for a User Product means any methods,
+ procedures, authorization keys, or other information required to install
+ and execute modified versions of a covered work in that User Product from
+ a modified version of its Corresponding Source.  The information must
+ suffice to ensure that the continued functioning of the modified object
+ code is in no case prevented or interfered with solely because
+ modification has been made.
+ .
+   If you convey an object code work under this section in, or with, or
+ specifically for use in, a User Product, and the conveying occurs as
+ part of a transaction in which the right of possession and use of the
+ User Product is transferred to the recipient in perpetuity or for a
+ fixed term (regardless of how the transaction is characterized), the
+ Corresponding Source conveyed under this section must be accompanied
+ by the Installation Information.  But this requirement does not apply
+ if neither you nor any third party retains the ability to install
+ modified object code on the User Product (for example, the work has
+ been installed in ROM).
+ .
+   The requirement to provide Installation Information does not include a
+ requirement to continue to provide support service, warranty, or updates
+ for a work that has been modified or installed by the recipient, or for
+ the User Product in which it has been modified or installed.  Access to a
+ network may be denied when the modification itself materially and
+ adversely affects the operation of the network or violates the rules and
+ protocols for communication across the network.
+ .
+   Corresponding Source conveyed, and Installation Information provided,
+ in accord with this section must be in a format that is publicly
+ documented (and with an implementation available to the public in
+ source code form), and must require no special password or key for
+ unpacking, reading or copying.
+ .
+   7. Additional Terms.
+ .
+   "Additional permissions" are terms that supplement the terms of this
+ License by making exceptions from one or more of its conditions.
+ Additional permissions that are applicable to the entire Program shall
+ be treated as though they were included in this License, to the extent
+ that they are valid under applicable law.  If additional permissions
+ apply only to part of the Program, that part may be used separately
+ under those permissions, but the entire Program remains governed by
+ this License without regard to the additional permissions.
+ .
+   When you convey a copy of a covered work, you may at your option
+ remove any additional permissions from that copy, or from any part of
+ it.  (Additional permissions may be written to require their own
+ removal in certain cases when you modify the work.)  You may place
+ additional permissions on material, added by you to a covered work,
+ for which you have or can give appropriate copyright permission.
+ .
+   Notwithstanding any other provision of this License, for material you
+ add to a covered work, you may (if authorized by the copyright holders of
+ that material) supplement the terms of this License with terms:
+ .
+     a) Disclaiming warranty or limiting liability differently from the
+     terms of sections 15 and 16 of this License; or
+ .
+     b) Requiring preservation of specified reasonable legal notices or
+     author attributions in that material or in the Appropriate Legal
+     Notices displayed by works containing it; or
+ .
+     c) Prohibiting misrepresentation of the origin of that material, or
+     requiring that modified versions of such material be marked in
+     reasonable ways as different from the original version; or
+ .
+     d) Limiting the use for publicity purposes of names of licensors or
+     authors of the material; or
+ .
+     e) Declining to grant rights under trademark law for use of some
+     trade names, trademarks, or service marks; or
+ .
+     f) Requiring indemnification of licensors and authors of that
+     material by anyone who conveys the material (or modified versions of
+     it) with contractual assumptions of liability to the recipient, for
+     any liability that these contractual assumptions directly impose on
+     those licensors and authors.
+ .
+   All other non-permissive additional terms are considered "further
+ restrictions" within the meaning of section 10.  If the Program as you
+ received it, or any part of it, contains a notice stating that it is
+ governed by this License along with a term that is a further
+ restriction, you may remove that term.  If a license document contains
+ a further restriction but permits relicensing or conveying under this
+ License, you may add to a covered work material governed by the terms
+ of that license document, provided that the further restriction does
+ not survive such relicensing or conveying.
+ .
+   If you add terms to a covered work in accord with this section, you
+ must place, in the relevant source files, a statement of the
+ additional terms that apply to those files, or a notice indicating
+ where to find the applicable terms.
+ .
+   Additional terms, permissive or non-permissive, may be stated in the
+ form of a separately written license, or stated as exceptions;
+ the above requirements apply either way.
+ .
+   8. Termination.
+ .
+   You may not propagate or modify a covered work except as expressly
+ provided under this License.  Any attempt otherwise to propagate or
+ modify it is void, and will automatically terminate your rights under
+ this License (including any patent licenses granted under the third
+ paragraph of section 11).
+ .
+   However, if you cease all violation of this License, then your
+ license from a particular copyright holder is reinstated (a)
+ provisionally, unless and until the copyright holder explicitly and
+ finally terminates your license, and (b) permanently, if the copyright
+ holder fails to notify you of the violation by some reasonable means
+ prior to 60 days after the cessation.
+ .
+   Moreover, your license from a particular copyright holder is
+ reinstated permanently if the copyright holder notifies you of the
+ violation by some reasonable means, this is the first time you have
+ received notice of violation of this License (for any work) from that
+ copyright holder, and you cure the violation prior to 30 days after
+ your receipt of the notice.
+ .
+   Termination of your rights under this section does not terminate the
+ licenses of parties who have received copies or rights from you under
+ this License.  If your rights have been terminated and not permanently
+ reinstated, you do not qualify to receive new licenses for the same
+ material under section 10.
+ .
+   9. Acceptance Not Required for Having Copies.
+ .
+   You are not required to accept this License in order to receive or
+ run a copy of the Program.  Ancillary propagation of a covered work
+ occurring solely as a consequence of using peer-to-peer transmission
+ to receive a copy likewise does not require acceptance.  However,
+ nothing other than this License grants you permission to propagate or
+ modify any covered work.  These actions infringe copyright if you do
+ not accept this License.  Therefore, by modifying or propagating a
+ covered work, you indicate your acceptance of this License to do so.
+ .
+   10. Automatic Licensing of Downstream Recipients.
+ .
+   Each time you convey a covered work, the recipient automatically
+ receives a license from the original licensors, to run, modify and
+ propagate that work, subject to this License.  You are not responsible
+ for enforcing compliance by third parties with this License.
+ .
+   An "entity transaction" is a transaction transferring control of an
+ organization, or substantially all assets of one, or subdividing an
+ organization, or merging organizations.  If propagation of a covered
+ work results from an entity transaction, each party to that
+ transaction who receives a copy of the work also receives whatever
+ licenses to the work the party's predecessor in interest had or could
+ give under the previous paragraph, plus a right to possession of the
+ Corresponding Source of the work from the predecessor in interest, if
+ the predecessor has it or can get it with reasonable efforts.
+ .
+   You may not impose any further restrictions on the exercise of the
+ rights granted or affirmed under this License.  For example, you may
+ not impose a license fee, royalty, or other charge for exercise of
+ rights granted under this License, and you may not initiate litigation
+ (including a cross-claim or counterclaim in a lawsuit) alleging that
+ any patent claim is infringed by making, using, selling, offering for
+ sale, or importing the Program or any portion of it.
+ .
+   11. Patents.
+ .
+   A "contributor" is a copyright holder who authorizes use under this
+ License of the Program or a work on which the Program is based.  The
+ work thus licensed is called the contributor's "contributor version".
+ .
+   A contributor's "essential patent claims" are all patent claims
+ owned or controlled by the contributor, whether already acquired or
+ hereafter acquired, that would be infringed by some manner, permitted
+ by this License, of making, using, or selling its contributor version,
+ but do not include claims that would be infringed only as a
+ consequence of further modification of the contributor version.  For
+ purposes of this definition, "control" includes the right to grant
+ patent sublicenses in a manner consistent with the requirements of
+ this License.
+ .
+   Each contributor grants you a non-exclusive, worldwide, royalty-free
+ patent license under the contributor's essential patent claims, to
+ make, use, sell, offer for sale, import and otherwise run, modify and
+ propagate the contents of its contributor version.
+ .
+   In the following three paragraphs, a "patent license" is any express
+ agreement or commitment, however denominated, not to enforce a patent
+ (such as an express permission to practice a patent or covenant not to
+ sue for patent infringement).  To "grant" such a patent license to a
+ party means to make such an agreement or commitment not to enforce a
+ patent against the party.
+ .
+   If you convey a covered work, knowingly relying on a patent license,
+ and the Corresponding Source of the work is not available for anyone
+ to copy, free of charge and under the terms of this License, through a
+ publicly available network server or other readily accessible means,
+ then you must either (1) cause the Corresponding Source to be so
+ available, or (2) arrange to deprive yourself of the benefit of the
+ patent license for this particular work, or (3) arrange, in a manner
+ consistent with the requirements of this License, to extend the patent
+ license to downstream recipients.  "Knowingly relying" means you have
+ actual knowledge that, but for the patent license, your conveying the
+ covered work in a country, or your recipient's use of the covered work
+ in a country, would infringe one or more identifiable patents in that
+ country that you have reason to believe are valid.
+ .
+   If, pursuant to or in connection with a single transaction or
+ arrangement, you convey, or propagate by procuring conveyance of, a
+ covered work, and grant a patent license to some of the parties
+ receiving the covered work authorizing them to use, propagate, modify
+ or convey a specific copy of the covered work, then the patent license
+ you grant is automatically extended to all recipients of the covered
+ work and works based on it.
+ .
+   A patent license is "discriminatory" if it does not include within
+ the scope of its coverage, prohibits the exercise of, or is
+ conditioned on the non-exercise of one or more of the rights that are
+ specifically granted under this License.  You may not convey a covered
+ work if you are a party to an arrangement with a third party that is
+ in the business of distributing software, under which you make payment
+ to the third party based on the extent of your activity of conveying
+ the work, and under which the third party grants, to any of the
+ parties who would receive the covered work from you, a discriminatory
+ patent license (a) in connection with copies of the covered work
+ conveyed by you (or copies made from those copies), or (b) primarily
+ for and in connection with specific products or compilations that
+ contain the covered work, unless you entered into that arrangement,
+ or that patent license was granted, prior to 28 March 2007.
+ .
+   Nothing in this License shall be construed as excluding or limiting
+ any implied license or other defenses to infringement that may
+ otherwise be available to you under applicable patent law.
+ .
+   12. No Surrender of Others' Freedom.
+ .
+   If conditions are imposed on you (whether by court order, agreement or
+ otherwise) that contradict the conditions of this License, they do not
+ excuse you from the conditions of this License.  If you cannot convey a
+ covered work so as to satisfy simultaneously your obligations under this
+ License and any other pertinent obligations, then as a consequence you may
+ not convey it at all.  For example, if you agree to terms that obligate you
+ to collect a royalty for further conveying from those to whom you convey
+ the Program, the only way you could satisfy both those terms and this
+ License would be to refrain entirely from conveying the Program.
+ .
+   13. Use with the GNU Affero General Public License.
+ .
+   Notwithstanding any other provision of this License, you have
+ permission to link or combine any covered work with a work licensed
+ under version 3 of the GNU Affero General Public License into a single
+ combined work, and to convey the resulting work.  The terms of this
+ License will continue to apply to the part which is the covered work,
+ but the special requirements of the GNU Affero General Public License,
+ section 13, concerning interaction through a network will apply to the
+ combination as such.
+ .
+   14. Revised Versions of this License.
+ .
+   The Free Software Foundation may publish revised and/or new versions of
+ the GNU General Public License from time to time.  Such new versions will
+ be similar in spirit to the present version, but may differ in detail to
+ address new problems or concerns.
+ .
+   Each version is given a distinguishing version number.  If the
+ Program specifies that a certain numbered version of the GNU General
+ Public License "or any later version" applies to it, you have the
+ option of following the terms and conditions either of that numbered
+ version or of any later version published by the Free Software
+ Foundation.  If the Program does not specify a version number of the
+ GNU General Public License, you may choose any version ever published
+ by the Free Software Foundation.
+ .
+   If the Program specifies that a proxy can decide which future
+ versions of the GNU General Public License can be used, that proxy's
+ public statement of acceptance of a version permanently authorizes you
+ to choose that version for the Program.
+ .
+   Later license versions may give you additional or different
+ permissions.  However, no additional obligations are imposed on any
+ author or copyright holder as a result of your choosing to follow a
+ later version.
+ .
+   15. Disclaimer of Warranty.
+ .
+   THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+ APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+ HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+ OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+ THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+ IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+ .
+   16. Limitation of Liability.
+ .
+   IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+ WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+ THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+ GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+ USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+ DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+ PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+ EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+ SUCH DAMAGES.
+ .
+   17. Interpretation of Sections 15 and 16.
+ .
+   If the disclaimer of warranty and limitation of liability provided
+ above cannot be given local legal effect according to their terms,
+ reviewing courts shall apply local law that most closely approximates
+ an absolute waiver of all civil liability in connection with the
+ Program, unless a warranty or assumption of liability accompanies a
+ copy of the Program in return for a fee.
+ .
+                      END OF TERMS AND CONDITIONS
+ .
+             How to Apply These Terms to Your New Programs
+ .
+   If you develop a new program, and you want it to be of the greatest
+ possible use to the public, the best way to achieve this is to make it
+ free software which everyone can redistribute and change under these terms.
+ .
+   To do so, attach the following notices to the program.  It is safest
+ to attach them to the start of each source file to most effectively
+ state the exclusion of warranty; and each file should have at least
+ the "copyright" line and a pointer to where the full notice is found.
+ .
+     <one line to give the program's name and a brief idea of what it does.>
+     Copyright (C) <year>  <name of author>
+ .
+     This program is free software: you can redistribute it and/or modify
+     it under the terms of the GNU General Public License as published by
+     the Free Software Foundation, either version 3 of the License, or
+     (at your option) any later version.
+ .
+     This program is distributed in the hope that it will be useful,
+     but WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+     GNU General Public License for more details.
+ .
+     You should have received a copy of the GNU General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ .
+ Also add information on how to contact you by electronic and paper mail.
+ .
+   If the program does terminal interaction, make it output a short
+ notice like this when it starts in an interactive mode:
+ .
+     <program>  Copyright (C) <year>  <name of author>
+     This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+     This is free software, and you are welcome to redistribute it
+     under certain conditions; type `show c' for details.
+ .
+ The hypothetical commands `show w' and `show c' should show the appropriate
+ parts of the General Public License.  Of course, your program's commands
+ might be different; for a GUI interface, you would use an "about box".
+ .
+   You should also get your employer (if you work as a programmer) or school,
+ if any, to sign a "copyright disclaimer" for the program, if necessary.
+ For more information on this, and how to apply and follow the GNU GPL, see
+ <http://www.gnu.org/licenses/>.
+ .
+   The GNU General Public License does not permit incorporating your program
+ into proprietary programs.  If your program is a subroutine library, you
+ may consider it more useful to permit linking proprietary applications with
+ the library.  If this is what you want to do, use the GNU Lesser General
+ Public License instead of this License.  But first, please read
+ <http://www.gnu.org/philosophy/why-not-lgpl.html>.

debian/meli.docs

@@ -0,0 +1,3 @@
+docs/meli.1
+docs/meli.conf.5
+docs/meli-themes.5

debian/patches/fix-prefix-for-debian.patch

@@ -0,0 +1,14 @@
+Description: Fix PREFIX env var in Makefile for use in Debian
+Author: Manos Pitsidianakis <epilys@nessuent.xyz>
+Last-Update: 2020-01-30
+--- a/Makefile
++++ b/Makefile
+@@ -18,7 +18,7 @@
+ # along with meli. If not, see <http://www.gnu.org/licenses/>.
+ 
+ # Options
+-PREFIX ?= /usr/local
++PREFIX ?= /usr
+ EXPANDED_PREFIX := `cd ${PREFIX} && pwd -P`
+ BINDIR ?= ${EXPANDED_PREFIX}/bin
+ MANDIR ?= ${EXPANDED_PREFIX}/share/man

debian/patches/series

@@ -0,0 +1 @@
+fix-prefix-for-debian.patch

debian/rules

@@ -0,0 +1,16 @@
+#!/usr/bin/make -f
+# You must remove unused comment lines for the released package.
+#export DH_VERBOSE = 1
+#export DEB_BUILD_MAINT_OPTIONS = hardening=+all
+#export DEB_CFLAGS_MAINT_APPEND  = -Wall -pedantic
+#export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed
+export MELI_FEATURES = cli-docs sqlite3
+
+%:
+	dh $@  --with quilt
+
+#override_dh_auto_install:
+#	dh_auto_install -- prefix=/usr
+
+#override_dh_install:
+#	dh_install --list-missing -X.pyc -X.pyo

debian/source/format

@@ -0,0 +1 @@
+3.0 (quilt)

debian/source/local-options

@@ -0,0 +1,2 @@
+#abort-on-upstream-changes
+unapply-patches

debug_printer/Cargo.toml

@@ -1,17 +0,0 @@
-[package]
-name = "debug_printer"
-version = "0.0.1"  #:version
-authors = ["Manos Pitsidianakis <el13635@mail.ntua.gr>"]
-workspace = ".."
-edition = "2018"
-
-[lib]
-name = "debugprinter"
-crate-type = ["dylib"]
-path = "src/lib.rs"
-
-
-[dependencies]
-libc = {version = "0.2.55", features = ["extra_traits",] }
-melib = { path = "../melib", version = "*" }
-ui = { path = "../ui", version = "*" }

debug_printer/src/lib.rs

@@ -1,44 +0,0 @@
-extern crate libc;
-extern crate melib;
-
-use melib::Envelope;
-use std::ffi::CString;
-use std::os::raw::c_char;
-
-#[no_mangle]
-pub extern "C" fn print_envelope(ptr: *const Envelope) -> *const c_char {
-    unsafe {
-        assert!(!ptr.is_null(), "Null pointer in print_envelope");
-        //println!("got addr {}", p as u64);
-        //unsafe { CString::new("blah".to_string()).unwrap().as_ptr() }
-        let s = CString::new(format!("{:?}", *ptr)).unwrap();
-        drop(ptr);
-        let p = s.as_ptr();
-        std::mem::forget(s);
-        p
-    }
-}
-
-#[no_mangle]
-pub extern "C" fn get_empty_envelope() -> *mut Envelope {
-    let mut ret = Envelope::default();
-    let ptr = std::ptr::NonNull::new(&mut ret as *mut Envelope)
-        .expect("Envelope::default() has a NULL pointer?");
-
-    let ptr = ptr.as_ptr();
-    std::mem::forget(ret);
-    ptr
-}
-
-#[no_mangle]
-pub extern "C" fn destroy_cstring(ptr: *mut c_char) {
-    unsafe {
-        let slice = CString::from_raw(ptr);
-        drop(slice);
-    }
-}
-
-#[no_mangle]
-pub extern "C" fn envelope_size() -> libc::size_t {
-    std::mem::size_of::<Envelope>()
-}

docs/meli-themes.5

@@ -0,0 +1,619 @@
+.\" meli - meli-themes.5
+.\"
+.\" Copyright 2017-2020 Manos Pitsidianakis
+.\"
+.\" This file is part of meli.
+.\"
+.\" meli is free software: you can redistribute it and/or modify
+.\" it under the terms of the GNU General Public License as published by
+.\" the Free Software Foundation, either version 3 of the License, or
+.\" (at your option) any later version.
+.\"
+.\" meli is distributed in the hope that it will be useful,
+.\" but WITHOUT ANY WARRANTY; without even the implied warranty of
+.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+.\" GNU General Public License for more details.
+.\"
+.\" You should have received a copy of the GNU General Public License
+.\" along with meli. If not, see <http://www.gnu.org/licenses/>.
+.\"
+.Dd January 23, 2020
+.Dt MELI-THEMES 5
+.Os
+.Sh NAME
+.Nm meli-themes
+.Nd themes for the
+.Nm meli
+mail client
+.Sh SYNOPSIS
+.Nm meli
+comes with two themes,
+.Ic dark
+(default) and
+.Ic light .
+.sp
+Custom themes are defined as lists of key-values in the configuration files:
+.Bl -bullet  -compact
+.It
+.Pa $XDG_CONFIG_HOME/meli/config.toml
+.It
+.Pa $XDG_CONFIG_HOME/meli/themes/*.toml
+.El
+.sp
+The application theme is defined in the configuration as follows:
+.Bd -literal
+[terminal]
+theme = "dark"
+.Ed
+.Sh DESCRIPTION
+Themes for
+.Nm meli
+are described in the configuration language TOML, as they are key-value tables defined in the TERMINAL section of the configuration file.
+Each key defines the semantic location of the theme attribute within the application.
+For example,
+.Ic mail.listing.compact.*
+keys are settings for the
+.Ic compact
+mail listing style.
+A setting contains three fields: fg for foreground color, bg for background color, and attrs for text attribute.
+.sp
+.Dl \&"widget.key.label\&" = { fg = \&"Default\&", bg = \&"Default\&", attrs = \&"Default\&" }
+.sp
+Each field contains a value, which may be either a color/attribute, a link (key name) or a valid alias.
+An alias is a string starting with the \&"\&$\&" character and must be declared in advance in the
+.Ic color_aliases
+or
+.Ic attr_aliases
+fields of a theme.
+An alias' value can be any valid value, including links and other aliases, as long as they are valid.
+In the case of a link the setting's real value depends on the value of the referred key.
+This allows for defaults within a group of associated values.
+Cyclic references in a theme results in an error:
+.sp
+.Dl spooky theme contains a cycle: fg:  mail.listing.compact.even -> mail.listing.compact.highlighted -> mail.listing.compact.odd -> mail.listing.compact.even
+.Pp
+Two themes are included by default, `light` and `dark`.
+.Sh EXAMPLES
+Specific settings from already defined themes can be overwritten:
+.Bd -literal
+[terminal]
+theme = "dark"
+.sp
+[terminal.themes.dark]
+"mail.sidebar_highlighted_account" = { bg = "#ff4529" }
+"mail.listing.attachment_flag" = { fg = "#ff4529" }
+"mail.view.headers" = { fg = "30" }
+"mail.view.body" = {fg = "HotPink3", bg = "LightSalmon1"}
+# Linked value keys can be whatever key:
+"mail.listing.compact.even_unseen" = { bg = "mail.sidebar_highlighted_account" }
+# Linked color value keys can optionally refer to another field:
+"mail.listing.compact.odd_unseen" = { bg = "mail.sidebar_highlighted_account.fg" }
+.sp
+# define new theme. Undefined settings will inherit from the default "dark" theme.
+[terminal.themes."hunter2"]
+color_aliases= { "Jebediah" = "#b4da55" }
+"mail.listing.tag_default" = { fg = "$Jebediah" }
+"mail.view.headers" = { fg = "White", bg = "Black" }
+.Ed
+.Sh CUSTOM THEMES
+Custom themes can be included in your configuration files or be saved independently in your
+.Pa $XDG_CONFIG_HOME/meli/themes/
+directory as TOML files.
+To start creating a theme right away, you can begin by editing the default theme keys and values:
+.sp
+.Dl meli --print-default-theme > ~/.config/meli/themes/new_theme.toml
+.sp
+.Pa new_theme.toml
+will now include all keys and values of the "dark" theme.
+.sp
+.Dl meli --print-loaded-themes
+.sp
+will print all loaded themes with the links resolved.
+.Sh VALID ATTRIBUTE VALUES
+Case-sensitive.
+.Bl -bullet -compact
+.It
+"Default"
+.It
+"Bold"
+.It
+"Dim"
+.It
+"Italics"
+.It
+"Underline"
+.It
+"Blink"
+.It
+"Reverse"
+.It
+"Hidden"
+.It
+Any combo of the above separated by a bitwise XOR "\&|" eg "Dim | Italics"
+.El
+.Sh VALID COLOR VALUES
+Color values are of type String with the following valid contents:
+.Bl -bullet -compact
+.It
+"Default" is the terminal default. (Case-sensitive)
+.It
+Hex triplet e.g. #FFFFFF for RGB colors.
+Three character shorthand is also valid, e.g. #09c → #0099cc (Case-insensitive)
+.It
+0-255 byte for 256 colors.
+.It
+.Xr xterm 1
+name but with some modifications (for a full table see COLOR NAMES addendum) (Case-sensitive)
+.El
+.Sh NO COLOR
+To completely disable ANSI colors, there are two options:
+.Bl -bullet -compact
+.It
+Set the
+.Ic use_color
+option (section
+.Ic terminal Ns
+) to false, which is true by default.
+.It
+The
+.Ev NO_COLOR
+environmental variable, when present (regardless of its value), prevents the addition of ANSI color.
+When the configuration value
+.Ic use_color
+is explicitly set to true by the user,
+.Ev NO_COLOR
+is ignored.
+.El
+.sp
+In this mode, cursor locations (i.e., currently selected entries/items) will use the "reverse video" ANSI attribute to invert the terminal's default foreground/background colors.
+.Sh VALID KEYS
+.Bl -bullet  -compact
+.It
+theme_default
+.It
+status.bar
+.It
+status.notification
+.It
+tab.focused
+.It
+tab.unfocused
+.It
+tab.bar
+.It
+widgets.list.header
+.It
+widgets.form.label
+.It
+widgets.form.field
+.It
+widgets.form.highlighted
+.It
+widgets.options.highlighted
+.It
+mail.sidebar
+.It
+mail.sidebar_divider
+.It
+mail.sidebar_unread_count
+.It
+mail.sidebar_index
+.It
+mail.sidebar_highlighted
+.It
+mail.sidebar_highlighted_unread_count
+.It
+mail.sidebar_highlighted_index
+.It
+mail.sidebar_highlighted_account
+.It
+mail.sidebar_highlighted_account_unread_count
+.It
+mail.sidebar_highlighted_account_index
+.It
+mail.listing.compact.even
+.It
+mail.listing.compact.odd
+.It
+mail.listing.compact.even_unseen
+.It
+mail.listing.compact.odd_unseen
+.It
+mail.listing.compact.even_selected
+.It
+mail.listing.compact.odd_selected
+.It
+mail.listing.compact.even_highlighted
+.It
+mail.listing.compact.odd_highlighted
+.It
+mail.listing.plain.even
+.It
+mail.listing.plain.odd
+.It
+mail.listing.plain.even_unseen
+.It
+mail.listing.plain.odd_unseen
+.It
+mail.listing.plain.even_selected
+.It
+mail.listing.plain.odd_selected
+.It
+mail.listing.plain.even_highlighted
+.It
+mail.listing.plain.odd_highlighted
+.It
+mail.listing.conversations
+.It
+mail.listing.conversations.subject
+.It
+mail.listing.conversations.from
+.It
+mail.listing.conversations.date
+.It
+mail.listing.conversations.padding
+.It
+mail.listing.conversations.unseen
+.It
+mail.listing.conversations.unseen_padding
+.It
+mail.listing.conversations.highlighted
+.It
+mail.listing.conversations.selected
+.It
+mail.view.headers
+.It
+mail.view.headers_names
+.It
+mail.view.headers_area
+.It
+mail.view.body
+.It
+mail.view.thread.indentation.a
+.It
+mail.view.thread.indentation.b
+.It
+mail.view.thread.indentation.c
+.It
+mail.view.thread.indentation.d
+.It
+mail.view.thread.indentation.e
+.It
+mail.view.thread.indentation.f
+.It
+mail.listing.attachment_flag
+.It
+mail.listing.thread_snooze_flag
+.It
+mail.listing.tag_default
+.It
+pager.highlight_search
+.It
+pager.highlight_search_current
+.El
+.Sh COLOR NAMES
+.TS
+allbox tab(:);
+lb|lb|l|lb|lb
+l  l|l|l  l.
+name ↓:byte:_:name:byte ↓
+Aqua:14:_:Black:0
+Aquamarine1:122:_:Maroon:1
+Aquamarine2:86:_:Green:2
+Aquamarine3:79:_:Olive:3
+Black:0:_:Navy:4
+Blue:12:_:Purple1:5
+Blue1:21:_:Teal:6
+Blue2:19:_:Silver:7
+Blue3:20:_:Grey:8
+BlueViolet:57:_:Red:9
+CadetBlue:72:_:Lime:10
+CadetBlue1:73:_:Yellow:11
+Chartreuse1:118:_:Blue:12
+Chartreuse2:112:_:Fuchsia:13
+Chartreuse3:82:_:Aqua:14
+Chartreuse4:70:_:White:15
+Chartreuse5:76:_:Grey0:16
+Chartreuse6:64:_:NavyBlue:17
+CornflowerBlue:69:_:DarkBlue:18
+Cornsilk1:230:_:Blue2:19
+Cyan1:51:_:Blue3:20
+Cyan2:50:_:Blue1:21
+Cyan3:43:_:DarkGreen:22
+DarkBlue:18:_:DeepSkyBlue5:23
+DarkCyan:36:_:DeepSkyBlue6:24
+DarkGoldenrod:136:_:DeepSkyBlue7:25
+DarkGreen:22:_:DodgerBlue3:26
+DarkKhaki:143:_:DodgerBlue2:27
+DarkMagenta:90:_:Green4:28
+DarkMagenta1:91:_:SpringGreen6:29
+.TE
+.TS
+allbox tab(:);
+lb|lb|l|lb|lb
+l  l|l|l  l.
+name ↓:byte:_:name:byte ↓
+DarkOliveGreen1:192:_:Turquoise4:30
+DarkOliveGreen2:155:_:DeepSkyBlue3:31
+DarkOliveGreen3:191:_:DeepSkyBlue4:32
+DarkOliveGreen4:107:_:DodgerBlue1:33
+DarkOliveGreen5:113:_:Green2:34
+DarkOliveGreen6:149:_:SpringGreen4:35
+DarkOrange:208:_:DarkCyan:36
+DarkOrange2:130:_:LightSeaGreen:37
+DarkOrange3:166:_:DeepSkyBlue2:38
+DarkRed:52:_:DeepSkyBlue1:39
+DarkRed2:88:_:Green3:40
+DarkSeaGreen:108:_:SpringGreen5:41
+DarkSeaGreen1:158:_:SpringGreen2:42
+DarkSeaGreen2:193:_:Cyan3:43
+DarkSeaGreen3:151:_:DarkTurquoise:44
+DarkSeaGreen4:157:_:Turquoise2:45
+DarkSeaGreen5:115:_:Green1:46
+DarkSeaGreen6:150:_:SpringGreen3:47
+DarkSeaGreen7:65:_:SpringGreen1:48
+DarkSeaGreen8:71:_:MediumSpringGreen:49
+DarkSlateGray1:123:_:Cyan2:50
+DarkSlateGray2:87:_:Cyan1:51
+DarkSlateGray3:116:_:DarkRed:52
+DarkTurquoise:44:_:DeepPink8:53
+DarkViolet:128:_:Purple4:54
+DarkViolet1:92:_:Purple5:55
+DeepPink1:199:_:Purple3:56
+DeepPink2:197:_:BlueViolet:57
+DeepPink3:198:_:Orange3:58
+DeepPink4:125:_:Grey37:59
+.TE
+.TS
+allbox tab(:);
+lb|lb|l|lb|lb
+l  l|l|l  l.
+name ↓:byte:_:name:byte ↓
+DeepPink6:162:_:MediumPurple6:60
+DeepPink7:89:_:SlateBlue2:61
+DeepPink8:53:_:SlateBlue3:62
+DeepPink9:161:_:RoyalBlue1:63
+DeepSkyBlue1:39:_:Chartreuse6:64
+DeepSkyBlue2:38:_:DarkSeaGreen7:65
+DeepSkyBlue3:31:_:PaleTurquoise4:66
+DeepSkyBlue4:32:_:SteelBlue:67
+DeepSkyBlue5:23:_:SteelBlue3:68
+DeepSkyBlue6:24:_:CornflowerBlue:69
+DeepSkyBlue7:25:_:Chartreuse4:70
+DodgerBlue1:33:_:DarkSeaGreen8:71
+DodgerBlue2:27:_:CadetBlue:72
+DodgerBlue3:26:_:CadetBlue1:73
+Fuchsia:13:_:SkyBlue3:74
+Gold1:220:_:SteelBlue1:75
+Gold2:142:_:Chartreuse5:76
+Gold3:178:_:PaleGreen4:77
+Green:2:_:SeaGreen4:78
+Green1:46:_:Aquamarine3:79
+Green2:34:_:MediumTurquoise:80
+Green3:40:_:SteelBlue2:81
+Green4:28:_:Chartreuse3:82
+GreenYellow:154:_:SeaGreen3:83
+Grey:8:_:SeaGreen1:84
+Grey0:16:_:SeaGreen2:85
+Grey100:231:_:Aquamarine2:86
+Grey11:234:_:DarkSlateGray2:87
+Grey15:235:_:DarkRed2:88
+Grey19:236:_:DeepPink7:89
+.TE
+.TS
+allbox tab(:);
+lb|lb|l|lb|lb
+l  l|l|l  l.
+name ↓:byte:_:name:byte ↓
+Grey23:237:_:DarkMagenta:90
+Grey27:238:_:DarkMagenta1:91
+Grey3:232:_:DarkViolet1:92
+Grey30:239:_:Purple2:93
+Grey35:240:_:Orange4:94
+Grey37:59:_:LightPink3:95
+Grey39:241:_:Plum4:96
+Grey42:242:_:MediumPurple4:97
+Grey46:243:_:MediumPurple5:98
+Grey50:244:_:SlateBlue1:99
+Grey53:102:_:Yellow4:100
+Grey54:245:_:Wheat4:101
+Grey58:246:_:Grey53:102
+Grey62:247:_:LightSlateGrey:103
+Grey63:139:_:MediumPurple:104
+Grey66:248:_:LightSlateBlue:105
+Grey69:145:_:Yellow5:106
+Grey7:233:_:DarkOliveGreen4:107
+Grey70:249:_:DarkSeaGreen:108
+Grey74:250:_:LightSkyBlue2:109
+Grey78:251:_:LightSkyBlue3:110
+Grey82:252:_:SkyBlue2:111
+Grey84:188:_:Chartreuse2:112
+Grey85:253:_:DarkOliveGreen5:113
+Grey89:254:_:PaleGreen3:114
+Grey93:255:_:DarkSeaGreen5:115
+Honeydew2:194:_:DarkSlateGray3:116
+HotPink:205:_:SkyBlue1:117
+HotPink1:206:_:Chartreuse1:118
+HotPink2:169:_:LightGreen:119
+.TE
+.TS
+allbox tab(:);
+lb|lb|l|lb|lb
+l  l|l|l  l.
+name ↓:byte:_:name:byte ↓
+HotPink3:132:_:LightGreen1:120
+HotPink4:168:_:PaleGreen1:121
+IndianRed:131:_:Aquamarine1:122
+IndianRed1:167:_:DarkSlateGray1:123
+IndianRed2:204:_:Red2:124
+IndianRed3:203:_:DeepPink4:125
+Khaki1:228:_:MediumVioletRed:126
+Khaki3:185:_:Magenta4:127
+LightCoral:210:_:DarkViolet:128
+LightCyan2:195:_:Purple:129
+LightCyan3:152:_:DarkOrange2:130
+LightGoldenrod1:227:_:IndianRed:131
+LightGoldenrod2:222:_:HotPink3:132
+LightGoldenrod3:179:_:MediumOrchid3:133
+LightGoldenrod4:221:_:MediumOrchid:134
+LightGoldenrod5:186:_:MediumPurple2:135
+LightGreen:119:_:DarkGoldenrod:136
+LightGreen1:120:_:LightSalmon2:137
+LightPink1:217:_:RosyBrown:138
+LightPink2:174:_:Grey63:139
+LightPink3:95:_:MediumPurple3:140
+LightSalmon1:216:_:MediumPurple1:141
+LightSalmon2:137:_:Gold2:142
+LightSalmon3:173:_:DarkKhaki:143
+LightSeaGreen:37:_:NavajoWhite3:144
+LightSkyBlue1:153:_:Grey69:145
+LightSkyBlue2:109:_:LightSteelBlue3:146
+LightSkyBlue3:110:_:LightSteelBlue:147
+LightSlateBlue:105:_:Yellow6:148
+LightSlateGrey:103:_:DarkOliveGreen6:149
+.TE
+.TS
+allbox tab(:);
+lb|lb|l|lb|lb
+l  l|l|l  l.
+name ↓:byte:_:name:byte ↓
+LightSteelBlue:147:_:DarkSeaGreen6:150
+LightSteelBlue1:189:_:DarkSeaGreen3:151
+LightSteelBlue3:146:_:LightCyan3:152
+LightYellow3:187:_:LightSkyBlue1:153
+Lime:10:_:GreenYellow:154
+Magenta1:201:_:DarkOliveGreen2:155
+Magenta2:165:_:PaleGreen2:156
+Magenta3:200:_:DarkSeaGreen4:157
+Magenta4:127:_:DarkSeaGreen1:158
+Magenta5:163:_:PaleTurquoise1:159
+Magenta6:164:_:Red3:160
+Maroon:1:_:DeepPink9:161
+MediumOrchid:134:_:DeepPink6:162
+MediumOrchid1:171:_:Magenta5:163
+MediumOrchid2:207:_:Magenta6:164
+MediumOrchid3:133:_:Magenta2:165
+MediumPurple:104:_:DarkOrange3:166
+MediumPurple1:141:_:IndianRed1:167
+MediumPurple2:135:_:HotPink4:168
+MediumPurple3:140:_:HotPink2:169
+MediumPurple4:97:_:Orchid:170
+MediumPurple5:98:_:MediumOrchid1:171
+MediumPurple6:60:_:Orange2:172
+MediumSpringGreen:49:_:LightSalmon3:173
+MediumTurquoise:80:_:LightPink2:174
+MediumVioletRed:126:_:Pink3:175
+MistyRose1:224:_:Plum3:176
+MistyRose3:181:_:Violet:177
+NavajoWhite1:223:_:Gold3:178
+NavajoWhite3:144:_:LightGoldenrod3:179
+.TE
+.TS
+allbox tab(:);
+lb|lb|l|lb|lb
+l  l|l|l  l.
+name ↓:byte:_:name:byte ↓
+Navy:4:_:Tan:180
+NavyBlue:17:_:MistyRose3:181
+Olive:3:_:Thistle3:182
+Orange1:214:_:Plum2:183
+Orange2:172:_:Yellow3:184
+Orange3:58:_:Khaki3:185
+Orange4:94:_:LightGoldenrod5:186
+OrangeRed1:202:_:LightYellow3:187
+Orchid:170:_:Grey84:188
+Orchid1:213:_:LightSteelBlue1:189
+Orchid2:212:_:Yellow2:190
+PaleGreen1:121:_:DarkOliveGreen3:191
+PaleGreen2:156:_:DarkOliveGreen1:192
+PaleGreen3:114:_:DarkSeaGreen2:193
+PaleGreen4:77:_:Honeydew2:194
+PaleTurquoise1:159:_:LightCyan2:195
+PaleTurquoise4:66:_:Red1:196
+PaleVioletRed1:211:_:DeepPink2:197
+Pink1:218:_:DeepPink3:198
+Pink3:175:_:DeepPink1:199
+Plum1:219:_:Magenta3:200
+Plum2:183:_:Magenta1:201
+Plum3:176:_:OrangeRed1:202
+Plum4:96:_:IndianRed3:203
+Purple:129:_:IndianRed2:204
+Purple1:5:_:HotPink:205
+Purple2:93:_:HotPink1:206
+Purple3:56:_:MediumOrchid2:207
+Purple4:54:_:DarkOrange:208
+Purple5:55:_:Salmon1:209
+.TE
+.TS
+allbox tab(:);
+lb|lb|l|lb|lb
+l  l|l|l  l.
+name ↓:byte:_:name:byte ↓
+Red:9:_:LightCoral:210
+Red1:196:_:PaleVioletRed1:211
+Red2:124:_:Orchid2:212
+Red3:160:_:Orchid1:213
+RosyBrown:138:_:Orange1:214
+RoyalBlue1:63:_:SandyBrown:215
+Salmon1:209:_:LightSalmon1:216
+SandyBrown:215:_:LightPink1:217
+SeaGreen1:84:_:Pink1:218
+SeaGreen2:85:_:Plum1:219
+SeaGreen3:83:_:Gold1:220
+SeaGreen4:78:_:LightGoldenrod4:221
+Silver:7:_:LightGoldenrod2:222
+SkyBlue1:117:_:NavajoWhite1:223
+SkyBlue2:111:_:MistyRose1:224
+SkyBlue3:74:_:Thistle1:225
+SlateBlue1:99:_:Yellow1:226
+SlateBlue2:61:_:LightGoldenrod1:227
+SlateBlue3:62:_:Khaki1:228
+SpringGreen1:48:_:Wheat1:229
+SpringGreen2:42:_:Cornsilk1:230
+SpringGreen3:47:_:Grey100:231
+SpringGreen4:35:_:Grey3:232
+SpringGreen5:41:_:Grey7:233
+SpringGreen6:29:_:Grey11:234
+SteelBlue:67:_:Grey15:235
+SteelBlue1:75:_:Grey19:236
+SteelBlue2:81:_:Grey23:237
+SteelBlue3:68:_:Grey27:238
+Tan:180:_:Grey30:239
+.TE
+.TS
+allbox tab(:);
+lb|lb|l|lb|lb
+l  l|l|l  l.
+name ↓:byte:_:name:byte ↓
+Teal:6:_:Grey35:240
+Thistle1:225:_:Grey39:241
+Thistle3:182:_:Grey42:242
+Turquoise2:45:_:Grey46:243
+Turquoise4:30:_:Grey50:244
+Violet:177:_:Grey54:245
+Wheat1:229:_:Grey58:246
+Wheat4:101:_:Grey62:247
+White:15:_:Grey66:248
+Yellow:11:_:Grey70:249
+Yellow1:226:_:Grey74:250
+Yellow2:190:_:Grey78:251
+Yellow3:184:_:Grey82:252
+Yellow4:100:_:Grey85:253
+Yellow5:106:_:Grey89:254
+Yellow6:148:_:Grey93:255
+.TE
+.Sh SEE ALSO
+.Xr meli 1 ,
+.Xr meli.conf 5
+.Sh CONFORMING TO
+TOML Standard v.0.5.0 https://toml.io/en/v0.5.0
+.sp
+https://no-color.org/
+.Sh AUTHORS
+Copyright 2017-2019
+.An Manos Pitsidianakis Aq epilys@nessuent.xyz
+Released under the GPL, version 3 or greater.
+This software carries no warranty of any kind.
+(See COPYING for full copyright and warranty notices.)
+.Pp
+.Aq https://meli.delivery

docs/meli.1

@@ -0,0 +1,599 @@
+.\" meli - meli.1
+.\"
+.\" Copyright 2017-2019 Manos Pitsidianakis
+.\"
+.\" This file is part of meli.
+.\"
+.\" meli is free software: you can redistribute it and/or modify
+.\" it under the terms of the GNU General Public License as published by
+.\" the Free Software Foundation, either version 3 of the License, or
+.\" (at your option) any later version.
+.\"
+.\" meli is distributed in the hope that it will be useful,
+.\" but WITHOUT ANY WARRANTY; without even the implied warranty of
+.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+.\" GNU General Public License for more details.
+.\"
+.\" You should have received a copy of the GNU General Public License
+.\" along with meli. If not, see <http://www.gnu.org/licenses/>.
+.\"
+.Dd July 29, 2019
+.Dt MELI 1
+.Os
+.Sh NAME
+.Nm meli
+.Nd Meli Mail User Agent. meli is the Greek word for honey
+.Sh SYNOPSIS
+.Nm
+.Op Fl -help | h
+.Op Fl -version | v
+.Op Fl -config Ar path
+.Bl -tag -width flag -offset indent
+.It Fl -help | h
+Show help message and exit.
+.It Fl -version | v
+Show version and exit.
+.It Fl -config Ar path
+Start meli with given configuration file.
+.It Cm create-config Op Ar path
+Create configuration file in
+.Pa path
+if given, or at
+.Pa $XDG_CONFIG_HOME/meli/config.toml
+.It Cm test-config Op Ar path
+Test a configuration file for syntax issues or missing options.
+.It Cm man Op Ar page
+Print documentation page and exit (Piping to a pager is recommended.)
+.It Cm print-default-theme
+Print default theme keys and values in TOML syntax, to be used as a blueprint.
+.It Cm print-loaded-themes
+Print all loaded themes in TOML syntax.
+.It Cm view
+View mail from input file.
+.El
+.Sh DESCRIPTION
+.Nm
+is a terminal mail client aiming for extensive and user-frendly configurability.
+.Bd -literal
+     ^^      .-=-=-=-.  ^^
+ ^^        (`-=-=-=-=-`)         ^^
+         (`-=-=-=-=-=-=-`)  ^^         ^^
+   ^^   (`-=-=-=-=-=-=-=-`)   ^^
+       ( `-=-=-=-(@)-=-=-` )      ^^
+       (`-=-=-=-=-=-=-=-=-`)  ^^
+       (`-=-=-=-=-=-=-=-=-`)          ^^
+       (`-=-=-=-=-=-=-=-=-`)
+^^     (`-=-=-=-=-=-=-=-=-`)  ^^
+   ^^   (`-=-=-=-=-=-=-=-`)          ^^
+         (`-=-=-=-=-=-=-`)  ^^
+  ^^       (`-=-=-=-=-`)
+            `-=-=-=-=-`          ^^
+.Ed
+.Sh STARTING WITH meli
+When launched for the first time,
+.Nm
+will search for its configuration directory,
+.Pa $XDG_CONFIG_HOME/meli/ Ns
+\&.
+If it doesn't exist, you will be asked if you want to create one and presented with a sample configuration file
+.Pq Pa $XDG_CONFIG_HOME/meli/config.toml
+that includes the basic settings required for setting up accounts allowing you to copy and edit right away.
+See
+.Xr meli.conf 5
+for the available configuration options.
+.Pp
+At any time, you may press
+.Cm \&?
+for a searchable list of all available actions and shortcuts, along with every possible setting and command that your version supports.
+.Pp
+The main visual navigation tool, the left-side sidebar may be toggled with
+.Cm `
+(shortcuts.listing:
+.Ic toggle_menu_visibility Ns
+).
+.Pp
+Each mailbox may be viewed in 4 modes:
+Plain views each mail individually, Threaded shows their thread relationship visually, Conversations collapses each thread of emails into a single entry, Compact shows one row per thread.
+.Pp
+If you're using a light color palette in your terminal, you should set
+.Em theme = "light"
+in the
+.Em terminal
+section of your configuration.
+See
+.Xr meli-themes 5
+for complete documentation on user themes.
+.Sh VIEWING MAIL
+Open attachments by typing their index in the attachments list and then
+.Cm a
+.Po
+shortcut
+.Ic open_attachment
+.Pc .
+.Nm
+will attempt to open text inside its pager, and other content via
+.Cm xdg-open Ns
+\&.
+Press
+.Cm m
+.Po
+shortcut
+.Ic open_mailcap
+.Pc
+instead to use the mailcap entry for the MIME type of the attachment, if any.
+See
+.Sx FILES
+for the location of the mailcap files and
+.Xr mailcap 5
+for their syntax.
+You can save individual attachments with the
+.Em COMMAND
+.Cm save-attachment Ar INDEX Ar path-to-file
+where
+.Ar INDEX
+is the attachment's index in the listing.
+If the zeroth index is provided, the entire message is saved.
+If the path provided is a directory, the message is saved as an eml file with its filename set to the messages message-id.
+.Sh SEARCH
+Each e-mail storage backend has a default search method assigned.
+.Em IMAP
+uses the SEARCH command,
+.Em notmuch
+uses libnotmuch and
+.Em Maildir/mbox
+performs a slow linear search.
+It is advised to use a search backend on
+.Em Maildir/mbox
+accounts.
+.Nm Ns
+, if built with sqlite3, includes the ability to perform full text search on the following fields:
+.Em From ,
+.Em To ,
+.Em Cc ,
+.Em Bcc ,
+.Em In-Reply-To ,
+.Em References ,
+.Em Subject
+and
+.Em Date .
+The message body (in plain text human readable form) and the flags can also be queried.
+To enable sqlite3 indexing for an account set
+.Em search_backend
+to
+.Em sqlite3
+in the configuration file and to create the sqlite3 index issue command
+.Cm index Ar ACCOUNT_NAME Ns \&.
+.sp
+To search in the message body type your keywords without any special formatting.
+To search in specific fields, prepend your search keyword with "field:" like so:
+.Pp
+.D1 subject:helloooo or subject:\&"call for help\&" or \&"You remind me today of a small, Mexican chihuahua.\&"
+.Pp
+.D1 not ((from:unrealistic and (to:complex or not "query")) or flags:seen,draft)
+.Pp
+.D1 alladdresses:mailing@example.com and cc:me@example.com
+.Pp
+Boolean operators are
+.Em or Ns
+,
+.Em and
+and
+.Em not
+.Po
+alias:
+.Em \&!
+.Pc
+String keywords with spaces must be quoted.
+Quotes should always be escaped.
+.sp
+.Sy Important Notice about IMAP/JMAP
+.sp
+To prevent downloading all your messages from your IMAP/JMAP server, don't set
+.Em search_backend
+to
+.Em sqlite3 Ns
+\&.
+.Nm
+will relay your queries to the IMAP server.
+Expect a delay between query and response.
+Sqlite3 on the contrary at reasonable mailbox sizes should have a non noticable delay.
+.Ss QUERY ABNF SYNTAX
+.Bl -bullet
+.It
+.Li query = \&"(\&" query \&")\&" | from | to | cc | bcc | alladdresses | subject | flags | has_attachments | query \&"or\&" query | query \&"and\&" query | not query
+.It
+.Li not = \&"not\&" | \&"!\&"
+.It
+.Li quoted = ALPHA / SP *(ALPHA / DIGIT / SP)
+.It
+.Li term = ALPHA *(ALPHA / DIGIT) | DQUOTE quoted DQUOTE
+.It
+.Li tagname = term
+.It
+.Li flagval = \&"passed\&" | \&"replied\&" | \&"seen\&" | \&"read\&" | \&"junk\&" | \&"trash\&" | \&"trashed\&" | \&"draft\&" | \&"flagged\&" | tagname
+.It
+.Li flagterm = flagval | flagval \&",\&" flagterm
+.It
+.Li from = \&"from:\&" term
+.It
+.Li to = \&"to:\&" term
+.It
+.Li cc = \&"cc:\&" term
+.It
+.Li bcc = \&"bcc:\&" term
+.It
+.Li alladdresses = \&"alladdresses:\&" term
+.It
+.Li subject = \&"subject:\&" term
+.It
+.Li flags = \&"flags:\&" flag | \&"tags:\&" flag | \&"is:\&" flag
+.El
+.Sh TAGS
+.Nm
+supports tagging in notmuch and IMAP/JMAP backends.
+Tags can be searched with the `tags:` or `flags:` prefix in a search query, and can be modified by
+.Cm tag add TAG
+and
+.Cm tag remove TAG
+(see
+.Xr meli.conf 5 TAGS Ns
+, settings
+.Ic colors
+and
+.Ic ignore_tags
+for how to set tag colors and tag visiblity)
+.Sh COMPOSING
+.Ss Opening the message Composer tab
+To create a new mail message, press
+.Cm m
+(shortcut
+.Ic new_mail Ns
+) while viewing a mailbox.
+To reply to a mail, press
+.Cm R
+.Po
+shortcut
+.Ic reply
+.Pc .
+Both these actions open the mail composer view in a new tab.
+.Ss Editing text
+.Bl -bullet -compact
+.It
+Edit the header fields by selecting with the arrow keys and pressing
+.Cm enter
+to enter
+.Em INSERT
+mode and
+.Cm Esc
+key to exit.
+.It
+At any time you may press
+.Cm e
+(shortcut
+.Ic edit_mail Ns
+) to launch your editor (see
+.Xr meli.conf 5 COMPOSING Ns
+, setting
+.Ic editor_command
+for how to select which editor to launch).
+.It
+Your editor can be used in
+.Nm Ns
+\&'s embed terminal emulator by setting
+.Ic embed
+to
+.Em true
+in your composing settings.
+.It
+When launched, your editor captures all input until it exits or stops.
+.It
+To stop your editor and return to
+.Nm
+press Ctrl-z and to resume editing press the
+.Ic edit_mail
+command again
+.Po
+default
+.Em e
+.Pc .
+.El
+.Ss Attachments
+Attachments may be handled with the
+.Cm add-attachment Ns
+,
+.Cm remove-attachment
+commands (see below).
+.Ss Sending
+Finally, pressing
+.Cm s
+(shortcut
+.Ic send_mail Ns
+) will send your message according to your settings
+.Po
+see
+.Xr meli.conf 5 COMPOSING Ns
+, setting
+.Ic send_mail
+.Pc Ns
+\&.
+With no Draft or Sent mailbox,
+.Nm
+tries first saving mail in your INBOX and then at any other mailbox.
+On complete failure to save your draft or sent message it will be saved in your
+.Em tmp
+directory instead and you will be notified of its location.
+.Ss Drafts
+To save your draft without sending it, issue
+.Em COMMAND
+.Cm close
+and select 'save as draft'.
+.sp
+To open a draft for further editing, select your draft in the mail listing and press
+.Ic edit_mail Ns
+\&.
+.Sh CONTACTS
+.Nm
+supports two kinds of contact backends:
+.sp
+.Bl -enum -compact -offset indent
+.It
+an internal format that gets saved under
+.Pa $XDG_DATA_HOME/meli/account_name/addressbook Ns
+\&.
+.It
+vCard files (v3, v4) through the
+.Ic vcard_folder
+option in the account section.
+The path defined as
+.Ic vcard_folder
+can hold multiple vCards per file.
+They are loaded read only.
+.El
+.sp
+See
+.Xr meli.conf 5 ACCOUNTS
+for the complete account configuration values.
+.Sh MODES
+.Bl -tag -compact -width 8n
+.It NORMAL
+is the default mode
+.It COMMAND
+commands are issued in
+.Em COMMAND
+mode, by default started with
+.Cm \&:
+and exited with
+.Cm Esc
+key.
+.It EMBED
+is the mode of the embed terminal emulator
+.It INSERT
+captures all input as text input, and is exited with
+.Cm Esc
+key.
+.El
+.Ss COMMAND Mode
+.Ss Mail listing commands
+.Bl -tag -width 36n
+.It Cm set Ar plain | threaded | compact | conversations
+set the way mailboxes are displayed
+.El
+.TS
+allbox tab(:);
+lb l.
+conversations:shows one entry per thread
+compact:shows one row per thread
+threaded:shows threads as a tree structure
+plain:shows one row per mail, regardless of threading
+.TE
+.Bl -tag -width 36n
+.It Cm sort Ar subject | date \  Ar asc | desc
+sort mail listing
+.It Cm subsort Ar subject | date \  Ar asc | desc
+sorts only the first level of replies.
+.It Cm go Ar n
+where
+.Ar n
+is a mailbox prefixed with the
+.Ar n
+number in the side menu for the current account
+.It Cm toggle thread_snooze
+don't issue notifications for thread under cursor in thread listing
+.It Cm search Ar STRING
+search mailbox with
+.Ar STRING
+query.
+Escape exits search results.
+.It Cm select Ar STRING
+select threads matching
+.Ar STRING
+query.
+.It Cm set seen, set unseen
+Set seen status of message.
+.It Cm import Ar FILEPATH Ar MAILBOX_PATH
+Import mail from file into given mailbox.
+.It Cm copyto, moveto Ar MAILBOX_PATH
+Copy or move to other mailbox.
+.It Cm copyto, moveto Ar ACCOUNT Ar MAILBOX_PATH
+Copy or move to another account's  mailbox.
+.It Cm delete
+Delete selected threads.
+.It Cm export-mbox Ar FILEPATH
+Export selected threads to mboxcl2 file.
+.It Cm create-mailbox Ar ACCOUNT Ar MAILBOX_PATH
+create mailbox with given path.
+Be careful with backends and separator sensitivity (eg IMAP)
+.It Cm subscribe-mailbox Ar ACCOUNT Ar MAILBOX_PATH
+subscribe to mailbox with given path
+.It Cm unsubscribe-mailbox Ar ACCOUNT Ar MAILBOX_PATH
+unsubscribe to mailbox with given path
+.It Cm rename-mailbox Ar ACCOUNT Ar MAILBOX_PATH_SRC Ar MAILBOX_PATH_DEST
+rename mailbox
+.It Cm delete-mailbox Ar ACCOUNT Ar MAILBOX_PATH
+deletes mailbox in the mail backend.
+This action is unreversible.
+.El
+.Ss Mail view commands
+.Bl -tag -width 36n
+.It Cm pipe Ar EXECUTABLE Ar ARGS
+pipe pager contents to binary
+.It Cm list-post
+post in list of viewed envelope
+.It Cm list-unsubscribe
+unsubscribe automatically from list of viewed envelope
+.It Cm list-archive
+open list archive with
+.Cm xdg-open
+.El
+.Ss composing mail commands
+.Bl -tag -width 36n
+.It Cm add-attachment Ar PATH
+in composer, add
+.Ar PATH
+as an attachment
+.It Cm add-attachment < Ar CMD Ar ARGS
+in composer, pipe
+.Ar CMD Ar ARGS
+output into an attachment
+.It Cm add-attachment-file-picker
+Launch command defined in the configuration value
+.Ic file_picker_command
+in
+.Xr meli.conf 5 TERMINAL
+.It Cm add-attachment-file-picker < Ar CMD Ar ARGS
+Launch command
+.Ar CMD Ar ARGS Ns
+\&.
+The command should print file paths in stderr, separated by NULL bytes.
+.It Cm remove-attachment Ar INDEX
+remove attachment with given index
+.It Cm toggle sign
+toggle between signing and not signing this message.
+If the gpg invocation fails then the mail won't be sent.
+See
+.Xr meli.conf 5 PGP
+for PGP configuration.
+.It Cm save-draft
+saves a copy of the draft in the Draft folder
+.El
+.Ss generic commands
+.Bl -tag -width 36n
+.It Cm open-in-tab
+opens envelope view in new tab
+.It Cm close
+closes closeable tabs
+.It Cm setenv Ar KEY=VALUE
+set environment variable
+.Ar KEY
+to
+.Ar VALUE
+.It Cm printenv Ar KEY
+print environment variable
+.Ar KEY
+.It Cm quit
+Quits
+.Nm Ns
+\&.
+.It Cm reload-config
+Reloads configuration but only if account configuration is unchanged.
+Useful if you want to reload some settings without restarting
+.Nm Ns
+\&.
+.El
+.Sh SHORTCUTS
+See
+.Xr meli.conf 5 SHORTCUTS
+for shortcuts and their default values.
+.Sh EXIT STATUS
+.Nm
+exits with 0 on a successful run.
+Other exit statuses are:
+.Bl -tag -width 5n
+.It 1
+catchall for general errors
+.It 101
+process panic
+.El
+.Sh ENVIRONMENT
+.Bl -tag -width "$XDG_CONFIG_HOME/meli/plugins/*" -offset indent
+.It Ev EDITOR
+Specifies the editor to use
+.It Ev MELI_CONFIG
+Override the configuration file
+.It Ev NO_COLOR
+When present (regardless of its value), prevents the addition of ANSI color.
+The configuration value
+.Ic use_color
+overrides this.
+.El
+.Sh FILES
+.Nm
+uses the following parts of the XDG standard:
+.Bl -tag -width "$XDG_CONFIG_HOME/meli/plugins/*" -offset indent
+.It Ev XDG_CONFIG_HOME
+defaults to
+.Pa ~/.config/
+.It Ev XDG_CACHE_HOME
+defaults to
+.Pa ~/.cache/
+.El
+.Pp
+and appropriates the following locations:
+.Bl -tag -width "$XDG_CONFIG_HOME/meli/plugins/*" -offset indent
+.It Pa $XDG_CONFIG_HOME/meli/
+User configuration directory
+.It Pa $XDG_CONFIG_HOME/meli/config.toml
+User configuration file, see
+.Xr meli.conf 5
+for its syntax and values.
+.It Pa $XDG_CONFIG_HOME/meli/hooks/*
+Reserved for event hooks.
+.It Pa $XDG_CONFIG_HOME/meli/plugins/*
+Reserved for plugin files.
+.It Pa $XDG_CACHE_HOME/meli/*
+Internal cached data used by meli.
+.It Pa $XDG_DATA_HOME/meli/*
+Internal data used by meli.
+.It Pa $XDG_DATA_HOME/meli/meli.log
+Operation log.
+.It Pa /tmp/meli/*
+Temporary files generated by
+.Nm Ns
+\&.
+.El
+.Pp
+Mailcap entries are searched for in the following files, in this order:
+.Pp
+.Bl -enum -compact -offset indent
+.It
+.Pa $XDG_CONFIG_HOME/meli/mailcap
+.It
+.Pa $XDG_CONFIG_HOME/.mailcap
+.It
+.Pa $HOME/.mailcap
+.It
+.Pa /etc/mailcap
+.It
+.Pa /usr/etc/mailcap
+.It
+.Pa /usr/local/etc/mailcap
+.El
+.Sh SEE ALSO
+.Xr meli.conf 5 ,
+.Xr meli-themes 5 ,
+.Xr xdg-open 1 ,
+.Xr mailcap 5
+.Sh CONFORMING TO
+XDG Standard
+.Aq https://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html Ns
+, maildir
+.Aq https://cr.yp.to/proto/maildir.html Ns
+, IMAPv4rev1 RFC3501, The JSON Meta Application Protocol (JMAP) RFC8620, The JSON Meta Application Protocol (JMAP) for Mail RFC8621.
+.Sh AUTHORS
+Copyright 2017-2019
+.An Manos Pitsidianakis Aq epilys@nessuent.xyz
+Released under the GPL, version 3 or greater.
+This software carries no warranty of any kind.
+(See COPYING for full copyright and warranty notices.)
+.Pp
+.Aq https://meli.delivery

docs/samples/sample-config.toml

@@ -0,0 +1,135 @@
+## Look into meli.conf(5) for all valid configuration options, their
+## descriptions and default values
+##
+## The syntax for including other configuration files is enclosed in `:
+##`include("account_one")`
+##`include("./account_two")`
+##`include("/home/absolute/path/to/shortcuts/config.toml")`
+##
+##
+## Setting up a Maildir account
+#[accounts.account-name]
+#root_mailbox = "/path/to/root/mailbox"
+#format = "Maildir"
+#index_style = "Conversations" # or [plain, threaded, compact]
+#identity="email@example.com"
+#display_name = "Name"
+#subscribed_mailboxes = ["INBOX", "INBOX/Sent", "INBOX/Drafts", "INBOX/Junk"]
+#
+## Set mailbox-specific settings
+#  [accounts.account-name.mailboxes]
+#  "INBOX" = { rename="Inbox" }
+#  "drafts" = { rename="Drafts" }
+#  "foobar-devel" = { ignore = true } # don't show notifications for this mailbox
+#
+## Setting up an mbox account
+#[accounts.mbox]
+#root_mailbox = "/var/mail/username"
+#format = "mbox"
+#index_style = "Compact"
+#identity="username@hostname.local"
+#
+## Setting up an IMAP account
+#[accounts."imap"]
+#root_mailbox = "INBOX"
+#format = "imap"
+#server_hostname="mail.example.com"
+#server_password="pha2hiLohs2eeeish2phaii1We3ood4chakaiv0hien2ahie3m"
+#server_username="username@example.com"
+#server_port="993" # imaps
+#server_port="143" # STARTTLS
+#use_starttls=true #optional
+#index_style = "Conversations"
+#identity = "username@example.com"
+#display_name = "Name Name"
+### match every mailbox:
+#subscribed_mailboxes = ["*" ]
+### match specific mailboxes:
+##subscribed_mailboxes = ["INBOX", "INBOX/Sent", "INBOX/Drafts", "INBOX/Junk"]
+#
+## Setting up an account for an already existing notmuch database
+#[accounts.notmuch]
+#root_mailbox = "/path/to/folder" # where .notmuch/ directory is located
+#format = "notmuch"
+#index_style = "conversations"
+#identity="username@example.com"
+#display_name = "Name Name"
+#  # notmuch mailboxes are virtual, they are defined by their alias and the notmuch query that corresponds to their content.
+#  [accounts.notmuch.mailboxes]
+#  "INBOX" = {  query="tag:inbox", subscribe = true }
+#  "Drafts" = {  query="tag:draft", subscribe = true }
+#  "Sent" = {  query="from:username@example.com from:username2@example.com", subscribe = true }
+#
+## Setting up a Gmail account
+#[accounts."gmail"]
+#root_mailbox = '[Gmail]'
+#format = "imap"
+#server_hostname='imap.gmail.com'
+#server_password="password"
+#server_username="username@gmail.com"
+#server_port="993"
+#index_style = "Conversations"
+#identity = "username@gmail.com"
+#display_name = "Name Name"
+### match every mailbox:
+#subscribed_mailboxes = ["*" ]
+#composing.send_mail = 'msmtp --read-recipients --read-envelope-from'
+### Gmail auto saves sent mail to Sent folder, so don't duplicate the effort:
+#composing.store_sent_mail = false
+#
+#
+#[pager]
+#filter = "COLUMNS=72 /usr/local/bin/pygmentize -l email"
+#pager_context = 0 # default, optional
+#headers_sticky = true # default, optional
+#
+#[notifications]
+#script = "notify-send"
+#xbiff_file_path = "path" # for use with xbiff(1)
+#play_sound = true # default, optional
+#sound_file = "path" # optional
+#
+###shortcuts
+#[shortcuts.composing]
+#edit_mail = 'e'
+#
+##Thread view defaults:
+#[shortcuts.compact-listing]
+#exit_thread = 'i'
+#
+#[shortcuts.contact-list]
+#create_contact = 'c'
+#edit_contact = 'e'
+#
+##Mail listing defaults
+#[shortcuts.listing]
+#prev_page = "PageUp"
+#next_page = "PageDown"
+#prev_mailbox = 'K'
+#next_mailbox = 'J'
+#prev_account = 'l'
+#next_account = 'h'
+#new_mail = 'm'
+#set_seen = 'n'
+#
+##Pager defaults
+#
+#[shortcuts.pager]
+#scroll_up = 'k'
+#scroll_down = 'j'
+#page_up = "PageUp"
+#page_down = "PageDown"
+#
+#[composing]
+##required for sending e-mail
+#send_mail = 'msmtp --read-recipients --read-envelope-from'
+##send_mail = { hostname = "smtp.example.com", port = 587, auth = { type = "auto", username = "user", password = { type = "command_eval", value = "gpg2 --no-tty -q -d ~/.passwords/user.gpg" } }, security = { type = "STARTTLS" } }
+#editor_command = 'vim +/^$' # optional, by default $EDITOR is used.
+#
+#
+#[pgp]
+#auto_sign = false # always sign sent messages
+#auto_verify_signatures = true # always verify signatures when reading signed e-mails
+#
+#[terminal]
+#theme = "dark" # or "light"

docs/samples/themes/nord.toml

@@ -0,0 +1,70 @@
+[terminal.themes.nord]
+"theme_default" = { fg = "$nord6", bg = "$nord0", attrs = "Default" }
+"mail.listing.compact.even" = { fg = "theme_default", bg = "$nord1", attrs = "theme_default" }
+"mail.listing.compact.odd" = { fg = "theme_default", bg = "$nord2", attrs = "theme_default" }
+"mail.listing.plain.even" = { fg = "theme_default", bg = "$nord1", attrs = "theme_default" }
+"mail.listing.plain.odd" = { fg = "theme_default", bg = "$nord2", attrs = "theme_default" }
+"mail.listing.compact.even_highlighted" = { fg = "$nord0", bg = "$focused_bg", attrs = "theme_default" }
+"mail.listing.compact.odd_highlighted" = { fg = "$nord0", bg = "$focused_bg", attrs = "theme_default" }
+"mail.listing.conversations.highlighted" = { fg = "$nord0", bg = "$focused_bg", attrs = "theme_default" }
+"mail.listing.conversations.selected" = { fg = "theme_default", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.conversations.subject" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.conversations.unseen" = { fg = "theme_default", bg = "$nord2", attrs = "theme_default" }
+"mail.listing.plain.even_highlighted" = { fg = "$nord0", bg = "$focused_bg", attrs = "theme_default" }
+"mail.listing.plain.odd_highlighted" = { fg = "$nord0", bg = "$focused_bg", attrs = "theme_default" }
+"mail.listing.tag_default" = { fg = "theme_default", bg = "$nord8", attrs = "theme_default" }
+"mail.sidebar_highlighted" = { fg = "$nord1", bg = "$focused_bg", attrs = "theme_default" }
+"mail.sidebar_highlighted_account" = { fg = "$nord5", bg = "$nord1", attrs = "theme_default" }
+"mail.sidebar_highlighted_account_name" = { fg = "mail.sidebar_highlighted_account", bg = "mail.sidebar_highlighted_account", attrs = "theme_default" }
+"mail.sidebar_highlighted_account_index" = { fg = "mail.sidebar_highlighted_account", bg = "mail.sidebar_highlighted_account", attrs = "theme_default" }
+"mail.sidebar_highlighted_account_unread_count" = { fg = "mail.sidebar_highlighted_account", bg = "mail.sidebar_highlighted_account", attrs = "theme_default" }
+"mail.sidebar_highlighted_index" = { fg = "mail.sidebar_index", bg = "mail.sidebar_highlighted", attrs = "theme_default" }
+"mail.sidebar_highlighted_unread_count" = { fg = "mail.sidebar_highlighted", bg = "mail.sidebar_highlighted", attrs = "theme_default" }
+"mail.sidebar" = { fg = "$nord5", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar_account_name" = { fg = "$nord5", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar_index" = { fg = "$nord1", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar_unread_count" = { fg = "$nord1", bg = "theme_default", attrs = "theme_default" }
+"mail.view.body" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.view.headers" = { fg = "$nord9", bg = "theme_default", attrs = "theme_default" }
+"mail.view.thread.indentation.a" = { fg = "theme_default", bg = "$nord11", attrs = "theme_default" }
+"mail.view.thread.indentation.b" = { fg = "theme_default", bg = "$nord12", attrs = "theme_default" }
+"mail.view.thread.indentation.c" = { fg = "theme_default", bg = "$nord13", attrs = "theme_default" }
+"mail.view.thread.indentation.d" = { fg = "theme_default", bg = "$nord14", attrs = "theme_default" }
+"mail.view.thread.indentation.e" = { fg = "theme_default", bg = "$nord15", attrs = "theme_default" }
+"mail.view.thread.indentation.f" = { fg = "theme_default", bg = "$nord13", attrs = "theme_default" }
+"pager.highlight_search" = { fg = "$nord5", bg = "$nord7", attrs = "Bold" }
+"pager.highlight_search_current" = { fg = "$nord7", bg = "$nord10", attrs = "Bold" }
+"status.bar" = { fg = "$nord5", bg = "$nord3", attrs = "theme_default" }
+"status.notification" = { fg = "$nord5", bg = "$nord3", attrs = "theme_default" }
+"tab.bar" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"tab.focused" = { fg = "$nord1", bg = "$focused_bg", attrs = "theme_default" }
+"tab.unfocused" = { fg = "$nord4", bg = "$unfocused_bg", attrs = "theme_default" }
+"widgets.form.field" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"widgets.form.highlighted" = { fg = "theme_default", bg = "$nord2", attrs = "theme_default" }
+"widgets.form.label" = { fg = "theme_default", bg = "theme_default", attrs = "Bold" }
+"widgets.list.header" = { fg = "theme_default", bg = "theme_default", attrs = "Bold" }
+"widgets.options.highlighted" = { fg = "theme_default", bg = "$nord2", attrs = "theme_default" }
+
+[terminal.themes.nord.color_aliases]
+nord0 = "#2e3440"
+nord1 = "#3b4252"
+nord2 = "#434c5e"
+nord3 = "#4c566a"
+# snow storm
+nord4 = "#d8dee9"
+nord5 = "#e5e9f0"
+nord6 = "#eceff4"
+# frost
+nord7 = "#8fbcbb"
+nord8 = "#88c0d0"
+nord9 = "#81a1c1"
+nord10 = "#5e81ac"
+# aurora
+nord11 = "#bf616a"
+nord12 = "#d08770"
+nord13 = "#ebcb8b"
+nord14 = "#a3be8c"
+nord15 = "#b48ead"
+# semantics
+focused_bg = "$nord8"
+unfocused_bg = "$nord3"

docs/samples/themes/orca.toml

@@ -0,0 +1,62 @@
+[terminal.themes.orca]
+color_aliases = { "neon_green" = "#6ef9d4",  "darkgrey" = "#4a4a4a",  "neon_purple" = "#df2f94"  }
+"theme_default" = { fg = "White", bg = "Black", attrs = "Default" }
+"mail.listing.attachment_flag" = { fg = "$neon_green", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.compact.even" = { fg = "$darkgrey", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.compact.even_highlighted" = { fg = "theme_default", bg = "Grey58", attrs = "theme_default" }
+"mail.listing.compact.even_selected" = { fg = "theme_default", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.compact.even_unseen" = { fg = "Black", bg = "Grey78", attrs = "theme_default" }
+"mail.listing.compact.odd" = { fg = "$darkgrey", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.compact.odd_highlighted" = { fg = "theme_default", bg = "Grey58", attrs = "theme_default" }
+"mail.listing.compact.odd_selected" = { fg = "theme_default", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.compact.odd_unseen" = { fg = "Black", bg = "Grey78", attrs = "theme_default" }
+"mail.listing.conversations" = { fg = "$darkgrey", bg = "theme_default", attrs = "Default" }
+"mail.listing.conversations.date" = { fg = "$neon_purple", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.conversations.from" = { fg = "$darkgrey", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.conversations.highlighted" = { fg = "theme_default", bg = "Grey58", attrs = "theme_default" }
+"mail.listing.conversations.padding" = { fg = "Black", bg = "Black", attrs = "theme_default" }
+"mail.listing.conversations.selected" = { fg = "theme_default", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.conversations.subject" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.conversations.unseen" = { fg = "Black", bg = "Grey78", attrs = "theme_default" }
+"mail.listing.conversations.unseen_padding" = { fg = "Black", bg = "Black", attrs = "theme_default" }
+"mail.listing.plain.even" = { fg = "theme_default", bg = "Grey19", attrs = "theme_default" }
+"mail.listing.plain.even_highlighted" = { fg = "theme_default", bg = "Grey58", attrs = "theme_default" }
+"mail.listing.plain.even_selected" = { fg = "theme_default", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.plain.even_unseen" = { fg = "Black", bg = "Grey78", attrs = "theme_default" }
+"mail.listing.plain.odd" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.plain.odd_highlighted" = { fg = "theme_default", bg = "Grey58", attrs = "theme_default" }
+"mail.listing.plain.odd_selected" = { fg = "theme_default", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.plain.odd_unseen" = { fg = "Black", bg = "Grey78", attrs = "theme_default" }
+"mail.listing.tag_default" = { fg = "Black", bg = "$neon_green", attrs = "theme_default" }
+"mail.listing.thread_snooze_flag" = { fg = "Red", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar" = { fg = "$darkgrey", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar_highlighted" = { fg = "Grey7", bg = "White", attrs = "theme_default" }
+"mail.sidebar_highlighted_account" = { fg = "$darkgrey", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar_highlighted_account_name" = { fg = "White", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar_account_name" = { fg = "$darkgrey", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar_highlighted_account_index" = { fg = "mail.sidebar_index", bg = "mail.sidebar_highlighted_account", attrs = "theme_default" }
+"mail.sidebar_highlighted_account_unread_count" = { fg = "mail.sidebar_unread_count", bg = "mail.sidebar_highlighted_account", attrs = "theme_default" }
+"mail.sidebar_highlighted_index" = { fg = "mail.sidebar_index", bg = "mail.sidebar_highlighted", attrs = "theme_default" }
+"mail.sidebar_highlighted_unread_count" = { fg = "mail.sidebar_highlighted", bg = "mail.sidebar_highlighted", attrs = "theme_default" }
+"mail.sidebar_index" = { fg = "Grey46", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar_unread_count" = { fg = "Grey46", bg = "theme_default", attrs = "theme_default" }
+"mail.view.body" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.view.headers" = { fg = "DodgerBlue1", bg = "theme_default", attrs = "theme_default" }
+"mail.view.thread.indentation.a" = { fg = "theme_default", bg = "#EC4436", attrs = "theme_default" }
+"mail.view.thread.indentation.b" = { fg = "theme_default", bg = "#D301F9", attrs = "theme_default" }
+"mail.view.thread.indentation.c" = { fg = "theme_default", bg = "#314EFB", attrs = "theme_default" }
+"mail.view.thread.indentation.d" = { fg = "theme_default", bg = "#068ACD", attrs = "theme_default" }
+"mail.view.thread.indentation.e" = { fg = "theme_default", bg = "#019589", attrs = "theme_default" }
+"mail.view.thread.indentation.f" = { fg = "theme_default", bg = "#68A033", attrs = "theme_default" }
+"pager.highlight_search" = { fg = "White", bg = "Teal", attrs = "Bold" }
+"pager.highlight_search_current" = { fg = "White", bg = "NavyBlue", attrs = "Bold" }
+"status.bar" = { fg = "White", bg = "Black", attrs = "theme_default" }
+"status.notification" = { fg = "Plum1", bg = "theme_default", attrs = "theme_default" }
+"tab.bar" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"tab.focused" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"tab.unfocused" = { fg = "$darkgrey", bg = "Black", attrs = "theme_default" }
+"widgets.form.field" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"widgets.form.highlighted" = { fg = "theme_default", bg = "Grey58", attrs = "theme_default" }
+"widgets.form.label" = { fg = "theme_default", bg = "theme_default", attrs = "Bold" }
+"widgets.list.header" = { fg = "Black", bg = "White", attrs = "Bold" }
+"widgets.options.highlighted" = { fg = "theme_default", bg = "Grey", attrs = "theme_default" }

docs/samples/themes/sail.toml

@@ -0,0 +1,69 @@
+[terminal.themes.sail]
+color_aliases = { "unseen_fg" = "theme_default", "unseen_bg" = "theme_default", "sea" = "#91C7FF", "dimmed_text" = "#afbec5", "dimmed_bg" = "Grey78", "header" = "#edeff1" }
+"theme_default" = { fg = "#37474f", bg = "White", attrs = "Default" }
+"mail.listing.attachment_flag" = { fg = "Blue", bg = "theme_default", attrs = "theme_default" }
+
+"mail.listing.compact.even" = { fg = "$dimmed_text", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.compact.even_highlighted" = { fg = "theme_default", bg = "$dimmed_bg", attrs = "theme_default" }
+"mail.listing.compact.even_selected" = { fg = "$dimmed_text", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.compact.even_unseen" = { fg = "$unseen_fg", bg = "$unseen_bg", attrs = "theme_default" }
+
+"mail.listing.compact.odd" = { fg = "$dimmed_text", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.compact.odd_highlighted" = { fg = "theme_default", bg = "$dimmed_bg", attrs = "theme_default" }
+"mail.listing.compact.odd_selected" = { fg = "$dimmed_text", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.compact.odd_unseen" = { fg = "$unseen_fg", bg = "$unseen_bg", attrs = "theme_default" }
+
+"mail.listing.plain.even" = { fg = "mail.listing.compact.even", bg = "mail.listing.compact.even", attrs = "theme_default" }
+"mail.listing.plain.even_highlighted" = { fg = "mail.listing.compact.even_highlighted", bg = "mail.listing.compact.even_highlighted", attrs = "theme_default" }
+"mail.listing.plain.even_selected" = { fg = "mail.listing.compact.even_selected", bg = "mail.listing.compact.even_selected", attrs = "theme_default" }
+"mail.listing.plain.even_unseen" = { fg = "mail.listing.compact.even_unseen", bg = "mail.listing.compact.even_unseen", attrs = "theme_default" }
+"mail.listing.plain.odd" = { fg = "mail.listing.compact.odd", bg = "mail.listing.compact.odd", attrs = "theme_default" }
+"mail.listing.plain.odd_highlighted" = { fg = "mail.listing.compact.odd_highlighted", bg = "mail.listing.compact.odd_highlighted", attrs = "theme_default" }
+"mail.listing.plain.odd_selected" = { fg = "mail.listing.compact.odd_selected", bg = "mail.listing.compact.odd_selected", attrs = "theme_default" }
+"mail.listing.plain.odd_unseen" = { fg = "mail.listing.compact.odd_unseen", bg = "mail.listing.compact.odd_unseen", attrs = "theme_default" }
+
+"mail.listing.conversations" = { fg = "$dimmed_text", bg = "theme_default", attrs = "Default" }
+"mail.listing.conversations.date" = { fg = "mail.listing.conversations", bg = "mail.listing.conversations", attrs = "theme_default" }
+"mail.listing.conversations.from" = { fg = "mail.listing.conversations", bg = "mail.listing.conversations", attrs = "theme_default" }
+"mail.listing.conversations.subject" = { fg = "mail.listing.conversations", bg = "mail.listing.conversations", attrs = "theme_default" }
+"mail.listing.conversations.highlighted" = { fg = "theme_default", bg = "$dimmed_bg", attrs = "theme_default" }
+"mail.listing.conversations.selected" = { fg = "theme_default", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.conversations.unseen" = { fg = "$unseen_fg", bg = "$unseen_bg", attrs = "theme_default" }
+
+"mail.listing.tag_default" = { fg = "Black", bg = "$dimmed_text", attrs = "theme_default" }
+"mail.listing.thread_snooze_flag" = { fg = "Red", bg = "theme_default", attrs = "theme_default" }
+
+"mail.sidebar" = { fg = "$dimmed_text", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar_highlighted" = { fg = "theme_default", bg = "$dimmed_bg", attrs = "theme_default" }
+"mail.sidebar_highlighted_account" = { fg = "$dimmed_text", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar_highlighted_account_name" = { fg = "theme_default", bg = "$header", attrs = "Bold" }
+"mail.sidebar_account_name" = { fg = "mail.sidebar", bg = "mail.sidebar", attrs = "theme_default" }
+"mail.sidebar_highlighted_account_index" = { fg = "mail.sidebar_index", bg = "mail.sidebar_highlighted_account", attrs = "theme_default" }
+"mail.sidebar_highlighted_account_unread_count" = { fg = "mail.sidebar_unread_count", bg = "mail.sidebar_highlighted_account", attrs = "theme_default" }
+"mail.sidebar_highlighted_index" = { fg = "mail.sidebar_index", bg = "mail.sidebar_highlighted", attrs = "theme_default" }
+"mail.sidebar_highlighted_unread_count" = { fg = "mail.sidebar_highlighted", bg = "mail.sidebar_highlighted", attrs = "theme_default" }
+"mail.sidebar_index" = { fg = "mail.sidebar", bg = "mail.sidebar", attrs = "theme_default" }
+"mail.sidebar_unread_count" = { fg = "$dimmed_text", bg = "theme_default", attrs = "theme_default" }
+
+"mail.view.body" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.view.headers" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.view.headers_names" = { fg = "theme_default", bg = "theme_default", attrs = "Bold" }
+"mail.view.thread.indentation.a" = { fg = "theme_default", bg = "#EC633D", attrs = "theme_default" }
+"mail.view.thread.indentation.b" = { fg = "theme_default", bg = "#D347F9", attrs = "theme_default" }
+"mail.view.thread.indentation.c" = { fg = "theme_default", bg = "#317EFB", attrs = "theme_default" }
+"mail.view.thread.indentation.d" = { fg = "theme_default", bg = "#06B8CD", attrs = "theme_default" }
+"mail.view.thread.indentation.e" = { fg = "theme_default", bg = "#93DDB6", attrs = "theme_default" }
+"mail.view.thread.indentation.f" = { fg = "theme_default", bg = "#68A033", attrs = "theme_default" }
+
+"pager.highlight_search" = { fg = "White", bg = "Teal", attrs = "Bold" }
+"pager.highlight_search_current" = { fg = "White", bg = "NavyBlue", attrs = "Bold" }
+"status.bar" = { fg = "theme_default", bg = "$sea", attrs = "theme_default" }
+"status.notification" = { fg = "Plum1", bg = "theme_default", attrs = "theme_default" }
+"tab.bar" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"tab.focused" = { fg = "theme_default", bg = "theme_default", attrs = "Bold" }
+"tab.unfocused" = { fg = "White", bg = "$sea", attrs = "Bold" }
+"widgets.form.field" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"widgets.form.highlighted" = { fg = "theme_default", bg = "$dimmed_bg", attrs = "theme_default" }
+"widgets.form.label" = { fg = "theme_default", bg = "theme_default", attrs = "Bold" }
+"widgets.list.header" = { fg = "Black", bg = "White", attrs = "Bold" }
+"widgets.options.highlighted" = { fg = "theme_default", bg = "$dimmed_bg", attrs = "theme_default" }

docs/samples/themes/spooky.toml

@@ -0,0 +1,44 @@
+[terminal.themes.spooky]
+"theme_default" = { fg = "#333", bg = "#fe9b13", attrs = "Default" }
+"mail.listing.attachment_flag" = { fg = "LightSlateGrey", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.compact.even" = { fg = "theme_default", bg = "#bf200e", attrs = "theme_default" }
+"mail.listing.compact.odd" = { fg = "theme_default", bg = "#fa4113", attrs = "theme_default" }
+"mail.listing.compact.even_highlighted" = { fg = "theme_default", bg = "Yellow6", attrs = "theme_default" }
+"mail.listing.compact.odd_highlighted" = { fg = "theme_default", bg = "Yellow6", attrs = "theme_default" }
+"mail.listing.compact.even_selected" = { fg = "theme_default", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.compact.odd_selected" = { fg = "theme_default", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.compact.even_unseen" = { fg = "Black", bg = "Orange3", attrs = "theme_default" }
+"mail.listing.compact.odd_unseen" = { fg = "Black", bg = "Orange3", attrs = "theme_default" }
+"mail.listing.conversations.date" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.conversations" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.conversations.from" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.conversations.highlighted" = { fg = "theme_default", bg = "Grey58", attrs = "theme_default" }
+"mail.listing.conversations.padding" = { fg = "Grey15", bg = "Grey15", attrs = "theme_default" }
+"mail.listing.conversations.selected" = { fg = "theme_default", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.conversations.subject" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.conversations.unseen" = { fg = "Black", bg = "Grey78", attrs = "theme_default" }
+"mail.listing.conversations.unseen_padding" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.plain.even" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.plain.odd" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.plain.even_unseen" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.plain.odd_unseen" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.plain.even_highlighted" = { fg = "theme_default", bg = "Yellow6", attrs = "theme_default" }
+"mail.listing.plain.odd_highlighted" = { fg = "theme_default", bg = "Yellow6", attrs = "theme_default" }
+"mail.listing.plain.even_selected" = { fg = "theme_default", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.plain.odd_selected" = { fg = "theme_default", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.thread_snooze_flag" = { fg = "Red", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar_highlighted_account" = { fg = "White", bg = "Orange3", attrs = "theme_default" }
+"mail.sidebar_highlighted_account_index" = { fg = "mail.sidebar_index", bg = "mail.sidebar_highlighted_account", attrs = "theme_default" }
+"mail.sidebar_highlighted_account_unread_count" = { fg = "mail.sidebar_unread_count", bg = "mail.sidebar_highlighted_account", attrs = "theme_default" }
+"mail.sidebar_highlighted" = { fg = "Grey7", bg = "White", attrs = "theme_default" }
+"mail.sidebar_highlighted_index" = { fg = "mail.sidebar_index", bg = "mail.sidebar_highlighted", attrs = "theme_default" }
+"mail.sidebar_highlighted_unread_count" = { fg = "mail.sidebar_highlighted", bg = "mail.sidebar_highlighted", attrs = "theme_default" }
+"mail.sidebar_index" = { fg = "Grey46", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar_unread_count" = { fg = "Grey46", bg = "theme_default", attrs = "theme_default" }
+"mail.view.body" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.view.headers" = { fg = "DodgerBlue1", bg = "theme_default", attrs = "theme_default" }
+"status.bar" = { fg = "White", bg = "#A21500", attrs = "theme_default" }
+"tab.bar" = { fg = "theme_default", bg = "#332300", attrs = "theme_default" }
+"tab.unfocused" = { fg = "theme_default", bg = "#A26F00", attrs = "theme_default" }
+"tab.focused" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }

docs/samples/themes/watermelon.toml

@@ -0,0 +1,48 @@
+[terminal.themes.watermelon]
+color_aliases = { "JewelGreen" = "#157241", "PinkLace" = "#FFD5FD", "TorchRed" = "#F50431", "ChelseaCucumber" = "#6CA94A", "ScreaminGreen" = "#8FFF52", "SunsetOrange" = "#f74b41", "Melon" = "#fdbcb4", "BlueStone" = "#005F5F", "HotPink" = "#FF74D7" }
+"theme_default" = { fg = "$TorchRed", bg = "$PinkLace", attrs = "Default" }
+"widgets.list.header" = { fg = "$PinkLace", bg = "$TorchRed", attrs = "Bold" }
+"mail.listing.attachment_flag" = { fg = "LightSlateGrey", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.tag_default" = { bg = "$Melon", attrs = "Bold" }
+"mail.listing.compact.even" = { fg = "White", bg = "$ChelseaCucumber", attrs = "Bold" }
+"mail.listing.compact.odd" = { fg = "$PinkLace", bg = "$JewelGreen", attrs = "theme_default" }
+"mail.listing.compact.even_unseen" = { fg = "$JewelGreen", bg = "$ScreaminGreen", attrs = "theme_default" }
+"mail.listing.compact.odd_unseen" = { fg = "$JewelGreen", bg = "$ScreaminGreen", attrs = "theme_default" }
+"mail.listing.compact.even_highlighted" = { fg = "$JewelGreen", bg = "$SunsetOrange", attrs = "theme_default" }
+"mail.listing.compact.odd_highlighted" = { fg = "$JewelGreen", bg = "$SunsetOrange", attrs = "theme_default" }
+"mail.listing.compact.even_selected" = { fg = "theme_default", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.compact.odd_selected" = { fg = "theme_default", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.conversations.date" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.conversations" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.conversations.from" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.conversations.highlighted" = { fg = "$JewelGreen", bg = "$SunsetOrange", attrs = "theme_default" }
+"mail.listing.conversations.padding" = { fg = "$TorchRed", bg = "Grey15", attrs = "theme_default" }
+"mail.listing.conversations.selected" = { fg = "theme_default", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.conversations.subject" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.conversations.unseen" = { fg = "Black", bg = "mail.listing.compact.even_unseen", attrs = "theme_default" }
+"mail.listing.conversations.unseen_padding" = { fg = "$BlueStone", bg = "mail.listing.conversations.unseen", attrs = "theme_default" }
+"mail.listing.plain.even" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.plain.odd" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.listing.plain.even_unseen" = { fg = "theme_default", bg = "mail.listing.compact.even_unseen", attrs = "theme_default" }
+"mail.listing.plain.odd_unseen" = { fg = "theme_default", bg = "mail.listing.compact.odd_unseen", attrs = "theme_default" }
+"mail.listing.plain.even_highlighted" = { fg = "$JewelGreen", bg = "$SunsetOrange", attrs = "theme_default" }
+"mail.listing.plain.odd_highlighted" = { fg = "$JewelGreen", bg = "$SunsetOrange", attrs = "theme_default" }
+"mail.listing.plain.even_selected" = { fg = "theme_default", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.plain.odd_selected" = { fg = "theme_default", bg = "LightCoral", attrs = "theme_default" }
+"mail.listing.thread_snooze_flag" = { fg = "Red", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar_highlighted_account" = { fg = "White", bg = "$TorchRed", attrs = "theme_default" }
+"mail.sidebar_highlighted_account_index" = { fg = "mail.sidebar_index", bg = "mail.sidebar_highlighted_account", attrs = "theme_default" }
+"mail.sidebar_highlighted_account_unread_count" = { fg = "mail.sidebar_unread_count", bg = "mail.sidebar_highlighted_account", attrs = "theme_default" }
+"mail.sidebar_highlighted" = { fg = "Grey7", bg = "White", attrs = "theme_default" }
+"mail.sidebar_highlighted_index" = { fg = "mail.sidebar_index", bg = "mail.sidebar_highlighted", attrs = "theme_default" }
+"mail.sidebar_highlighted_unread_count" = { fg = "mail.sidebar_highlighted", bg = "mail.sidebar_highlighted", attrs = "theme_default" }
+"mail.sidebar_index" = { fg = "Grey46", bg = "theme_default", attrs = "theme_default" }
+"mail.sidebar_unread_count" = { fg = "Grey46", bg = "theme_default", attrs = "theme_default" }
+"mail.view.body" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"mail.view.headers" = { fg = "DodgerBlue1", bg = "theme_default", attrs = "theme_default" }
+"status.bar" = { fg = "$PinkLace", bg = "$TorchRed", attrs = "theme_default" }
+"status.notification" = { fg = "theme_default", bg = "theme_default", attrs = "Default" }
+"tab.bar" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }
+"tab.unfocused" = { fg = "$PinkLace", bg = "$HotPink", attrs = "theme_default" }
+"tab.focused" = { fg = "theme_default", bg = "theme_default", attrs = "theme_default" }

fuzz/.gitignore

@@ -0,0 +1,4 @@
+
+target
+corpus
+artifacts

fuzz/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "melib-fuzz"
+version = "0.0.0"
+authors = ["Automatically generated"]
+publish = false
+edition = "2018"
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+libfuzzer-sys = "0.3"
+
+[dependencies.melib]
+path = "../melib"
+features = ["unicode_algorithms"]
+
+# Prevent this from interfering with workspaces
+[workspace]
+members = ["."]
+
+[[bin]]
+name = "envelope_parse"
+path = "fuzz_targets/envelope_parse.rs"

fuzz/envelope_tokens.dict

@@ -0,0 +1,25 @@
+","
+";"
+"<"
+">"
+"@"
+":"
+# tab character
+"\x09"
+# new line character
+"\x0A"
+"  "
+"Subject: "
+"Subject"
+"To"
+"To: "
+"Date"
+"Date: "
+"Message-Id"
+"Message-Id: "
+"From"
+"From: "
+"Cc"
+"Cc: "
+"Bcc"
+"Bcc: "

fuzz/fuzz_targets/envelope_parse.rs

@@ -0,0 +1,11 @@
+#![no_main]
+use libfuzzer_sys::fuzz_target;
+
+extern crate melib;
+
+use melib::Envelope;
+
+fuzz_target!(|data: &[u8]| {
+    // fuzzed code goes here
+    let _envelope = Envelope::from_bytes(data, None);
+});

meli.1

@@ -1,329 +0,0 @@
-.\" meli - meli.1
-.\"
-.\" Copyright 2017-2019 Manos Pitsidianakis
-.\"
-.\" This file is part of meli.
-.\"
-.\" meli is free software: you can redistribute it and/or modify
-.\" it under the terms of the GNU General Public License as published by
-.\" the Free Software Foundation, either version 3 of the License, or
-.\" (at your option) any later version.
-.\"
-.\" meli is distributed in the hope that it will be useful,
-.\" but WITHOUT ANY WARRANTY; without even the implied warranty of
-.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-.\" GNU General Public License for more details.
-.\"
-.\" You should have received a copy of the GNU General Public License
-.\" along with meli. If not, see <http://www.gnu.org/licenses/>.
-.\"
-.Dd July 29, 2019
-.Dt MELI 1
-.Os Linux
-.Sh NAME
-.Nm meli
-.Nd Meli Mail User Agent. meli is the Greek word for honey.
-.Sh SYNOPSIS
-.Nm meli
-.Op Fl -help | h
-.Op Fl -version | v
-.Op Fl -create-config Op Ar path
-.Op Fl -config Ar path
-.Sh DESCRIPTION
-Experimental terminal mail client
-.Bl -tag -width flag -offset indent
-.It Fl -help, h
-Show help message and exit.
-.It Fl -version, v
-Show version and exit.
-.It Fl -create-config Op Ar path
-Create configuration file in
-.Pa path
-if given, or at
-.Pa $XDG_CONFIG_HOME/meli/config
-.It Fl -config Ar path
-Start meli with given configuration file.
-.El
-.Sh STARTING WITH meli
-When launched for the first time, meli will search for its configuration directory,
-.Pa $XDG_CONFIG_HOME/meli/ Ns
-\&. If it doesn't exist, you will be asked if you want to create one along with a sample configuration. The sample configuration
-.Pa $XDG_CONFIG_HOME/meli/config
-includes comments with the basic settings required for setting up accounts allowing you to copy and edit right away. See
-.Xr meli.conf 5
-for the available configuration options.
-.Pp
-At any time, you can press
-.Cm \&?
-to show a searchable list of all available actions and shortcuts, along with every possible setting and command that your version supports.
-.Pp
-The main visual navigation tool is the left-side sidebar. The menu's visibility can be toggled (default shortcut
-.Cm ` Ns
-).
-.Pp
-The view into each folder has 4 modes: plain, threaded, conversations and compact. Plain views each mail indvidually, threaded shows their thread relationship visually, and conversations includes one entry per thread of emails (compact is one row per thread).
-.Pp
-If you're using a light color palette in your terminal, you can set
-.Em theme = "light"
-in the
-.Em terminal
-section of your configuration.
-.Bd -literal
-         ^^      .-=-=-=-.  ^^
-     ^^        (`-=-=-=-=-`)         ^^
-             (`-=-=-=-=-=-=-`)  ^^         ^^
-       ^^   (`-=-=-=-=-=-=-=-`)   ^^
-           ( `-=-=-=-(@)-=-=-` )      ^^
-           (`-=-=-=-=-=-=-=-=-`)  ^^
-           (`-=-=-=-=-=-=-=-=-`)          ^^
-           (`-=-=-=-=-=-=-=-=-`)
-    ^^     (`-=-=-=-=-=-=-=-=-`)  ^^
-       ^^   (`-=-=-=-=-=-=-=-`)          ^^
-             (`-=-=-=-=-=-=-`)  ^^
-      ^^       (`-=-=-=-=-`)
-                `-=-=-=-=-`          ^^
-.Ed
-.Sh COMPOSING
-To send mail, press
-.Cm m
-while viewing the appropriate account to open a new composing tab. To reply to a mail, press
-.Cm R Ns
-\&. You can edit some of the header fields from within the view, by selecting with the arrow keys and pressing
-.Cm enter
-to enter
-.Ar INSERT
-mode. At any time you can press
-.Cm e
-to launch your editor (see
-.Xr meli.conf 5
-.Em COMPOSING
-for how to select which editor to launch). Attachments can be handled with the
-.Em add-attachment Ns
-,
-.Em remove-attachment
-commands (see below). Finally, pressing
-.Ar s
-will send your message by piping it into a binary of your choosing (see
-.Xr meli.conf 5
-.Em COMPOSING Ns
-, setting
-.Em mailer_cmd Ns
-). To save your draft without sending it, issue command
-.Cm close
-and select 'save as draft'.
-.Pp
-If there is no Draft or Sent folder, meli tries first saving mail in your INBOX and then at any other folder. On complete failure to save your draft or sent message it will be saved in your
-.Em tmp
-directory instead and you will be notified of its location.
-.Pp
-To open a draft for editing later, select your draft in the mail listing and press
-.Cm e Ns
-\&.
-.Sh EXECUTE mode
-Commands are issued in EXECUTE mode, by default started with the space character and exited with Escape key.
-.Pp
-the following commands are valid in the mail listing context:
-.Bl -tag -width "rename-folder ACCOUNT FOLDER_PATH_SRC FOLDER_PATH_DEST"
-.It Ic set Ar plain | threaded | compact | conversations
-set the way mailboxes are displayed
-.Bl -tag -width "conversations" -compact
-.It Cm plain
-shows one row per mail, regardless of threading
-.It Cm threaded
-shows threads as a tree structure, with one row per thread entry
-.It Cm conversations
-shows one entry per thread
-.It Cm compact
-shows one row per thread
-.El
-.It Ic sort Ar subject | date \  Ar asc | desc
-sort mail listing
-.It Ic subsort Ar subject | date \  Ar asc | desc
-sorts only the first level of replies.
-.It Ic go Ar n
-where
-.Ar n
-is a mailbox prefixed with the
-.Ar n
-number in the side menu for the current account
-.It Ic toggle_thread_snooze
-don't issue notifications for thread under cursor in thread listing
-.It Ic filter Ar STRING
-filter mailbox with
-.Ar STRING
-key. Escape exits filter results
-.It Ic set read, set unread
-.It Ic create-folder Ar ACCOUNT Ar FOLDER_PATH
-create folder with given path. be careful with backends and separator sensitivity (eg IMAP)
-.It Ic subscribe-folder Ar ACCOUNT Ar FOLDER_PATH
-subscribe to folder with given path
-.It Ic unsubscribe-folder Ar ACCOUNT Ar FOLDER_PATH
-unsubscribe to folder with given path
-.It Ic rename-folder Ar ACCOUNT Ar FOLDER_PATH_SRC Ar FOLDER_PATH_DEST
-rename folder
-.It Ic delete-folder Ar ACCOUNT Ar FOLDER_PATH
-delete folder
-.El
-.Pp
-envelope view commands:
-.Bl -tag -width "rename-folder ACCOUNT FOLDER_PATH_SRC FOLDER_PATH_DEST" -offset indent
-.It Cm pipe Ar EXECUTABLE Ar ARGS
-pipe pager contents to binary
-.It Cm list-post
-post in list of currently viewed envelope
-.It Cm list-unsubscribe
-unsubscribe automatically from list of currently viewed envelope
-.It Cm list-archive
-open list archive with
-.Cm xdg-open
-.El
-.Pp
-composing mail commands:
-.Bl -tag -width "rename-folder ACCOUNT FOLDER_PATH_SRC FOLDER_PATH_DEST" -offset indent
-.It Ic add-attachment Ar PATH
-in composer, add
-.Ar PATH
-as an attachment
-.It Ic remove-attachment Ar INDEX
-remove attachment with given index
-.It Ic toggle sign
-toggle between signing and not signing this message. If the gpg invocation fails then the mail won't be sent.
-.El
-.Pp
-generic commands:
-.Bl -tag -width "rename-folder ACCOUNT FOLDER_PATH_SRC FOLDER_PATH_DEST" -offset indent
-.It Cm open-in-tab
-opens envelope view in new tab
-.It Ic close
-closes closeable tabs
-.It Cm setenv Ar KEY=VALUE
-set environment variable
-.Ar KEY
-to
-.Ar VALUE
-.It Cm printenv Ar KEY
-print environment variable
-.Ar KEY
-.El
-.Sh SHORTCUTS
-Non-complete list of shortcuts and their default values.
-.Bl -tag -width "rename-folder ACCOUNT FOLDER_PATH_SRC FOLDER_PATH_DEST" -offset indent
-.It Cm open_thread
-\&'\\n'
-.It Cm exit_thread
-\&'i'
-.It Cm create_contact
-\&'c'
-.It Cm edit_contact
-\&'e'
-.It Cm prev_page
-PageUp,
-.It Cm next_page
-PageDown
-.It Cm prev_folder
-\&'K'
-.It Cm next_folder
-\&'J'
-.It Cm prev_account
-\&'l'
-.It Cm next_account
-\&'h'
-.It Cm new_mail
-\&'m'
-.It Cm scroll_up
-\&'k'
-.It Cm scroll_down
-\&'j'
-.It Cm page_up
-PageUp
-.It Cm page_down
-PageDown
-.It Cm toggle-menu-visibility
-\&'`'
-.It Cm select
-\&'v'
-.El
-.Bl -tag -width "rename-folder ACCOUNT FOLDER_PATH_SRC FOLDER_PATH_DEST" -offset indent
-.It Cm `
-toggles hiding of sidebar in mail listings
-.It Cm \&?
-opens up a shortcut window that shows available actions in the current component you are using (eg mail listing, contact list, mail composing)
-.It Cm m
-starts a new mail composer
-.It Cm R
-replies to the currently viewed mail.
-.It Cm u
-displays numbers next to urls in the body text of an email and
-.Ar n Ns Cm g
-opens the
-.Ar n Ns
-th
-url with xdg-open
-.It Ar n Ns Cm a
-opens the
-.Ar n Ns
-th
-attachment.
-.It Cm v
-(un)selects mail entries in mail listings
-.El
-.Sh EXIT STATUS
-.Nm
-exits with 0 on a successful run. Other exit statuses are:
-.Bl -tag -width 2n
-.It 1
-catchall for general errors
-.El
-.Sh ENVIRONMENT
-.Bl -tag -width "$XDG_CONFIG_HOME/meli/plugins/*" -offset indent
-.It Ev EDITOR
-Specifies the editor to use
-.It Ev MELI_CONFIG
-Override the configuration file
-.El
-.Sh FILES
-meli uses the following parts of the XDG standard:
-.Bl -tag -width "$XDG_CONFIG_HOME/meli/plugins/*" -offset indent
-.It Ev XDG_CONFIG_HOME
-defaults to
-.Pa ~/.config/
-.It Ev XDG_CACHE_HOME
-defaults to
-.Pa ~/.cache/
-.El
-.Pp
-and appropriates the following locations:
-.Bl -tag -width "$XDG_CONFIG_HOME/meli/plugins/*" -offset indent
-.It Pa $XDG_CONFIG_HOME/meli/
-User configuration directory.
-.It Pa $XDG_CONFIG_HOME/meli/config
-User configuration file. See
-.Xr meli.conf 5
-for its syntax and values.
-.It Pa $XDG_CONFIG_HOME/meli/hooks/*
-Reserved for event hooks.
-.It Pa $XDG_CONFIG_HOME/meli/plugins/*
-Reserved for plugin files.
-.It Pa $XDG_CACHE_HOME/meli/*
-Internal cached data used by meli.
-.It Pa $XDG_DATA_HOME/meli/*
-Internal data used by meli.
-.It Pa $XDG_DATA_HOME/meli/meli.log
-Operation log.
-.It Pa /tmp/meli/*
-Temporary files generated by meli.
-.El
-.Sh SEE ALSO
-.Xr xdg-open 1 ,
-.Xr meli.conf 5
-.Sh CONFORMING TO
-XDG Standard
-.Aq https://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html Ns
-, maildir
-.Aq https://cr.yp.to/proto/maildir.html
-.Sh AUTHORS
-Copyright 2017-2019
-.An Manos Pitsidianakis Aq epilys@nessuent.xyz
-Released under the GPL, version 3 or greater. This software carries no warranty of any kind. (See COPYING for full copyright and warranty notices.)
-.Pp
-.Aq https://meli.delivery

meli.conf.5

@@ -1,338 +0,0 @@
-.\" meli - meli.1
-.\"
-.\" Copyright 2017-2019 Manos Pitsidianakis
-.\"
-.\" This file is part of meli.
-.\"
-.\" meli is free software: you can redistribute it and/or modify
-.\" it under the terms of the GNU General Public License as published by
-.\" the Free Software Foundation, either version 3 of the License, or
-.\" (at your option) any later version.
-.\"
-.\" meli is distributed in the hope that it will be useful,
-.\" but WITHOUT ANY WARRANTY; without even the implied warranty of
-.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-.\" GNU General Public License for more details.
-.\"
-.\" You should have received a copy of the GNU General Public License
-.\" along with meli. If not, see <http://www.gnu.org/licenses/>.
-.\"
-.Dd September 16, 2019
-.Dt MELI.CONF 5
-.Os Linux
-.Sh NAME
-.Nm meli.conf
-.Nd configuration file for the Meli Mail User Agent
-.Sh SYNOPSIS
-.Pa $XDG_CONFIG_HOME/meli/config
-.Sh DESCRIPTION
-Configuration for meli is written in TOML. Few things to consider before writing TOML (quoting the spec):
-.Pp
-.Bl -bullet -compact
-.It
-TOML is case sensitive.
-.It
-A TOML file must be a valid UTF-8 encoded Unicode document.
-.It
-Whitespace means tab (0x09) or space (0x20).
-.It
-Newline means LF (0x0A) or CRLF (0x0D 0x0A).
-.El
-.Pp
-Refer to TOML documentation for valid TOML syntax.
-.Sh SECTIONS
-The top level sections of the config are accounts, shortcuts, notifications, pager, composing, pgp, terminal.
-.Pp
-.Sy example configuration
-.Bd -literal
-# Setting up a Maildir account
-[accounts.account-name]
-root_folder = "/path/to/root/folder"
-format = "Maildir"
-index_style = "Compact"
-identity="email@address.tld"
-subscribed_folders = ["folder", "folder/Sent"]
-display_name = "Name"
-
-# Set folder-specific settings
-  [accounts.account-name.folders]
-  "INBOX" = { rename="Inbox" } #inline table
-  "drafts" = { rename="Drafts" } #inline table
-  [accounts.account-name.folders."foobar-devel"] # or a regular table
-    ignore = true # don't show notifications for this folder
-
-# Setting up an mbox account
-[accounts.mbox]
-root_folder = "/var/mail/username"
-format = "mbox"
-index_style = "Compact"
-identity="username@hostname.local"
-
-[pager]
-filter = "/usr/bin/pygmentize"
-html_filter = "w3m -I utf-8 -T text/html"
-
-[notifications]
-script = "notify-send"
-
-[composing]
-# required for sending e-mail
-mailer_cmd = 'msmtp --read-recipients --read-envelope-from'
-editor_cmd = 'vim +/^$'
-
-[shortcuts]
-scroll_up = 'k'
-scroll_down = 'j'
-page_up = PageUp
-page_down = PageDown
-
-[terminal]
-theme = "light"
-.Ed
-.Pp
-available options are listed below.
-.Sy default values are shown in parentheses.
-.Sh ACCOUNTS
-.Bl -tag -width "danger_accept_invalid_certs boolean" -offset -indent
-.It Cm root_folder Ar String
-the backend-specific path of the root_folder, usually INBOX
-.It Cm format Ar String Op maildir mbox imap
-the format of the mail backend.
-.It Cm subscribed_folders Ar [String,]
-an array of folder paths to display in the UI. Paths are relative to the root folder (eg "INBOX/Sent", not "Sent")
-.It Cm identity Ar String
-your e-mail address that is inserted in the From: headers of outgoing mail
-.It Cm index_style Ar String
-set the way mailboxes are displayed
-.Bl -tag -width "conversations" -compact
-.It Cm plain
-shows one row per mail, regardless of threading
-.It Cm threaded
-shows threads as a tree structure, with one row per thread entry
-.It Cm conversations
-shows one entry per thread
-.It Cm compact
-shows one row per thread
-.El
-.It Cm display_name Ar String
-(optional) a name which can be combined with your address:
-"Name <email@address.tld>"
-.It Cm read_only Ar boolean
-attempt to not make any changes to this account.
-.Pq Em false
-.It Cm folders Ar folder_config
-(optional) configuration for each folder. Its format is described below in
-.Sx FOLDERS Ns
-\&.
-.El
-.Pp
-IMAP specific options are:
-.Bl -tag -width "danger_accept_invalid_certs boolean" -offset -indent
-.It Cm server_hostname Ar String
-example:
-.Qq mail.example.tld
-.It Cm server_username Ar String
-.It Cm server_password Ar String
-.It Cm server_port Ar number
-(optional)
-.\" default value
-.Pq Em 143
-.It Cm use_starttls Ar boolean
-(optional) if port is 993 and use_starttls is unspecified, it becomes false by default.
-.\" default value
-.Pq Em true
-.It Cm danger_accept_invalid_certs Ar boolean
-(optional) do not validate TLS certificates.
-.\" default value
-.Pq Em false
-.El
-.Sh FOLDERS
-.Bl -tag -width "danger_accept_invalid_certs boolean" -offset -indent
-.It Cm rename Ar String
-(optional) show a different name for this folder in the UI
-.It Cm autoload Ar boolean
-(optional) load this folder on startup (not functional yet)
-.It Cm subscribe Ar boolean
-(optional) watch this folder for updates
-.\" default value
-.Pq Em true
-.It Cm ignore Ar boolean
-(optional) silently insert updates for this folder, if any
-.\" default value
-.Pq Em false
-.It Cm usage Ar boolean
-(optional) special usage of this folder. valid values are:
-.Bl -bullet -compact
-.It
-.Ar Normal
-.It
-.Ar Inbox
-.It
-.Ar Archive
-.It
-.Ar Drafts
-.It
-.Ar Flagged
-.It
-.Ar Junk
-.It
-.Ar Sent
-.It
-.Ar Trash
-.El
-otherwise usage is inferred from the folder title.
-.It Cm conf_override Ar boolean
-(optional) override global settings for this folder. available sections to override are
-.Em pager, notifications, shortcuts, composing
-and the account options
-.Em identity and index_style Ns
-\&. example:
-.Bd -literal
-[accounts."imap.domain.tld".folders."INBOX"]
-  index_style = "plain"
-  [accounts."imap.domain.tld".folders."INBOX".pager]
-    filter = ""
-.Ed
-.El
-.Sh COMPOSING
-.Bl -tag -width "danger_accept_invalid_certs boolean" -offset -indent
-.It Cm mailer_cmd Ar String
-command to pipe new mail to, exit code must be 0 for success.
-.It Cm editor_cmd Ar String
-command to launch editor. Can have arguments. Draft filename is given as the last argument. If it's missing, the environment variable $EDITOR is looked up.
-.El
-.Sh SHORTCUTS
-Shortcuts can take the following values:
-.Qq Em Backspace
-.Qq Em Left
-.Qq Em Right
-.Qq Em Up
-.Qq Em Down
-.Qq Em Home
-.Qq Em End
-.Qq Em PageUp
-.Qq Em PageDown
-.Qq Em Delete
-.Qq Em Insert
-.Qq Em Esc
-and
-.Qq Em char Ns
-, where char is a single character string.
-.Bl -tag -width "danger_accept_invalid_certs boolean" -offset -indent
-.It Cm prev_page
-Go to previous page.
-.It Cm next_page
-Go to next page.
-.It Cm prev_folder
-Go to previous folder.
-.It Cm next_folder
-Go to next folder.
-.It Cm prev_account
-Go to previous account.
-.It Cm next_account
-Go to next account.
-.It Cm new_mail
-Start new mail draft in new tab
-.It Cm open_thread
-Open thread.
-.It Cm exit_thread
-Exit thread view
-.It Cm scroll_up
-Scroll up pager.
-.It Cm scroll_down
-Scroll down pager.
-.It Cm page_up
-Go to previous pager page
-.It Cm page_down
-Go to next pager pag
-.It Cm create_contact
-Create new contact.
-.It Cm edit_contact
-Edit contact under cursor
-.El
-.Sh NOTIFICATIONS
-.Bl -tag -width "danger_accept_invalid_certs boolean" -offset -indent
-.It Cm enable Ar boolean
-enable freedesktop-spec notifications. this is usually what you want
-.\" default value
-.Pq Em true
-.It Cm script Ar String
-(optional) script to pass notifications to, with title as 1st arg and body as 2nd
-.\" default value
-.Pq Em none
-.It Cm xbiff_file_path Ar String
-(optional) file that gets its size updated when new mail arrives
-.Pq Em none
-.\" default value
-.It Cm play_sound Ar boolean
-(optional) play theme sound in notifications if possible
-.Pq Em false
-.\" default value
-.It Cm sound_file Ar String
-(optional) play sound file in notifications if possible
-.\" default value
-.Pq Em none
-.El
-.Sh PAGER
-.Bl -tag -width "danger_accept_invalid_certs boolean" -offset -indent
-.It Cm pager_context Ar num
-(optional) number of context lines when going to next page.
-.\" default value
-.Pq Em 0
-.It Cm headers_sticky Ar boolean
-(optional) always show headers when scrolling.
-.\" default value
-.Pq Em false
-.It Cm html_filter Ar String
-(optional) pipe html attachments through this filter before display
-.\" default value
-.Pq Em none
-.It Cm filter Ar String
-(optional) a command to pipe mail output through for viewing in pager.
-.\" default value
-.Pq Em none
-.El
-.Sh PGP
-.Bl -tag -width "danger_accept_invalid_certs boolean" -offset -indent
-.It Cm auto_verify_signatures Ar boolean
-auto verify signed e-mail according to RFC3156
-.\" default value
-.Pq Em true
-.It Cm auto_sign Ar boolean
-(optional) always sign sent messages
-.\" default value
-.Pq Em false
-.It Cm key Ar String
-(optional) key to be used when signing/encrypting (not functional yet)
-.\" default value
-.Pq Em none
-.It Cm gpg_binary Ar String
-(optional) gpg binary name or file location to use
-.\" default value
-.Pq Em "gpg2"
-.El
-.Sh TERMINAL
-.Bl -tag -width "danger_accept_invalid_certs boolean" -offset -indent
-.It Cm theme Ar String
-(optional) select between these themes: light / dark
-.\" default value
-.Pq Em dark
-.It Cm ascii_drawing Ar boolean
-(optional) if true, box drawing will be done with ascii characters.
-.\" default value
-.Pq Em false
-.It Cm window_title Ar String
-(optional) set window title in xterm compatible terminals (empty string means no window title is set)
-.\" default value
-.Pq Em "meli"
-.El
-.Sh SEE ALSO
-.Xr meli 1
-.Sh CONFORMING TO
-TOML Standard v.0.5.0 https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.5.0.md
-.Sh AUTHORS
-Copyright 2017-2019
-.An Manos Pitsidianakis Aq epilys@nessuent.xyz
-Released under the GPL, version 3 or greater. This software carries no warranty of any kind. (See COPYING for full copyright and warranty notices.)
-.Pp
-.Aq https://meli.delivery

melib/Cargo.toml

@@ -1,37 +1,69 @@
 [package]
 name = "melib"
-version = "0.3.2"
-authors = ["Manos Pitsidianakis <el13635@mail.ntua.gr>"]
+version = "0.6.2"
+authors = ["Manos Pitsidianakis <epilys@nessuent.xyz>"]
 workspace = ".."
 edition = "2018"
+build = "build.rs"
+
+homepage = "https://meli.delivery"
+repository = "https://git.meli.delivery/meli/meli.git"
+description = "mail library"
+keywords = ["mail", "mua", "maildir", "imap", "jmap"]
+categories = [ "email", "parser-implementations"]
+license = "GPL-3.0-or-later"
+readme = "README.md"
+
+[lib]
+name = "melib"
+path = "src/lib.rs"
 
 [dependencies]
 bitflags = "1.0"
-chrono = { version = "0.4", features = ["serde"] }
-crossbeam = "0.7.2"
 data-encoding = "2.1.1"
 encoding = "0.2.33"
-fnv = "1.0.3"
-memmap = { version = "0.5.2", optional = true }
-nom = "3.2.0"
-notify = { version = "4.0.1", optional = true }
-notify-rust = { version = "^3", optional = true }
-termion = "1.5.1"
+nom = { version = "5.1.1" }
+
+indexmap = { version = "^1.5", features = ["serde-1", ] }
+notify = { version = "4.0.15", optional = true }
 xdg = "2.1.0"
-native-tls = { version ="0.2", optional=true }
-serde = "1.0.71"
+native-tls = { version ="0.2.3", optional=true }
+serde = { version = "1.0.71", features = ["rc", ] }
 serde_derive = "1.0.71"
-bincode = "1.2.0"
-uuid = { version = "0.7.4", features = ["serde", "v4"] }
-text_processing = { path = "../text_processing", version = "*", optional= true }
+bincode = "^1.3.0"
+uuid = { version = "0.8.1", features = ["serde", "v4", "v5"] }
+
+unicode-segmentation = { version = "1.2.1", optional = true }
 libc = {version = "0.2.59", features = ["extra_traits",]}
+isahc = { version = "0.9.7", optional = true, default-features = false, features = ["http2", "json", "text-decoding"]}
+serde_json = { version = "1.0", optional = true, features = ["raw_value",] }
+smallvec = { version = "^1.5.0", features = ["serde", ] }
+nix = "0.17.0"
+rusqlite = {version = "0.24.0", optional = true }
+
+libloading = "0.6.2"
+futures = "0.3.5"
+smol = "1.0.0"
+async-stream = "0.2.1"
+base64 = { version = "0.12.3", optional = true }
+flate2 = { version = "1.0.16", optional = true }
+xdg-utils = "^0.4.0"
 
 [features]
-default = ["unicode_algorithms", "imap_backend", "maildir_backend", "mbox_backend", "vcard"]
+default = ["unicode_algorithms", "imap_backend", "maildir_backend", "mbox_backend", "vcard", "sqlite3", "smtp", "deflate_compression"]
 
 debug-tracing = []
-unicode_algorithms = ["text_processing"]
-imap_backend = ["native-tls"]
-maildir_backend = ["notify", "notify-rust", "memmap"]
-mbox_backend = ["notify", "notify-rust", "memmap"]
+deflate_compression = ["flate2", ]
+gpgme = []
+http = ["isahc"]
+http-static = ["isahc", "isahc/static-curl"]
+imap_backend = ["tls"]
+jmap_backend = ["http", "serde_json"]
+maildir_backend = ["notify"]
+mbox_backend = ["notify"]
+notmuch_backend = []
+smtp = ["tls", "base64"]
+sqlite3 = ["rusqlite", ]
+tls = ["native-tls"]
+unicode_algorithms = ["unicode-segmentation"]
 vcard = []

melib/README.md

@@ -0,0 +1,88 @@
+# melib
+
+[![GitHub license](https://img.shields.io/github/license/meli/meli)](https://github.com/meli/meli/blob/master/COPYING) [![Crates.io](https://img.shields.io/crates/v/melib)](https://crates.io/crates/melib) [![docs.rs](https://docs.rs/melib/badge.svg)](https://docs.rs/melib)
+
+Library for handling mail.
+
+## optional features
+
+| feature flag           | dependencies                        | notes                    |
+| ---------------------- | ----------------------------------- | ------------------------ |
+| `imap_backend`         | `native-tls`                        |                          |
+| `deflate_compression`  | `flate2`                            | for use with IMAP        |
+| `jmap_backend`         | `isahc`, `native-tls`, `serde_json` |                          |
+| `maildir_backend`      | `notify`                            |                          |
+| `mbox_backend`         | `notify`                            |                          |
+| `notmuch_backend`      | `notify`                            |                          |
+| `sqlite`               | `rusqlite`                          | used in IMAP cache       |
+| `unicode_algorithms`   | `unicode-segmentation`              | linebreaking algo etc    |
+| `vcard`                |                                     | vcard parsing            |
+| `gpgme`                |                                     | GPG use with libgpgme    |
+| `smtp`                 | `native-tls`, `base64`              | async SMTP communication |
+
+## Example: Parsing bytes into an `Envelope`
+
+An `Envelope` represents the information you can get from an email's headers
+and body structure. Addresses in `To`, `From` fields etc are parsed into
+`Address` types.
+
+```rust
+use melib::{Attachment, Envelope};
+
+let raw_mail = r#"From: "some name" <some@example.com>
+To: "me" <myself@example.com>
+Cc:
+Subject: =?utf-8?Q?gratuitously_encoded_subject?=
+Message-ID: <h2g7f.z0gy2pgaen5m@example.com>
+MIME-Version: 1.0
+Content-Type: multipart/mixed; charset="utf-8";
+ boundary="bzz_bzz__bzz__"
+
+This is a MIME formatted message with attachments. Use a MIME-compliant client to view it properly.
+--bzz_bzz__bzz__
+
+hello world.
+--bzz_bzz__bzz__
+Content-Type: image/gif; name="test_image.gif"; charset="utf-8"
+Content-Disposition: attachment
+Content-Transfer-Encoding: base64
+
+R0lGODdhKAAXAOfZAAABzAADzQAEzgQFtBEAxAAGxBcAxwALvRcFwAAPwBcLugATuQEUuxoNuxYQ
+sxwOvAYVvBsStSAVtx8YsRUcuhwhth4iuCQsyDAwuDc1vTc3uDg4uT85rkc9ukJBvENCvURGukdF
+wUVKt0hLuUxPvVZSvFlYu1hbt2BZuFxdul5joGhqlnNuf3FvlnBvwXJyt3Jxw3N0oXx1gH12gV99
+z317f3N7spFxwHp5wH99gYB+goF/g25+26tziIOBhWqD3oiBjICAuudkjIN+zHeC2n6Bzc1vh4eF
+iYaBw8F0kImHi4KFxYyHmIWIvI2Lj4uIvYaJyY+IuJGMi5iJl4qKxZSMmIuLxpONnpGPk42NvI2M
+1LKGl46OvZePm5ORlZiQnJqSnpaUmLyJnJuTn5iVmZyUoJGVyZ2VoZSVw5iXoZmWrO18rJiUyp6W
+opuYnKaVnZ+Xo5yZncaMoaCYpJiaqo+Z2Z2annuf5qGZpa2WoJybpZmayZ2Z0KCZypydrZ6dp6Cd
+oZ6a0aGay5ucy5+eqKGeouWMgp+b0qKbzKCfqdqPnp2ezaGgqqOgpKafqrScpp+gz6ajqKujr62j
+qayksKmmq62lsaiosqqorOyWnaqqtKeqzLGptaurta2rr7Kqtq+ssLOrt6+uuLGusuqhfbWtubCv
+ubKvs7GwurOwtPSazbevu+ali7SxtbiwvOykjLOyvLWytuCmqOankrSzvbazuLmyvrW0vre0uba1
+wLi1ury0wLm2u721wbe3wbq3vMC2vLi4wr+3w7m5w8C4xLi6yry6vsG5xbu7xcC6zMK6xry8xry+
+u8O7x729x8C9wb++yMG+wsO+vMK/w8a+y8e/zMnBzcXH18nL2///////////////////////////
+////////////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////ywAAAAAKAAXAAAI/gBP4Cjh
+IYMLEh0w4EgBgsMLEyFGFBEB5cOFABgzatS4AVssZAOsLOHCxooVMzCyoNmzaBOkJlS0VEDyZMjG
+mxk3XOMF60CDBgsoPABK9KcDCRImPCiQYAECAgQCRMU4VSrGCjFarBgUSJCgQ10FBTrkNRCfPnz4
+dA3UNa1btnDZqgU7Ntqzu3ej2X2mFy9eaHuhNRtMGJrhwYYN930G2K7eaNIY34U2mfJkwpgzI9Yr
+GBqwR2KSvAlMOXHnw5pTNzPdLNoWIWtU9XjGjDEYS8LAlFm1SrVvzIKj5TH0KpORSZOryPgCZgqL
+Ob+jG0YVRBErUrOiiGJ8KxgtYsh27xWL/tswnTtEbsiRVYdJNMHk4yOGhswGjR88UKjQ9Ey+/8TL
+XKKGGn7Akph/8XX2WDTTcAYfguVt9hhrEPqmzIOJ3VUheb48WJiHG6amC4i+WVJKKCimqGIoYxyj
+WWK8kKjaJ9bA18sxvXjYhourmbbMMrjI+OIn1QymDCVXANGFK4S1gQw0PxozzC+33FLLKUJq9gk1
+gyWDhyNwrMLkYGUEM4wvuLRiCiieXIJJJVlmJskcZ9TZRht1lnFGGmTMkMoonVQSSSOFAGJHHI0w
+ouiijDaaCCGQRgrpH3q4QYYXWDihxBE+7KCDDjnUIEVAADs=
+--bzz_bzz__bzz__--"#;
+
+let envelope = Envelope::from_bytes(raw_mail.as_bytes(), None).expect("Could not parse mail");
+assert_eq!(envelope.subject().as_ref(), "gratuitously encoded subject");
+assert_eq!(envelope.message_id_display().as_ref(), "<h2g7f.z0gy2pgaen5m@example.com>");
+
+let body = envelope.body_bytes(raw_mail.as_bytes());
+assert_eq!(body.content_type().to_string().as_str(), "multipart/mixed");
+
+let body_text = body.text();
+assert_eq!(body_text.as_str(), "hello world.");
+
+let subattachments: Vec<Attachment> = body.attachments();
+assert_eq!(subattachments.len(), 3);
+assert_eq!(subattachments[2].content_type().name().unwrap(), "test_image.gif");
+```

melib/build.rs

@@ -0,0 +1,426 @@
+/*
+ * meli - melib crate.
+ *
+ * Copyright 2017-2020 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#[cfg(feature = "unicode_algorithms")]
+include!("src/text_processing/types.rs");
+
+fn main() -> Result<(), std::io::Error> {
+    #[cfg(feature = "unicode_algorithms")]
+    {
+        const MOD_PATH: &str = "src/text_processing/tables.rs";
+        println!("cargo:rerun-if-changed=build.rs");
+        println!("cargo:rerun-if-changed={}", MOD_PATH);
+        /* Line break tables */
+        use std::fs::File;
+        use std::io::prelude::*;
+        use std::io::BufReader;
+        use std::path::Path;
+        use std::process::{Command, Stdio};
+        const LINE_BREAK_TABLE_URL: &str =
+            "http://www.unicode.org/Public/UCD/latest/ucd/LineBreak.txt";
+        /* Grapheme width tables */
+        const UNICODE_DATA_URL: &str =
+            "http://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt";
+        const EAW_URL: &str = "http://www.unicode.org/Public/UCD/latest/ucd/EastAsianWidth.txt";
+        const EMOJI_DATA_URL: &str =
+            "https://www.unicode.org/Public/UCD/latest/ucd/emoji/emoji-data.txt";
+
+
+        let mod_path = Path::new(MOD_PATH);
+        if mod_path.exists() {
+            eprintln!(
+                "{} already exists, delete it if you want to replace it.",
+                mod_path.display()
+            );
+            std::process::exit(0);
+        }
+        let mut child = Command::new("curl")
+            .args(&["-o", "-", LINE_BREAK_TABLE_URL])
+            .stdout(Stdio::piped())
+            .stdin(Stdio::null())
+            .stderr(Stdio::inherit())
+            .spawn()?;
+
+        let buf_reader = BufReader::new(child.stdout.take().unwrap());
+
+        let mut line_break_table: Vec<(u32, u32, LineBreakClass)> = Vec::with_capacity(3800);
+        for line in buf_reader.lines() {
+            let line = line.unwrap();
+            if line.starts_with('#') || line.starts_with(' ') || line.is_empty() {
+                continue;
+            }
+            let tokens: &str = line.split_whitespace().next().unwrap();
+
+            let semicolon_idx: usize = tokens.chars().position(|c| c == ';').unwrap();
+            /* LineBreak.txt list is ascii encoded so we can assume each char takes one byte: */
+            let chars_str: &str = &tokens[..semicolon_idx];
+
+            let mut codepoint_iter = chars_str.split("..");
+
+            let first_codepoint: u32 =
+                u32::from_str_radix(codepoint_iter.next().unwrap(), 16).unwrap();
+
+            let sec_codepoint: u32 = codepoint_iter
+                .next()
+                .map(|v| u32::from_str_radix(v, 16).unwrap())
+                .unwrap_or(first_codepoint);
+            let class = &tokens[semicolon_idx + 1..semicolon_idx + 1 + 2];
+            line_break_table.push((first_codepoint, sec_codepoint, LineBreakClass::from(class)));
+        }
+        child.wait()?;
+
+        let child = Command::new("curl")
+            .args(&["-o", "-", UNICODE_DATA_URL])
+            .stdout(Stdio::piped())
+            .output()?;
+
+        let unicode_data = String::from_utf8_lossy(&child.stdout);
+
+        let child = Command::new("curl")
+            .args(&["-o", "-", EAW_URL])
+            .stdout(Stdio::piped())
+            .output()?;
+
+        let eaw_data = String::from_utf8_lossy(&child.stdout);
+
+        let child = Command::new("curl")
+            .args(&["-o", "-", EMOJI_DATA_URL])
+            .stdout(Stdio::piped())
+            .output()?;
+
+        let emoji_data = String::from_utf8_lossy(&child.stdout);
+
+        const MAX_CODEPOINT: usize = 0x110000;
+        // See https://www.unicode.org/L2/L1999/UnicodeData.html
+        const FIELD_CODEPOINT: usize = 0;
+        const FIELD_CATEGORY: usize = 2;
+        // Ambiguous East Asian characters
+        const WIDTH_AMBIGUOUS_EASTASIAN: isize = -3;
+
+        // Width changed from 1 to 2 in Unicode 9.0
+        const WIDTH_WIDENED_IN_9: isize = -6;
+        // Category for unassigned codepoints.
+        const CAT_UNASSIGNED: &str = "Cn";
+
+        // Category for private use codepoints.
+        const CAT_PRIVATE_USE: &str = "Co";
+
+        // Category for surrogates.
+        const CAT_SURROGATE: &str = "Cs";
+
+        struct Codepoint<'cat> {
+            raw: u32,
+            width: Option<isize>,
+            category: &'cat str,
+        }
+
+        let mut codepoints: Vec<Codepoint> = Vec::with_capacity(MAX_CODEPOINT + 1);
+        for i in 0..=MAX_CODEPOINT {
+            codepoints.push(Codepoint {
+                raw: i as u32,
+                width: None,
+                category: CAT_UNASSIGNED,
+            });
+        }
+
+        set_general_categories(&mut codepoints, &unicode_data);
+        set_eaw_widths(&mut codepoints, &eaw_data);
+        set_emoji_widths(&mut codepoints, &emoji_data);
+        set_hardcoded_ranges(&mut codepoints);
+        fn hexrange_to_range(hexrange: &str) -> std::ops::Range<usize> {
+            /* Given a string like 1F300..1F320 representing an inclusive range,
+            return the range of codepoints.
+            If the string is like 1F321, return a range of just that element.
+            */
+            let hexrange = hexrange.trim();
+            let fields = hexrange
+                .split("..")
+                .map(|h| usize::from_str_radix(h.trim(), 16).unwrap())
+                .collect::<Vec<usize>>();
+            if fields.len() == 1 {
+                fields[0]..(fields[0] + 1)
+            } else {
+                fields[0]..(fields[1] + 1)
+            }
+        }
+
+        fn set_general_categories<'u>(codepoints: &mut Vec<Codepoint<'u>>, unicode_data: &'u str) {
+            for line in unicode_data.lines() {
+                let fields = line.trim().split(";").collect::<Vec<_>>();
+                if fields.len() > FIELD_CATEGORY {
+                    for idx in hexrange_to_range(fields[FIELD_CODEPOINT]) {
+                        codepoints[idx].category = fields[FIELD_CATEGORY];
+                    }
+                }
+            }
+        }
+
+        fn set_eaw_widths(codepoints: &mut Vec<Codepoint<'_>>, eaw_data_lines: &str) {
+            //  Read from EastAsianWidth.txt, set width values on the codepoints
+            for line in eaw_data_lines.lines() {
+                let line = line.trim().split('#').next().unwrap_or(line);
+                let fields = line.trim().split(';').collect::<Vec<_>>();
+                if fields.len() != 2 {
+                    continue;
+                }
+                let hexrange = fields[0];
+                let width_type = fields[1];
+                // width_types:
+                //  A: ambiguous, F: fullwidth, H: halfwidth,
+                // . N: neutral, Na: east-asian Narrow
+                let width: isize = if width_type == "A" {
+                    WIDTH_AMBIGUOUS_EASTASIAN
+                } else if width_type == "F" || width_type == "W" {
+                    2
+                } else {
+                    1
+                };
+                for cp in hexrange_to_range(hexrange) {
+                    codepoints[cp].width = Some(width);
+                }
+            }
+            // Apply the following special cases:
+            //  - The unassigned code points in the following blocks default to "W":
+            //         CJK Unified Ideographs Extension A: U+3400..U+4DBF
+            //         CJK Unified Ideographs:             U+4E00..U+9FFF
+            //         CJK Compatibility Ideographs:       U+F900..U+FAFF
+            //  - All undesignated code points in Planes 2 and 3, whether inside or
+            //      outside of allocated blocks, default to "W":
+            //         Plane 2:                            U+20000..U+2FFFD
+            //         Plane 3:                            U+30000..U+3FFFD
+            const WIDE_RANGES: [(usize, usize); 5] = [
+                (0x3400, 0x4DBF),
+                (0x4E00, 0x9FFF),
+                (0xF900, 0xFAFF),
+                (0x20000, 0x2FFFD),
+                (0x30000, 0x3FFFD),
+            ];
+            for &wr in WIDE_RANGES.iter() {
+                for cp in wr.0..(wr.1 + 1) {
+                    if codepoints[cp].width.is_none() {
+                        codepoints[cp].width = Some(2);
+                    }
+                }
+            }
+        }
+        fn set_emoji_widths(codepoints: &mut Vec<Codepoint<'_>>, emoji_data_lines: &str) {
+            // Read from emoji-data.txt, set codepoint widths
+            for line in emoji_data_lines.lines() {
+                if !line.contains("#") || line.trim().starts_with("#") {
+                    continue;
+                }
+                let mut fields = line.trim().split('#').collect::<Vec<_>>();
+                if fields.len() != 2 {
+                    continue;
+                }
+                let comment = fields.pop().unwrap();
+                let fields = fields.pop().unwrap();
+
+                let hexrange = fields.split(";").next().unwrap();
+
+                // In later versions of emoji-data.txt there are some "reserved"
+                // entries that have "NA" instead of a Unicode version number
+                // of first use, they will now return a zero version instead of
+                // crashing the script
+                if comment.trim().starts_with("NA") {
+                    continue;
+                }
+
+                use std::str::FromStr;
+                let mut v = comment.trim().split_whitespace().next().unwrap();
+                if v.starts_with("E") {
+                    v = &v[1..];
+                }
+                if v.as_bytes()
+                    .get(0)
+                    .map(|c| !c.is_ascii_digit())
+                    .unwrap_or(true)
+                {
+                    continue;
+                }
+                let mut idx = 1;
+                while v
+                    .as_bytes()
+                    .get(idx)
+                    .map(|c| c.is_ascii_digit())
+                    .unwrap_or(false)
+                {
+                    idx += 1;
+                }
+                if v.as_bytes().get(idx).map(|&c| c != b'.').unwrap_or(true) {
+                    continue;
+                }
+                idx += 1;
+                while v
+                    .as_bytes()
+                    .get(idx)
+                    .map(|c| c.is_ascii_digit())
+                    .unwrap_or(false)
+                {
+                    idx += 1;
+                }
+                v = &v[0..idx];
+
+                let version = f32::from_str(v).unwrap();
+                for cp in hexrange_to_range(hexrange) {
+                    // Don't consider <=1F000 values as emoji. These can only be made
+                    // emoji through the variation selector which interacts terribly
+                    // with wcwidth().
+                    if cp < 0x1F000 {
+                        continue;
+                    }
+                    // Skip codepoints that are explicitly not wide.
+                    // For example U+1F336 ("Hot Pepper") renders like any emoji but is
+                    // marked as neutral in EAW so has width 1 for some reason.
+                    //if codepoints[cp].width == Some(1) {
+                    //    continue;
+                    //}
+
+                    // If this emoji was introduced before Unicode 9, then it was widened in 9.
+                    codepoints[cp].width = if version >= 9.0 {
+                        Some(2)
+                    } else {
+                        Some(WIDTH_WIDENED_IN_9)
+                    };
+                }
+            }
+        }
+        fn set_hardcoded_ranges(codepoints: &mut Vec<Codepoint<'_>>) {
+            // Mark private use and surrogate codepoints
+            // Private use can be determined awkwardly from UnicodeData.txt,
+            // but we just hard-code them.
+            // We do not treat "private use high surrogate" as private use
+            // so as to match wcwidth9().
+            const PRIVATE_RANGES: [(usize, usize); 3] =
+                [(0xE000, 0xF8FF), (0xF0000, 0xFFFFD), (0x100000, 0x10FFFD)];
+            for &(first, last) in PRIVATE_RANGES.iter() {
+                for idx in first..=last {
+                    codepoints[idx].category = CAT_PRIVATE_USE;
+                }
+            }
+
+            const SURROGATE_RANGES: [(usize, usize); 2] = [(0xD800, 0xDBFF), (0xDC00, 0xDFFF)];
+            for &(first, last) in SURROGATE_RANGES.iter() {
+                for idx in first..=last {
+                    codepoints[idx].category = CAT_SURROGATE;
+                }
+            }
+        }
+
+        let mut file = File::create(&mod_path)?;
+        file.write_all(
+            br#"/*
+ * meli - text_processing crate.
+ *
+ * Copyright 2017-2020 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::types::LineBreakClass::{self, *};
+
+pub const LINE_BREAK_RULES: &[(u32, u32, LineBreakClass)] = &[
+"#,
+        )
+        .unwrap();
+        for l in &line_break_table {
+            file.write_all(format!("    (0x{:X}, 0x{:X}, {:?}),\n", l.0, l.1, l.2).as_bytes())
+                .unwrap();
+        }
+        file.write_all(b"];\n").unwrap();
+
+        for (name, filter) in [
+            (
+                "ASCII",
+                Box::new(|c: &&Codepoint| c.raw < 0x7f && c.raw >= 0x20)
+                    as Box<dyn Fn(&&Codepoint) -> bool>,
+            ),
+            (
+                "PRIVATE",
+                Box::new(|c: &&Codepoint| c.category == CAT_PRIVATE_USE),
+            ),
+            (
+                "NONPRINT",
+                Box::new(|c: &&Codepoint| {
+                    ["Cc", "Cf", "Zl", "Zp", CAT_SURROGATE].contains(&c.category)
+                }),
+            ),
+            (
+                "COMBINING",
+                Box::new(|c: &&Codepoint| ["Mn", "Mc", "Me"].contains(&c.category)),
+            ),
+            ("DOUBLEWIDE", Box::new(|c: &&Codepoint| c.width == Some(2))),
+            (
+                "UNASSIGNED",
+                Box::new(|c: &&Codepoint| c.category == CAT_UNASSIGNED),
+            ),
+            (
+                "AMBIGUOUS",
+                Box::new(|c: &&Codepoint| c.width == Some(WIDTH_AMBIGUOUS_EASTASIAN)),
+            ),
+            (
+                "WIDENEDIN9",
+                Box::new(|c: &&Codepoint| c.width == Some(WIDTH_WIDENED_IN_9)),
+            ),
+        ]
+        .iter()
+        {
+            file.write_all(
+                format!(
+                    r#"
+pub const {}: &[(u32, u32)] = &[
+"#,
+                    name
+                )
+                .as_bytes(),
+            )
+            .unwrap();
+            let mut iter = codepoints.iter().filter(filter);
+            let mut prev = iter.next().unwrap().raw;
+            let mut a = prev;
+            for cp in iter {
+                if prev + 1 != cp.raw {
+                    file.write_all(format!("    (0x{:X}, 0x{:X}),\n", a, prev).as_bytes())
+                        .unwrap();
+                    a = cp.raw;
+                }
+                prev = cp.raw;
+            }
+            file.write_all(format!("    (0x{:X}, 0x{:X}),\n", a, prev).as_bytes())
+                .unwrap();
+            file.write_all(b"];\n").unwrap();
+        }
+    }
+    Ok(())
+}

melib/src/addressbook.rs

@@ -22,8 +22,8 @@
 #[cfg(feature = "vcard")]
 pub mod vcard;
 
-use chrono::{DateTime, Local};
-use fnv::FnvHashMap;
+use crate::datetime::{self, UnixTimestamp};
+use std::collections::HashMap;
 use uuid::Uuid;
 
 use std::ops::Deref;
@@ -59,9 +59,9 @@ impl From<String> for CardId {
 #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
 pub struct AddressBook {
     display_name: String,
-    created: DateTime<Local>,
-    last_edited: DateTime<Local>,
-    cards: FnvHashMap<CardId, Card>,
+    created: UnixTimestamp,
+    last_edited: UnixTimestamp,
+    pub cards: HashMap<CardId, Card>,
 }
 
 #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
@@ -73,14 +73,14 @@ pub struct Card {
     name_prefix: String,
     name_suffix: String,
     //address
-    birthday: Option<DateTime<Local>>,
+    birthday: Option<UnixTimestamp>,
     email: String,
     url: String,
     key: String,
 
     color: u8,
-    last_edited: DateTime<Local>,
-    extra_properties: FnvHashMap<String, String>,
+    last_edited: UnixTimestamp,
+    extra_properties: HashMap<String, String>,
 
     /// If true, we can't make any changes because we do not manage this resource.
     external_resource: bool,
@@ -90,11 +90,31 @@ impl AddressBook {
     pub fn new(display_name: String) -> AddressBook {
         AddressBook {
             display_name,
-            created: Local::now(),
-            last_edited: Local::now(),
-            cards: FnvHashMap::default(),
+            created: datetime::now(),
+            last_edited: datetime::now(),
+            cards: HashMap::default(),
         }
     }
+
+    pub fn with_account(s: &crate::conf::AccountSettings) -> AddressBook {
+        #[cfg(not(feature = "vcard"))]
+        {
+            AddressBook::new(s.name.clone())
+        }
+        #[cfg(feature = "vcard")]
+        {
+            let mut ret = AddressBook::new(s.name.clone());
+            if let Some(vcard_path) = s.vcard_folder() {
+                if let Ok(cards) = vcard::load_cards(&std::path::Path::new(vcard_path)) {
+                    for c in cards {
+                        ret.add_card(c);
+                    }
+                }
+            }
+            ret
+        }
+    }
+
     pub fn add_card(&mut self, card: Card) {
         self.cards.insert(card.id, card);
     }
@@ -108,15 +128,15 @@ impl AddressBook {
         self.cards
             .values()
             .filter(|c| c.email.contains(term))
-            .map(|c| c.email.clone())
+            .map(|c| format!("{} <{}>", &c.name, &c.email))
             .collect()
     }
 }
 
 impl Deref for AddressBook {
-    type Target = FnvHashMap<CardId, Card>;
+    type Target = HashMap<CardId, Card>;
 
-    fn deref(&self) -> &FnvHashMap<CardId, Card> {
+    fn deref(&self) -> &HashMap<CardId, Card> {
         &self.cards
     }
 }
@@ -136,9 +156,9 @@ impl Card {
             url: String::new(),
             key: String::new(),
 
-            last_edited: Local::now(),
+            last_edited: datetime::now(),
             external_resource: false,
-            extra_properties: FnvHashMap::default(),
+            extra_properties: HashMap::default(),
             color: 0,
         }
     }
@@ -172,7 +192,7 @@ impl Card {
         self.key.as_str()
     }
     pub fn last_edited(&self) -> String {
-        self.last_edited.to_rfc2822()
+        datetime::timestamp_to_string(self.last_edited, None, false)
     }
 
     pub fn set_id(&mut self, new_val: CardId) {
@@ -211,7 +231,7 @@ impl Card {
         self.extra_properties.get(key).map(String::as_str)
     }
 
-    pub fn extra_properties(&self) -> &FnvHashMap<String, String> {
+    pub fn extra_properties(&self) -> &HashMap<String, String> {
         &self.extra_properties
     }
 
@@ -224,8 +244,8 @@ impl Card {
     }
 }
 
-impl From<FnvHashMap<String, String>> for Card {
-    fn from(mut map: FnvHashMap<String, String>) -> Card {
+impl From<HashMap<String, String>> for Card {
+    fn from(mut map: HashMap<String, String>) -> Card {
         let mut card = Card::new();
         if let Some(val) = map.remove("TITLE") {
             card.title = val;

melib/src/addressbook/vcard.rs

@@ -21,35 +21,43 @@
 
 /// Convert VCard strings to meli Cards (contacts).
 use super::*;
-use crate::chrono::TimeZone;
 use crate::error::{MeliError, Result};
-use fnv::FnvHashMap;
+use crate::parsec::{match_literal_anycase, one_or_more, peek, prefix, take_until, Parser};
+use std::collections::HashMap;
+use std::convert::TryInto;
 
 /* Supported vcard versions */
-pub trait VCardVersion {}
+pub trait VCardVersion: core::fmt::Debug {}
+
+#[derive(Debug)]
+pub struct VCardVersionUnknown;
+impl VCardVersion for VCardVersionUnknown {}
 
 /// https://tools.ietf.org/html/rfc6350
+#[derive(Debug)]
 pub struct VCardVersion4;
 impl VCardVersion for VCardVersion4 {}
 
 /// https://tools.ietf.org/html/rfc2426
+#[derive(Debug)]
 pub struct VCardVersion3;
 impl VCardVersion for VCardVersion3 {}
 
 pub struct CardDeserializer;
 
-static HEADER: &'static str = "BEGIN:VCARD\r\nVERSION:4.0\r\n";
-static FOOTER: &'static str = "END:VCARD\r\n";
+static HEADER: &str = "BEGIN:VCARD\r\n"; //VERSION:4.0\r\n";
+static FOOTER: &str = "END:VCARD\r\n";
 
+#[derive(Debug)]
 pub struct VCard<T: VCardVersion>(
-    fnv::FnvHashMap<String, ContentLine>,
+    HashMap<String, ContentLine>,
     std::marker::PhantomData<*const T>,
 );
 
 impl<V: VCardVersion> VCard<V> {
     pub fn new_v4() -> VCard<impl VCardVersion> {
         VCard(
-            FnvHashMap::default(),
+            HashMap::default(),
             std::marker::PhantomData::<*const VCardVersion4>,
         )
     }
@@ -70,7 +78,7 @@ impl CardDeserializer {
             &input[HEADER.len()..input.len() - FOOTER.len()]
         };
 
-        let mut ret = FnvHashMap::default();
+        let mut ret = HashMap::default();
 
         enum Stage {
             Group,
@@ -103,13 +111,18 @@ impl CardDeserializer {
                         el.params.push(l[value_start..i].to_string());
                         value_start = i + 1;
                     }
+                    (b';', Stage::Name) => {
+                        name = l[value_start..i].to_string();
+                        value_start = i + 1;
+                        stage = Stage::Param;
+                    }
                     (b':', Stage::Group) | (b':', Stage::Name) => {
                         name = l[value_start..i].to_string();
                         has_colon = true;
                         value_start = i + 1;
                         stage = Stage::Value;
                     }
-                    (b':', Stage::Param) if l.as_bytes()[i] != b'\\' => {
+                    (b':', Stage::Param) if l.as_bytes()[i.saturating_sub(1)] != b'\\' => {
                         el.params.push(l[value_start..i].to_string());
                         has_colon = true;
                         value_start = i + 1;
@@ -118,7 +131,6 @@ impl CardDeserializer {
                     _ => {}
                 }
             }
-            el.value = l[value_start..].to_string();
             if !has_colon {
                 return Err(MeliError::new(format!(
                     "Error while parsing vcard: error at line {}, no colon. {:?}",
@@ -131,13 +143,14 @@ impl CardDeserializer {
                     l, el
                 )));
             }
+            el.value = l[value_start..].replace("\\:", ":");
             ret.insert(name, el);
         }
         Ok(VCard(ret, std::marker::PhantomData::<*const VCardVersion4>))
     }
 }
 
-impl<V: VCardVersion> std::convert::TryInto<Card> for VCard<V> {
+impl<V: VCardVersion> TryInto<Card> for VCard<V> {
     type Error = crate::error::MeliError;
 
     fn try_into(mut self) -> crate::error::Result<Card> {
@@ -188,7 +201,8 @@ impl<V: VCardVersion> std::convert::TryInto<Card> for VCard<V> {
                       T102200Z
                       T102200-0800
                       */
-            card.birthday = chrono::Local.datetime_from_str(&val.value, "%Y%m%d").ok();
+            card.birthday = crate::datetime::timestamp_from_string(val.value.as_str(), "%Y%m%d\0")
+                .unwrap_or_default();
         }
         if let Some(val) = self.0.remove("EMAIL") {
             card.set_email(val.value);
@@ -200,6 +214,9 @@ impl<V: VCardVersion> std::convert::TryInto<Card> for VCard<V> {
             card.set_key(val.value);
         }
         for (k, v) in self.0.into_iter() {
+            if k.eq_ignore_ascii_case("VERSION") || k.eq_ignore_ascii_case("N") {
+                continue;
+            }
             card.set_extra_property(&k, v.value);
         }
 
@@ -207,6 +224,80 @@ impl<V: VCardVersion> std::convert::TryInto<Card> for VCard<V> {
     }
 }
 
+fn parse_card<'a>() -> impl Parser<'a, Vec<&'a str>> {
+    move |input| {
+        one_or_more(prefix(
+            peek(match_literal_anycase(HEADER)),
+            take_until(match_literal_anycase(FOOTER)),
+        ))
+        .parse(input)
+    }
+}
+
+#[test]
+fn test_load_cards() {
+    /*
+    let mut contents = String::with_capacity(256);
+    let p = &std::path::Path::new("/tmp/contacts.vcf");
+    use std::io::Read;
+    contents.clear();
+    std::fs::File::open(&p)
+        .unwrap()
+        .read_to_string(&mut contents)
+        .unwrap();
+    for s in parse_card().parse(contents.as_str()).unwrap().1 {
+        println!("");
+        println!("{}", s);
+        println!("{:?}", CardDeserializer::from_str(s));
+        println!("");
+    }
+    */
+}
+
+pub fn load_cards(p: &std::path::Path) -> Result<Vec<Card>> {
+    let vcf_dir = std::fs::read_dir(p);
+    let mut ret: Vec<Result<_>> = Vec::new();
+    let mut is_any_valid = false;
+    if vcf_dir.is_ok() {
+        let mut contents = String::with_capacity(256);
+        for f in vcf_dir? {
+            if f.is_err() {
+                continue;
+            }
+            let f = f?.path();
+            if f.is_file() {
+                use std::io::Read;
+                contents.clear();
+                std::fs::File::open(&f)?.read_to_string(&mut contents)?;
+                if let Ok((_, c)) = parse_card().parse(contents.as_str()) {
+                    for s in c {
+                        ret.push(
+                            CardDeserializer::from_str(s)
+                                .and_then(TryInto::try_into)
+                                .and_then(|mut card| {
+                                    Card::set_external_resource(&mut card, true);
+                                    is_any_valid = true;
+                                    Ok(card)
+                                }),
+                        );
+                    }
+                }
+            }
+        }
+    }
+    for c in &ret {
+        if c.is_err() {
+            debug!(&c);
+        }
+    }
+    if !is_any_valid {
+        ret.into_iter().collect::<Result<Vec<Card>>>()
+    } else {
+        ret.retain(Result::is_ok);
+        ret.into_iter().collect::<Result<Vec<Card>>>()
+    }
+}
+
 #[test]
 fn test_card() {
     let j = "BEGIN:VCARD\r\nVERSION:4.0\r\nN:Gump;Forrest;;Mr.;\r\nFN:Forrest Gump\r\nORG:Bubba Gump Shrimp Co.\r\nTITLE:Shrimp Man\r\nPHOTO;MEDIATYPE=image/gif:http://www.example.com/dir_photos/my_photo.gif\r\nTEL;TYPE=work,voice;VALUE=uri:tel:+1-111-555-1212\r\nTEL;TYPE=home,voice;VALUE=uri:tel:+1-404-555-1212\r\nADR;TYPE=WORK;PREF=1;LABEL=\"100 Waters Edge\\nBaytown\\, LA 30314\\nUnited States of America\":;;100 Waters Edge;Baytown;LA;30314;United States of America\r\nADR;TYPE=HOME;LABEL=\"42 Plantation St.\\nBaytown\\, LA 30314\\nUnited States of America\":;;42 Plantation St.;Baytown;LA;30314;United States of America\r\nEMAIL:forrestgump@example.com\r\nREV:20080424T195243Z\r\nx-qq:21588891\r\nEND:VCARD\r\n";

melib/src/async_workers.rs

@@ -1,261 +0,0 @@
-/*
- * meli - async module
- *
- * Copyright 2017 Manos Pitsidianakis
- *
- * This file is part of meli.
- *
- * meli is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * meli is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with meli. If not, see <http://www.gnu.org/licenses/>.
- */
-
-/*!
- * Primitive Async/Wait implementation.
- *
- * To create an Async promise, create an AsyncBuilder. Ask for its channel receiver/sender with
- * `tx` and `rx` methods to pass them in your worker's closure. Build an `Async<T>` with your
- * `JoinHandle<T>`. The thread must communicate with the `Async<T>` object via `AsyncStatus`
- * messages.
- *
- * When `Async<T>` receives `AsyncStatus::Finished` it joins the thread and takes its value which
- * can be extracted with `extract`.
- */
-
-use crossbeam::{
-    bounded,
-    channel::{Receiver, Sender},
-    select,
-};
-use std::fmt;
-use std::sync::Arc;
-
-#[derive(Clone, Debug)]
-pub struct WorkContext {
-    pub new_work: Sender<Work>,
-    pub set_name: Sender<(std::thread::ThreadId, String)>,
-    pub set_status: Sender<(std::thread::ThreadId, String)>,
-    pub finished: Sender<std::thread::ThreadId>,
-}
-
-#[derive(Clone)]
-pub struct Work {
-    priority: u64,
-    pub is_static: bool,
-    pub closure: Arc<Box<dyn Fn(WorkContext) -> () + Send + Sync>>,
-    name: String,
-    status: String,
-}
-
-impl Ord for Work {
-    fn cmp(&self, other: &Work) -> std::cmp::Ordering {
-        self.priority.cmp(&other.priority)
-    }
-}
-
-impl PartialOrd for Work {
-    fn partial_cmp(&self, other: &Work) -> Option<std::cmp::Ordering> {
-        Some(self.priority.cmp(&other.priority))
-    }
-}
-
-impl PartialEq for Work {
-    fn eq(&self, other: &Work) -> bool {
-        self.priority == other.priority
-    }
-}
-
-impl Eq for Work {}
-
-impl Work {
-    pub fn compute(&self, work_context: WorkContext) {
-        (self.closure)(work_context);
-    }
-}
-
-impl fmt::Debug for Work {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "Work object")
-    }
-}
-
-/// Messages to pass between `Async<T>` owner and its worker thread.
-#[derive(Clone)]
-pub enum AsyncStatus<T> {
-    NoUpdate,
-    Payload(T),
-    Finished,
-    ///The number may hold whatever meaning the user chooses.
-    ProgressReport(usize),
-}
-
-impl<T> fmt::Debug for AsyncStatus<T> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            AsyncStatus::NoUpdate => write!(f, "AsyncStatus<T>::NoUpdate"),
-            AsyncStatus::Payload(_) => write!(f, "AsyncStatus<T>::Payload(_)"),
-            AsyncStatus::Finished => write!(f, "AsyncStatus<T>::Finished"),
-            AsyncStatus::ProgressReport(u) => write!(f, "AsyncStatus<T>::ProgressReport({})", u),
-        }
-    }
-}
-
-/// A builder object for `Async<T>`
-#[derive(Debug, Clone)]
-pub struct AsyncBuilder<T: Send + Sync> {
-    tx: Sender<AsyncStatus<T>>,
-    rx: Receiver<AsyncStatus<T>>,
-    priority: u64,
-    is_static: bool,
-}
-
-#[derive(Clone, Debug)]
-pub struct Async<T: Send + Sync> {
-    work: Work,
-    active: bool,
-    tx: Sender<AsyncStatus<T>>,
-    rx: Receiver<AsyncStatus<T>>,
-}
-
-impl<T: Send + Sync> Default for AsyncBuilder<T> {
-    fn default() -> Self {
-        AsyncBuilder::<T>::new()
-    }
-}
-
-impl<T> AsyncBuilder<T>
-where
-    T: Send + Sync,
-{
-    pub fn new() -> Self {
-        let (sender, receiver) = bounded(8 * ::std::mem::size_of::<AsyncStatus<T>>());
-        AsyncBuilder {
-            tx: sender,
-            rx: receiver,
-            priority: 0,
-            is_static: false,
-        }
-    }
-    /// Returns the sender object of the promise's channel.
-    pub fn tx(&mut self) -> Sender<AsyncStatus<T>> {
-        self.tx.clone()
-    }
-    /// Returns the receiver object of the promise's channel.
-    pub fn rx(&mut self) -> Receiver<AsyncStatus<T>> {
-        self.rx.clone()
-    }
-
-    pub fn set_priority(&mut self, new_val: u64) -> &mut Self {
-        self.priority = new_val;
-        self
-    }
-
-    pub fn set_is_static(&mut self, new_val: bool) -> &mut Self {
-        self.is_static = new_val;
-        self
-    }
-
-    /// Returns an `Async<T>` object that contains a `Thread` join handle that returns a `T`
-    pub fn build(self, work: Box<dyn Fn(WorkContext) -> () + Send + Sync>) -> Async<T> {
-        Async {
-            work: Work {
-                priority: self.priority,
-                is_static: self.is_static,
-                closure: Arc::new(work),
-                name: String::new(),
-                status: String::new(),
-            },
-            tx: self.tx,
-            rx: self.rx,
-            active: false,
-        }
-    }
-}
-
-impl<T> Async<T>
-where
-    T: Send + Sync,
-{
-    pub fn work(&mut self) -> Option<Work> {
-        if !self.active {
-            self.active = true;
-            Some(self.work.clone())
-        } else {
-            None
-        }
-    }
-    /// Returns the sender object of the promise's channel.
-    pub fn tx(&mut self) -> Sender<AsyncStatus<T>> {
-        self.tx.clone()
-    }
-    /// Returns the receiver object of the promise's channel.
-    pub fn rx(&mut self) -> Receiver<AsyncStatus<T>> {
-        self.rx.clone()
-    }
-    /// Polls worker thread and returns result.
-    pub fn poll_block(&mut self) -> Result<AsyncStatus<T>, ()> {
-        if !self.active {
-            return Ok(AsyncStatus::Finished);
-        }
-
-        let rx = &self.rx;
-        select! {
-            recv(rx) -> r => {
-                match r {
-                    Ok(p @ AsyncStatus::Payload(_)) => {
-                        return Ok(p);
-                    },
-                    Ok(f @ AsyncStatus::Finished) => {
-                        self.active = false;
-                        return Ok(f);
-                    },
-                    Ok(a) => {
-                        return Ok(a);
-                    }
-                    Err(_) => {
-                        return Err(());
-                    },
-                }
-            },
-        };
-    }
-    /// Polls worker thread and returns result.
-    pub fn poll(&mut self) -> Result<AsyncStatus<T>, ()> {
-        if !self.active {
-            return Ok(AsyncStatus::Finished);
-        }
-
-        let rx = &self.rx;
-        select! {
-            default => {
-                return Ok(AsyncStatus::NoUpdate);
-            },
-            recv(rx) -> r => {
-                match r {
-                    Ok(p @ AsyncStatus::Payload(_)) => {
-                        return Ok(p);
-                    },
-                    Ok(f @ AsyncStatus::Finished) => {
-                        self.active = false;
-                        return Ok(f);
-                    },
-                    Ok(a) => {
-                        return Ok(a);
-                    }
-                    Err(_) => {
-                        return Err(());
-                    },
-                }
-            },
-        };
-    }
-}

melib/src/backends.rs

@@ -18,16 +18,37 @@
  * You should have received a copy of the GNU General Public License
  * along with meli. If not, see <http://www.gnu.org/licenses/>.
  */
+
+use smallvec::SmallVec;
+#[macro_export]
+macro_rules! tag_hash {
+    ($tag:ident) => {{
+        use std::collections::hash_map::DefaultHasher;
+        use std::hash::Hasher;
+        let mut hasher = DefaultHasher::new();
+        hasher.write($tag.as_bytes());
+        hasher.finish()
+    }};
+}
+
 #[cfg(feature = "imap_backend")]
 pub mod imap;
+#[cfg(feature = "imap_backend")]
+pub mod nntp;
+#[cfg(feature = "notmuch_backend")]
+pub mod notmuch;
+#[cfg(feature = "notmuch_backend")]
+pub use self::notmuch::NotmuchDb;
+#[cfg(feature = "jmap_backend")]
+pub mod jmap;
 #[cfg(feature = "maildir_backend")]
 pub mod maildir;
 #[cfg(feature = "mbox_backend")]
 pub mod mbox;
-
 #[cfg(feature = "imap_backend")]
 pub use self::imap::ImapType;
-use crate::async_workers::*;
+#[cfg(feature = "imap_backend")]
+pub use self::nntp::NntpType;
 use crate::conf::AccountSettings;
 use crate::error::{MeliError, Result};
 
@@ -36,20 +57,47 @@ use self::maildir::MaildirType;
 #[cfg(feature = "mbox_backend")]
 use self::mbox::MboxType;
 use super::email::{Envelope, EnvelopeHash, Flag};
+use std::any::Any;
+use std::collections::BTreeSet;
 use std::fmt;
 use std::fmt::Debug;
 use std::ops::Deref;
+use std::sync::{Arc, RwLock};
 
-use fnv::FnvHashMap;
-use std;
+use futures::stream::Stream;
+use std::future::Future;
+use std::pin::Pin;
 
-pub type BackendCreator =
-    Box<dyn Fn(&AccountSettings, Box<dyn Fn(&str) -> bool + Send + Sync>) -> Box<dyn MailBackend>>;
+use std::collections::HashMap;
+
+#[macro_export]
+macro_rules! get_path_hash {
+    ($path:expr) => {{
+        use std::collections::hash_map::DefaultHasher;
+        use std::hash::{Hash, Hasher};
+        let mut hasher = DefaultHasher::new();
+        $path.hash(&mut hasher);
+        hasher.finish()
+    }};
+}
+
+pub type BackendCreator = Box<
+    dyn Fn(
+        &AccountSettings,
+        Box<dyn Fn(&str) -> bool + Send + Sync>,
+        BackendEventConsumer,
+    ) -> Result<Box<dyn MailBackend>>,
+>;
 
 /// A hashmap containing all available mail backends.
 /// An abstraction over any available backends.
 pub struct Backends {
-    map: FnvHashMap<std::string::String, Box<dyn Fn() -> BackendCreator>>,
+    map: HashMap<std::string::String, Backend>,
+}
+
+pub struct Backend {
+    pub create_fn: Box<dyn Fn() -> BackendCreator>,
+    pub validate_conf_fn: Box<dyn Fn(&AccountSettings) -> Result<()>>,
 }
 
 impl Default for Backends {
@@ -58,30 +106,74 @@ impl Default for Backends {
     }
 }
 
+#[cfg(feature = "notmuch_backend")]
+pub const NOTMUCH_ERROR_MSG: &str =
+    "libnotmuch5 was not found in your system. Make sure it is installed and in the library paths.\n";
+#[cfg(not(feature = "notmuch_backend"))]
+pub const NOTMUCH_ERROR_MSG: &str = "this version of meli is not compiled with notmuch support. Use an appropriate version and make sure libnotmuch5 is installed and in the library paths.\n";
+
 impl Backends {
     pub fn new() -> Self {
         let mut b = Backends {
-            map: FnvHashMap::with_capacity_and_hasher(1, Default::default()),
+            map: HashMap::with_capacity_and_hasher(1, Default::default()),
         };
         #[cfg(feature = "maildir_backend")]
         {
             b.register(
                 "maildir".to_string(),
-                Box::new(|| Box::new(|f, i| Box::new(MaildirType::new(f, i)))),
+                Backend {
+                    create_fn: Box::new(|| Box::new(|f, i, ev| MaildirType::new(f, i, ev))),
+                    validate_conf_fn: Box::new(MaildirType::validate_config),
+                },
             );
         }
         #[cfg(feature = "mbox_backend")]
         {
             b.register(
                 "mbox".to_string(),
-                Box::new(|| Box::new(|f, i| Box::new(MboxType::new(f, i)))),
+                Backend {
+                    create_fn: Box::new(|| Box::new(|f, i, ev| MboxType::new(f, i, ev))),
+                    validate_conf_fn: Box::new(MboxType::validate_config),
+                },
             );
         }
         #[cfg(feature = "imap_backend")]
         {
             b.register(
                 "imap".to_string(),
-                Box::new(|| Box::new(|f, i| Box::new(ImapType::new(f, i)))),
+                Backend {
+                    create_fn: Box::new(|| Box::new(|f, i, ev| imap::ImapType::new(f, i, ev))),
+                    validate_conf_fn: Box::new(imap::ImapType::validate_config),
+                },
+            );
+            b.register(
+                "nntp".to_string(),
+                Backend {
+                    create_fn: Box::new(|| Box::new(|f, i, ev| nntp::NntpType::new(f, i, ev))),
+                    validate_conf_fn: Box::new(nntp::NntpType::validate_config),
+                },
+            );
+        }
+        #[cfg(feature = "notmuch_backend")]
+        {
+            if libloading::Library::new("libnotmuch.so.5").is_ok() {
+                b.register(
+                    "notmuch".to_string(),
+                    Backend {
+                        create_fn: Box::new(|| Box::new(|f, i, ev| NotmuchDb::new(f, i, ev))),
+                        validate_conf_fn: Box::new(NotmuchDb::validate_config),
+                    },
+                );
+            }
+        }
+        #[cfg(feature = "jmap_backend")]
+        {
+            b.register(
+                "jmap".to_string(),
+                Backend {
+                    create_fn: Box::new(|| Box::new(|f, i, ev| jmap::JmapType::new(f, i, ev))),
+                    validate_conf_fn: Box::new(jmap::JmapType::validate_config),
+                },
             );
         }
         b
@@ -89,107 +181,221 @@ impl Backends {
 
     pub fn get(&self, key: &str) -> BackendCreator {
         if !self.map.contains_key(key) {
+            if key == "notmuch" {
+                eprint!("{}", NOTMUCH_ERROR_MSG);
+            }
             panic!("{} is not a valid mail backend", key);
         }
-        self.map[key]()
+        (self.map[key].create_fn)()
     }
 
-    pub fn register(&mut self, key: String, backend: Box<dyn Fn() -> BackendCreator>) {
+    pub fn register(&mut self, key: String, backend: Backend) {
         if self.map.contains_key(&key) {
             panic!("{} is an already registered backend", key);
         }
         self.map.insert(key, backend);
     }
+
+    pub fn validate_config(&self, key: &str, s: &AccountSettings) -> Result<()> {
+        (self
+            .map
+            .get(key)
+            .ok_or_else(|| {
+                MeliError::new(format!(
+                    "{}{} is not a valid mail backend",
+                    if key == "notmuch" {
+                        NOTMUCH_ERROR_MSG
+                    } else {
+                        ""
+                    },
+                    key
+                ))
+            })?
+            .validate_conf_fn)(s)
+    }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
+pub enum BackendEvent {
+    Notice {
+        description: Option<String>,
+        content: String,
+        level: crate::LoggingLevel,
+    },
+    Refresh(RefreshEvent),
+    //Job(Box<Future<Output = Result<()>> + Send + 'static>)
+}
+
+impl From<MeliError> for BackendEvent {
+    fn from(val: MeliError) -> BackendEvent {
+        BackendEvent::Notice {
+            description: val.summary.as_ref().map(|s| s.to_string()),
+            content: val.to_string(),
+            level: crate::LoggingLevel::ERROR,
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
 pub enum RefreshEventKind {
     Update(EnvelopeHash, Box<Envelope>),
     /// Rename(old_hash, new_hash)
     Rename(EnvelopeHash, EnvelopeHash),
     Create(Box<Envelope>),
     Remove(EnvelopeHash),
+    NewFlags(EnvelopeHash, (Flag, Vec<String>)),
     Rescan,
     Failure(MeliError),
+    MailboxCreate(Mailbox),
+    MailboxDelete(MailboxHash),
+    MailboxRename {
+        old_mailbox_hash: MailboxHash,
+        new_mailbox: Mailbox,
+    },
+    MailboxSubscribe(MailboxHash),
+    MailboxUnsubscribe(MailboxHash),
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct RefreshEvent {
-    hash: FolderHash,
-    kind: RefreshEventKind,
+    pub mailbox_hash: MailboxHash,
+    pub account_hash: AccountHash,
+    pub kind: RefreshEventKind,
 }
 
-impl RefreshEvent {
-    pub fn hash(&self) -> FolderHash {
-        self.hash
-    }
-    pub fn kind(self) -> RefreshEventKind {
-        /* consumes self! */
-        self.kind
+#[derive(Clone)]
+pub struct BackendEventConsumer(Arc<dyn Fn(AccountHash, BackendEvent) + Send + Sync>);
+impl BackendEventConsumer {
+    pub fn new(b: Arc<dyn Fn(AccountHash, BackendEvent) + Send + Sync>) -> Self {
+        BackendEventConsumer(b)
     }
 }
 
-/// A `RefreshEventConsumer` is a boxed closure that must be used to consume a `RefreshEvent` and
-/// send it to a UI provided channel. We need this level of abstraction to provide an interface for
-/// all users of mailbox refresh events.
-pub struct RefreshEventConsumer(Box<dyn Fn(RefreshEvent) -> () + Send + Sync>);
-impl RefreshEventConsumer {
-    pub fn new(b: Box<dyn Fn(RefreshEvent) -> () + Send + Sync>) -> Self {
-        RefreshEventConsumer(b)
-    }
-    pub fn send(&self, r: RefreshEvent) {
-        self.0(r);
-    }
-}
-
-pub struct NotifyFn(Box<dyn Fn(FolderHash) -> () + Send + Sync>);
-
-impl fmt::Debug for NotifyFn {
+impl fmt::Debug for BackendEventConsumer {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "NotifyFn Box")
+        write!(f, "BackendEventConsumer")
     }
 }
 
-impl From<Box<dyn Fn(FolderHash) -> () + Send + Sync>> for NotifyFn {
-    fn from(kind: Box<dyn Fn(FolderHash) -> () + Send + Sync>) -> Self {
-        NotifyFn(kind)
+impl Deref for BackendEventConsumer {
+    type Target = dyn Fn(AccountHash, BackendEvent) + Send + Sync;
+
+    fn deref(&self) -> &Self::Target {
+        &(*self.0)
     }
 }
 
-impl NotifyFn {
-    pub fn new(b: Box<dyn Fn(FolderHash) -> () + Send + Sync>) -> Self {
-        NotifyFn(b)
-    }
-    pub fn notify(&self, f: FolderHash) {
-        self.0(f);
-    }
+#[derive(Debug, Clone)]
+pub struct MailBackendCapabilities {
+    pub is_async: bool,
+    pub is_remote: bool,
+    pub extensions: Option<Vec<(String, MailBackendExtensionStatus)>>,
+    pub supports_search: bool,
+    pub supports_tags: bool,
+    pub supports_submission: bool,
 }
 
-#[derive(Debug, PartialEq, Eq, Hash, Clone)]
-pub enum FolderOperation {
-    Create,
-    Delete,
-    Subscribe,
-    Unsubscribe,
-    Rename(NewFolderName),
+#[derive(Debug, Copy, Clone)]
+pub enum MailBackendExtensionStatus {
+    Unsupported { comment: Option<&'static str> },
+    Supported { comment: Option<&'static str> },
+    Enabled { comment: Option<&'static str> },
 }
 
-type NewFolderName = String;
+pub type ResultFuture<T> = Result<Pin<Box<dyn Future<Output = Result<T>> + Send + 'static>>>;
 
-pub trait MailBackend: ::std::fmt::Debug {
-    fn is_online(&self) -> bool;
-    fn get(&mut self, folder: &Folder) -> Async<Result<Vec<Envelope>>>;
-    fn watch(
+pub trait MailBackend: ::std::fmt::Debug + Send + Sync {
+    fn capabilities(&self) -> MailBackendCapabilities;
+    fn is_online(&self) -> ResultFuture<()> {
+        Ok(Box::pin(async { Ok(()) }))
+    }
+
+    fn fetch(
+        &mut self,
+        mailbox_hash: MailboxHash,
+    ) -> Result<Pin<Box<dyn Stream<Item = Result<Vec<Envelope>>> + Send + 'static>>>;
+
+    fn refresh(&mut self, mailbox_hash: MailboxHash) -> ResultFuture<()>;
+    fn watch(&self) -> ResultFuture<()>;
+    fn mailboxes(&self) -> ResultFuture<HashMap<MailboxHash, Mailbox>>;
+    fn operation(&self, hash: EnvelopeHash) -> Result<Box<dyn BackendOp>>;
+
+    fn save(
         &self,
-        sender: RefreshEventConsumer,
-        work_context: WorkContext,
-    ) -> Result<std::thread::ThreadId>;
-    fn folders(&self) -> FnvHashMap<FolderHash, Folder>;
-    fn operation(&self, hash: EnvelopeHash, folder_hash: FolderHash) -> Box<dyn BackendOp>;
+        bytes: Vec<u8>,
+        mailbox_hash: MailboxHash,
+        flags: Option<Flag>,
+    ) -> ResultFuture<()>;
 
-    fn save(&self, bytes: &[u8], folder: &str, flags: Option<Flag>) -> Result<()>;
-    fn folder_operation(&mut self, _path: &str, _op: FolderOperation) -> Result<()> {
-        Ok(())
+    fn copy_messages(
+        &mut self,
+        env_hashes: EnvelopeHashBatch,
+        source_mailbox_hash: MailboxHash,
+        destination_mailbox_hash: MailboxHash,
+        move_: bool,
+    ) -> ResultFuture<()>;
+
+    fn set_flags(
+        &mut self,
+        env_hashes: EnvelopeHashBatch,
+        mailbox_hash: MailboxHash,
+        flags: SmallVec<[(std::result::Result<Flag, String>, bool); 8]>,
+    ) -> ResultFuture<()>;
+
+    fn delete_messages(
+        &mut self,
+        env_hashes: EnvelopeHashBatch,
+        mailbox_hash: MailboxHash,
+    ) -> ResultFuture<()>;
+
+    fn collection(&self) -> crate::Collection;
+    fn as_any(&self) -> &dyn Any;
+    fn as_any_mut(&mut self) -> &mut dyn Any;
+
+    fn create_mailbox(
+        &mut self,
+        _path: String,
+    ) -> ResultFuture<(MailboxHash, HashMap<MailboxHash, Mailbox>)> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn delete_mailbox(
+        &mut self,
+        _mailbox_hash: MailboxHash,
+    ) -> ResultFuture<HashMap<MailboxHash, Mailbox>> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn set_mailbox_subscription(
+        &mut self,
+        _mailbox_hash: MailboxHash,
+        _val: bool,
+    ) -> ResultFuture<()> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn rename_mailbox(
+        &mut self,
+        _mailbox_hash: MailboxHash,
+        _new_path: String,
+    ) -> ResultFuture<Mailbox> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn set_mailbox_permissions(
+        &mut self,
+        _mailbox_hash: MailboxHash,
+        _val: MailboxPermissions,
+    ) -> ResultFuture<()> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn search(
+        &self,
+        _query: crate::search::Query,
+        _mailbox_hash: Option<MailboxHash>,
+    ) -> ResultFuture<SmallVec<[EnvelopeHash; 512]>> {
+        Err(MeliError::new("Unimplemented."))
     }
 }
 
@@ -202,15 +408,15 @@ pub trait MailBackend: ::std::fmt::Debug {
 /// from (eg local or imap).
 ///
 /// # Creation
-/// ```no_run
+/// ```ignore
 /// /* Create operation from Backend */
 ///
-/// let op = backend.operation(message.hash(), mailbox.folder.hash());
+/// let op = backend.operation(message.hash(), mailbox.hash());
 /// ```
 ///
 /// # Example
-/// ```
-/// use melib::mailbox::backends::{BackendOp};
+/// ```ignore
+/// use melib::backends::{BackendOp};
 /// use melib::Result;
 /// use melib::{Envelope, Flag};
 ///
@@ -218,35 +424,19 @@ pub trait MailBackend: ::std::fmt::Debug {
 /// struct FooOp {}
 ///
 /// impl BackendOp for FooOp {
-///     fn description(&self) -> String {
-///         "Foobar".to_string()
-///     }
 ///     fn as_bytes(&mut self) -> Result<&[u8]> {
 ///         unimplemented!()
 ///     }
-///     fn fetch_headers(&mut self) -> Result<&[u8]> {
-///         unimplemented!()
-///     }
-///     fn fetch_body(&mut self) -> Result<&[u8]> {
-///         unimplemented!()
-///     }
-///     fn fetch_flags(&self) -> Flag {
+///     fn fetch_flags(&self) -> Result<Flag> {
 ///         unimplemented!()
 ///     }
 /// }
 ///
 /// let operation = Box::new(FooOp {});
-/// assert_eq!("Foobar", &operation.description());
 /// ```
 pub trait BackendOp: ::std::fmt::Debug + ::std::marker::Send {
-    fn description(&self) -> String;
-    fn as_bytes(&mut self) -> Result<&[u8]>;
-    //fn delete(&self) -> ();
-    //fn copy(&self
-    fn fetch_headers(&mut self) -> Result<&[u8]>;
-    fn fetch_body(&mut self) -> Result<&[u8]>;
-    fn fetch_flags(&self) -> Flag;
-    fn set_flag(&mut self, envelope: &mut Envelope, flag: Flag) -> Result<()>;
+    fn as_bytes(&mut self) -> ResultFuture<Vec<u8>>;
+    fn fetch_flags(&self) -> ResultFuture<Flag>;
 }
 
 /// Wrapper for BackendOps that are to be set read-only.
@@ -265,28 +455,16 @@ impl ReadOnlyOp {
 }
 
 impl BackendOp for ReadOnlyOp {
-    fn description(&self) -> String {
-        format!("read-only: {}", self.op.description())
-    }
-    fn as_bytes(&mut self) -> Result<&[u8]> {
+    fn as_bytes(&mut self) -> ResultFuture<Vec<u8>> {
         self.op.as_bytes()
     }
-    fn fetch_headers(&mut self) -> Result<&[u8]> {
-        self.op.fetch_headers()
-    }
-    fn fetch_body(&mut self) -> Result<&[u8]> {
-        self.op.fetch_body()
-    }
-    fn fetch_flags(&self) -> Flag {
+    fn fetch_flags(&self) -> ResultFuture<Flag> {
         self.op.fetch_flags()
     }
-    fn set_flag(&mut self, _envelope: &mut Envelope, _flag: Flag) -> Result<()> {
-        Err(MeliError::new("read-only set."))
-    }
 }
 
 #[derive(Debug, Copy, Hash, Eq, Clone, Serialize, Deserialize, PartialEq)]
-pub enum SpecialUseMailbox {
+pub enum SpecialUsageMailbox {
     Normal,
     Inbox,
     Archive,
@@ -297,67 +475,239 @@ pub enum SpecialUseMailbox {
     Trash,
 }
 
-pub trait BackendFolder: Debug {
-    fn hash(&self) -> FolderHash;
+impl std::fmt::Display for SpecialUsageMailbox {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        use SpecialUsageMailbox::*;
+        write!(
+            f,
+            "{}",
+            match self {
+                Normal => "Normal",
+                Inbox => "Inbox",
+                Archive => "Archive",
+                Drafts => "Drafts",
+                Flagged => "Flagged",
+                Junk => "Junk",
+                Sent => "Sent",
+                Trash => "Trash",
+            }
+        )
+    }
+}
+
+impl Default for SpecialUsageMailbox {
+    fn default() -> Self {
+        SpecialUsageMailbox::Normal
+    }
+}
+
+impl SpecialUsageMailbox {
+    pub fn detect_usage(name: &str) -> Option<SpecialUsageMailbox> {
+        if name.eq_ignore_ascii_case("inbox") {
+            Some(SpecialUsageMailbox::Inbox)
+        } else if name.eq_ignore_ascii_case("archive") {
+            Some(SpecialUsageMailbox::Archive)
+        } else if name.eq_ignore_ascii_case("drafts") {
+            Some(SpecialUsageMailbox::Drafts)
+        } else if name.eq_ignore_ascii_case("junk") || name.eq_ignore_ascii_case("spam") {
+            Some(SpecialUsageMailbox::Junk)
+        } else if name.eq_ignore_ascii_case("sent") {
+            Some(SpecialUsageMailbox::Sent)
+        } else if name.eq_ignore_ascii_case("trash") {
+            Some(SpecialUsageMailbox::Trash)
+        } else {
+            Some(SpecialUsageMailbox::Normal)
+        }
+    }
+}
+
+pub trait BackendMailbox: Debug {
+    fn hash(&self) -> MailboxHash;
     fn name(&self) -> &str;
-    /// Path of folder within the mailbox hierarchy, with `/` as separator.
+    /// Path of mailbox within the mailbox hierarchy, with `/` as separator.
     fn path(&self) -> &str;
     fn change_name(&mut self, new_name: &str);
-    fn clone(&self) -> Folder;
-    fn children(&self) -> &Vec<FolderHash>;
-    fn parent(&self) -> Option<FolderHash>;
+    fn clone(&self) -> Mailbox;
+    fn children(&self) -> &[MailboxHash];
+    fn parent(&self) -> Option<MailboxHash>;
+    fn is_subscribed(&self) -> bool;
+    fn set_is_subscribed(&mut self, new_val: bool) -> Result<()>;
+    fn set_special_usage(&mut self, new_val: SpecialUsageMailbox) -> Result<()>;
+    fn special_usage(&self) -> SpecialUsageMailbox;
+    fn permissions(&self) -> MailboxPermissions;
+    fn count(&self) -> Result<(usize, usize)>;
 }
 
-#[derive(Debug)]
-struct DummyFolder {
-    v: Vec<FolderHash>,
-}
+pub type AccountHash = u64;
+pub type MailboxHash = u64;
+pub type Mailbox = Box<dyn BackendMailbox + Send + Sync>;
 
-impl BackendFolder for DummyFolder {
-    fn hash(&self) -> FolderHash {
-        0
-    }
-
-    fn name(&self) -> &str {
-        ""
-    }
-
-    fn path(&self) -> &str {
-        ""
-    }
-
-    fn change_name(&mut self, _s: &str) {}
-
-    fn clone(&self) -> Folder {
-        folder_default()
-    }
-
-    fn children(&self) -> &Vec<FolderHash> {
-        &self.v
-    }
-
-    fn parent(&self) -> Option<FolderHash> {
-        None
-    }
-}
-
-pub fn folder_default() -> Folder {
-    Box::new(DummyFolder {
-        v: Vec::with_capacity(0),
-    })
-}
-
-pub type FolderHash = u64;
-pub type Folder = Box<dyn BackendFolder + Send + Sync>;
-
-impl Clone for Folder {
+impl Clone for Mailbox {
     fn clone(&self) -> Self {
-        BackendFolder::clone(self.deref())
+        BackendMailbox::clone(self.deref())
     }
 }
 
-impl Default for Folder {
+#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)]
+pub struct MailboxPermissions {
+    pub create_messages: bool,
+    pub remove_messages: bool,
+    pub set_flags: bool,
+    pub create_child: bool,
+    pub rename_messages: bool,
+    pub delete_messages: bool,
+    pub delete_mailbox: bool,
+    pub change_permissions: bool,
+}
+
+impl Default for MailboxPermissions {
     fn default() -> Self {
-        folder_default()
+        MailboxPermissions {
+            create_messages: false,
+            remove_messages: false,
+            set_flags: false,
+            create_child: false,
+            rename_messages: false,
+            delete_messages: false,
+            delete_mailbox: true,
+            change_permissions: false,
+        }
+    }
+}
+
+impl std::fmt::Display for MailboxPermissions {
+    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(fmt, "{:#?}", self)
+    }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct EnvelopeHashBatch {
+    pub first: EnvelopeHash,
+    pub rest: SmallVec<[EnvelopeHash; 64]>,
+}
+
+impl From<EnvelopeHash> for EnvelopeHashBatch {
+    fn from(value: EnvelopeHash) -> Self {
+        EnvelopeHashBatch {
+            first: value,
+            rest: SmallVec::new(),
+        }
+    }
+}
+
+impl std::convert::TryFrom<&[EnvelopeHash]> for EnvelopeHashBatch {
+    type Error = ();
+
+    fn try_from(value: &[EnvelopeHash]) -> std::result::Result<Self, Self::Error> {
+        if value.is_empty() {
+            return Err(());
+        }
+        Ok(EnvelopeHashBatch {
+            first: value[0],
+            rest: value[1..].iter().cloned().collect(),
+        })
+    }
+}
+
+impl EnvelopeHashBatch {
+    pub fn iter(&self) -> impl std::iter::Iterator<Item = EnvelopeHash> + '_ {
+        std::iter::once(self.first).chain(self.rest.iter().cloned())
+    }
+
+    pub fn len(&self) -> usize {
+        1 + self.rest.len()
+    }
+}
+
+#[derive(Default, Clone)]
+pub struct LazyCountSet {
+    not_yet_seen: usize,
+    set: BTreeSet<EnvelopeHash>,
+}
+
+impl fmt::Debug for LazyCountSet {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.debug_struct("LazyCountSet")
+            .field("not_yet_seen", &self.not_yet_seen)
+            .field("set", &self.set.len())
+            .field("total_len", &self.len())
+            .finish()
+    }
+}
+
+impl LazyCountSet {
+    pub fn set_not_yet_seen(&mut self, new_val: usize) {
+        self.not_yet_seen = new_val;
+    }
+
+    pub fn insert_existing(&mut self, new_val: EnvelopeHash) -> bool {
+        if self.not_yet_seen == 0 {
+            false
+        } else {
+            if !self.set.contains(&new_val) {
+                self.not_yet_seen -= 1;
+            }
+            self.set.insert(new_val);
+            true
+        }
+    }
+
+    pub fn insert_existing_set(&mut self, set: BTreeSet<EnvelopeHash>) {
+        let old_len = self.set.len();
+        self.set.extend(set.into_iter());
+        self.not_yet_seen = self.not_yet_seen.saturating_sub(self.set.len() - old_len);
+    }
+
+    #[inline(always)]
+    pub fn len(&self) -> usize {
+        self.set.len() + self.not_yet_seen
+    }
+
+    #[inline(always)]
+    pub fn clear(&mut self) {
+        self.set.clear();
+        self.not_yet_seen = 0;
+    }
+
+    pub fn insert_new(&mut self, new_val: EnvelopeHash) {
+        self.set.insert(new_val);
+    }
+
+    pub fn insert_set(&mut self, set: BTreeSet<EnvelopeHash>) {
+        self.set.extend(set.into_iter());
+    }
+
+    pub fn remove(&mut self, env_hash: EnvelopeHash) -> bool {
+        self.set.remove(&env_hash)
+    }
+}
+
+#[test]
+fn test_lazy_count_set() {
+    let mut new = LazyCountSet::default();
+    assert_eq!(new.len(), 0);
+    new.set_not_yet_seen(10);
+    assert_eq!(new.len(), 10);
+    for i in 0..10 {
+        assert!(new.insert_existing(i));
+    }
+    assert_eq!(new.len(), 10);
+    assert!(!new.insert_existing(10));
+    assert_eq!(new.len(), 10);
+}
+
+pub struct IsSubscribedFn(Box<dyn Fn(&str) -> bool + Send + Sync>);
+
+impl std::fmt::Debug for IsSubscribedFn {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "IsSubscribedFn Box")
+    }
+}
+
+impl std::ops::Deref for IsSubscribedFn {
+    type Target = Box<dyn Fn(&str) -> bool + Send + Sync>;
+    fn deref(&self) -> &Box<dyn Fn(&str) -> bool + Send + Sync> {
+        &self.0
     }
 }

melib/src/backends/imap/cache.rs

@@ -0,0 +1,751 @@
+/*
+ * meli - imap melib
+ *
+ * Copyright 2020 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+mod sync;
+use crate::{
+    backends::MailboxHash,
+    email::{Envelope, EnvelopeHash},
+    error::*,
+};
+use std::convert::TryFrom;
+
+#[derive(Debug, PartialEq, Hash, Eq, Ord, PartialOrd, Copy, Clone)]
+pub struct ModSequence(pub std::num::NonZeroU64);
+
+impl TryFrom<i64> for ModSequence {
+    type Error = ();
+    fn try_from(val: i64) -> std::result::Result<ModSequence, ()> {
+        std::num::NonZeroU64::new(val as u64)
+            .map(|u| Ok(ModSequence(u)))
+            .unwrap_or(Err(()))
+    }
+}
+
+impl core::fmt::Display for ModSequence {
+    fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
+        write!(fmt, "{}", &self.0)
+    }
+}
+
+#[derive(Debug)]
+pub struct CachedEnvelope {
+    pub inner: Envelope,
+    pub uid: UID,
+    pub mailbox_hash: MailboxHash,
+    pub modsequence: Option<ModSequence>,
+}
+
+pub trait ImapCache: Send + core::fmt::Debug {
+    fn reset(&mut self) -> Result<()>;
+    fn mailbox_state(&mut self, mailbox_hash: MailboxHash) -> Result<Option<()>>;
+
+    fn find_envelope(
+        &mut self,
+        identifier: std::result::Result<UID, EnvelopeHash>,
+        mailbox_hash: MailboxHash,
+    ) -> Result<Option<CachedEnvelope>>;
+
+    fn update(
+        &mut self,
+        mailbox_hash: MailboxHash,
+        refresh_events: &[(UID, RefreshEvent)],
+    ) -> Result<()>;
+
+    fn update_mailbox(
+        &mut self,
+        mailbox_hash: MailboxHash,
+        select_response: &SelectResponse,
+    ) -> Result<()>;
+
+    fn insert_envelopes(
+        &mut self,
+        mailbox_hash: MailboxHash,
+        fetches: &[FetchResponse<'_>],
+    ) -> Result<()>;
+
+    fn envelopes(&mut self, mailbox_hash: MailboxHash) -> Result<Option<Vec<EnvelopeHash>>>;
+
+    fn clear(&mut self, mailbox_hash: MailboxHash, select_response: &SelectResponse) -> Result<()>;
+
+    fn rfc822(
+        &mut self,
+        identifier: std::result::Result<UID, EnvelopeHash>,
+        mailbox_hash: MailboxHash,
+    ) -> Result<Option<Vec<u8>>>;
+}
+
+#[cfg(feature = "sqlite3")]
+pub use sqlite3_m::*;
+
+#[cfg(feature = "sqlite3")]
+mod sqlite3_m {
+    use super::*;
+    use crate::sqlite3::rusqlite::types::{
+        FromSql, FromSqlError, FromSqlResult, ToSql, ToSqlOutput,
+    };
+    use crate::sqlite3::{self, DatabaseDescription};
+
+    type Sqlite3UID = i32;
+
+    #[derive(Debug)]
+    pub struct Sqlite3Cache {
+        connection: crate::sqlite3::Connection,
+        loaded_mailboxes: BTreeSet<MailboxHash>,
+        uid_store: Arc<UIDStore>,
+    }
+
+    const DB_DESCRIPTION: DatabaseDescription = DatabaseDescription {
+        name: "header_cache.db",
+        init_script: Some(
+            "PRAGMA foreign_keys = true;
+    PRAGMA encoding = 'UTF-8';
+
+    CREATE TABLE IF NOT EXISTS envelopes (
+                    hash             INTEGER NOT NULL,
+                    mailbox_hash     INTEGER NOT NULL,
+                    uid              INTEGER NOT NULL,
+                    modsequence      INTEGER,
+                    rfc822           BLOB,
+                    envelope         BLOB NOT NULL,
+                    PRIMARY KEY (mailbox_hash, uid),
+                    FOREIGN KEY (mailbox_hash) REFERENCES mailbox(mailbox_hash) ON DELETE CASCADE
+                   );
+    CREATE TABLE IF NOT EXISTS mailbox (
+                mailbox_hash     INTEGER UNIQUE,
+                uidvalidity      INTEGER,
+                flags            BLOB NOT NULL,
+                highestmodseq    INTEGER,
+                PRIMARY KEY (mailbox_hash)
+               );
+    CREATE INDEX IF NOT EXISTS envelope_uid_idx ON envelopes(mailbox_hash, uid);
+    CREATE INDEX IF NOT EXISTS envelope_idx ON envelopes(hash);
+    CREATE INDEX IF NOT EXISTS mailbox_idx ON mailbox(mailbox_hash);",
+        ),
+        version: 2,
+    };
+
+    impl ToSql for ModSequence {
+        fn to_sql(&self) -> rusqlite::Result<ToSqlOutput> {
+            Ok(ToSqlOutput::from(self.0.get() as i64))
+        }
+    }
+
+    impl FromSql for ModSequence {
+        fn column_result(value: rusqlite::types::ValueRef) -> FromSqlResult<Self> {
+            let i: i64 = FromSql::column_result(value)?;
+            if i == 0 {
+                return Err(FromSqlError::OutOfRange(0));
+            }
+            Ok(ModSequence::try_from(i).unwrap())
+        }
+    }
+
+    impl Sqlite3Cache {
+        pub fn get(uid_store: Arc<UIDStore>) -> Result<Box<dyn ImapCache>> {
+            Ok(Box::new(Self {
+                connection: sqlite3::open_or_create_db(
+                    &DB_DESCRIPTION,
+                    Some(uid_store.account_name.as_str()),
+                )?,
+                loaded_mailboxes: BTreeSet::default(),
+                uid_store,
+            }))
+        }
+
+        fn max_uid(&self, mailbox_hash: MailboxHash) -> Result<UID> {
+            let mut stmt = self
+                .connection
+                .prepare("SELECT MAX(uid) FROM envelopes WHERE mailbox_hash = ?1;")?;
+
+            let mut ret: Vec<UID> = stmt
+                .query_map(sqlite3::params![mailbox_hash as i64], |row| {
+                    Ok(row.get(0).map(|i: Sqlite3UID| i as UID)?)
+                })?
+                .collect::<std::result::Result<_, _>>()?;
+            Ok(ret.pop().unwrap_or(0))
+        }
+    }
+
+    impl ImapCache for Sqlite3Cache {
+        fn reset(&mut self) -> Result<()> {
+            sqlite3::reset_db(&DB_DESCRIPTION, Some(self.uid_store.account_name.as_str()))
+        }
+
+        fn mailbox_state(&mut self, mailbox_hash: MailboxHash) -> Result<Option<()>> {
+            if self.loaded_mailboxes.contains(&mailbox_hash) {
+                return Ok(Some(()));
+            }
+            debug!("loading mailbox state {} from cache", mailbox_hash);
+            let mut stmt = self.connection.prepare(
+                "SELECT uidvalidity, flags, highestmodseq FROM mailbox WHERE mailbox_hash = ?1;",
+            )?;
+
+            let mut ret = stmt.query_map(sqlite3::params![mailbox_hash as i64], |row| {
+                Ok((
+                    row.get(0).map(|u: Sqlite3UID| u as UID)?,
+                    row.get(1)?,
+                    row.get(2)?,
+                ))
+            })?;
+            if let Some(v) = ret.next() {
+                let (uidvalidity, flags, highestmodseq): (
+                    UIDVALIDITY,
+                    Vec<u8>,
+                    Option<ModSequence>,
+                ) = v?;
+                debug!(
+                    "mailbox state {} in cache uidvalidity {}",
+                    mailbox_hash, uidvalidity
+                );
+                debug!(
+                    "mailbox state {} in cache highestmodseq {:?}",
+                    mailbox_hash, &highestmodseq
+                );
+                debug!(
+                    "mailbox state {} inserting flags: {:?}",
+                    mailbox_hash,
+                    to_str!(&flags)
+                );
+                self.uid_store
+                    .highestmodseqs
+                    .lock()
+                    .unwrap()
+                    .entry(mailbox_hash)
+                    .and_modify(|entry| *entry = highestmodseq.ok_or(()))
+                    .or_insert(highestmodseq.ok_or(()));
+                self.uid_store
+                    .uidvalidity
+                    .lock()
+                    .unwrap()
+                    .entry(mailbox_hash)
+                    .and_modify(|entry| *entry = uidvalidity)
+                    .or_insert(uidvalidity);
+                let mut tag_lck = self.uid_store.collection.tag_index.write().unwrap();
+                for f in to_str!(&flags).split('\0') {
+                    let hash = tag_hash!(f);
+                    //debug!("hash {} flag {}", hash, &f);
+                    if !tag_lck.contains_key(&hash) {
+                        tag_lck.insert(hash, f.to_string());
+                    }
+                }
+                self.loaded_mailboxes.insert(mailbox_hash);
+                Ok(Some(()))
+            } else {
+                debug!("mailbox state {} not in cache", mailbox_hash);
+                Ok(None)
+            }
+        }
+
+        fn clear(
+            &mut self,
+            mailbox_hash: MailboxHash,
+            select_response: &SelectResponse,
+        ) -> Result<()> {
+            debug!("clear mailbox_hash {} {:?}", mailbox_hash, select_response);
+            self.loaded_mailboxes.remove(&mailbox_hash);
+            self.connection
+                .execute(
+                    "DELETE FROM mailbox WHERE mailbox_hash = ?1",
+                    sqlite3::params![mailbox_hash as i64],
+                )
+                .chain_err_summary(|| {
+                    format!(
+                        "Could not clear cache of mailbox {} account {}",
+                        mailbox_hash, self.uid_store.account_name
+                    )
+                })?;
+
+            if let Some(Ok(highestmodseq)) = select_response.highestmodseq {
+                self.connection.execute(
+                "INSERT OR IGNORE INTO mailbox (uidvalidity, flags, highestmodseq, mailbox_hash) VALUES (?1, ?2, ?3, ?4)",
+                sqlite3::params![select_response.uidvalidity as Sqlite3UID, select_response.flags.1.iter().map(|s| s.as_str()).collect::<Vec<&str>>().join("\0").as_bytes(), highestmodseq, mailbox_hash as i64],
+            )
+            .chain_err_summary(|| {
+                format!(
+                    "Could not insert uidvalidity {} in header_cache of account {}",
+                    select_response.uidvalidity, self.uid_store.account_name
+                )
+            })?;
+            } else {
+                self.connection
+                    .execute(
+                        "INSERT OR IGNORE INTO mailbox (uidvalidity, flags, mailbox_hash) VALUES (?1, ?2, ?3)",
+                        sqlite3::params![
+                            select_response.uidvalidity as Sqlite3UID,
+                            select_response.flags.1.iter().map(|s| s.as_str()).collect::<Vec<&str>>().join("\0").as_bytes(),
+                            mailbox_hash as i64
+                        ],
+                    )
+                    .chain_err_summary(|| {
+                        format!(
+                            "Could not insert mailbox {} in header_cache of account {}",
+                            select_response.uidvalidity, self.uid_store.account_name
+                        )
+                    })?;
+            }
+            Ok(())
+        }
+
+        fn update_mailbox(
+            &mut self,
+            mailbox_hash: MailboxHash,
+            select_response: &SelectResponse,
+        ) -> Result<()> {
+            if self.mailbox_state(mailbox_hash)?.is_none() {
+                return self.clear(mailbox_hash, select_response);
+            }
+
+            if let Some(Ok(highestmodseq)) = select_response.highestmodseq {
+                self.connection
+                    .execute(
+                        "UPDATE mailbox SET flags=?1, highestmodseq =?2 where mailbox_hash = ?3;",
+                        sqlite3::params![
+                            select_response
+                                .flags
+                                .1
+                                .iter()
+                                .map(|s| s.as_str())
+                                .collect::<Vec<&str>>()
+                                .join("\0")
+                                .as_bytes(),
+                            highestmodseq,
+                            mailbox_hash as i64
+                        ],
+                    )
+                    .chain_err_summary(|| {
+                        format!(
+                            "Could not update mailbox {} in header_cache of account {}",
+                            mailbox_hash, self.uid_store.account_name
+                        )
+                    })?;
+            } else {
+                self.connection
+                    .execute(
+                        "UPDATE mailbox SET flags=?1 where mailbox_hash = ?2;",
+                        sqlite3::params![
+                            select_response
+                                .flags
+                                .1
+                                .iter()
+                                .map(|s| s.as_str())
+                                .collect::<Vec<&str>>()
+                                .join("\0")
+                                .as_bytes(),
+                            mailbox_hash as i64
+                        ],
+                    )
+                    .chain_err_summary(|| {
+                        format!(
+                            "Could not update mailbox {} in header_cache of account {}",
+                            mailbox_hash, self.uid_store.account_name
+                        )
+                    })?;
+            }
+            Ok(())
+        }
+
+        fn envelopes(&mut self, mailbox_hash: MailboxHash) -> Result<Option<Vec<EnvelopeHash>>> {
+            debug!("envelopes mailbox_hash {}", mailbox_hash);
+            if self.mailbox_state(mailbox_hash)?.is_none() {
+                return Ok(None);
+            }
+
+            let mut stmt = self.connection.prepare(
+                "SELECT uid, envelope, modsequence FROM envelopes WHERE mailbox_hash = ?1;",
+            )?;
+
+            let ret: Vec<(UID, Envelope, Option<ModSequence>)> = stmt
+                .query_map(sqlite3::params![mailbox_hash as i64], |row| {
+                    Ok((
+                        row.get(0).map(|i: Sqlite3UID| i as UID)?,
+                        row.get(1)?,
+                        row.get(2)?,
+                    ))
+                })?
+                .collect::<std::result::Result<_, _>>()?;
+            let mut max_uid = 0;
+            let mut env_lck = self.uid_store.envelopes.lock().unwrap();
+            let mut hash_index_lck = self.uid_store.hash_index.lock().unwrap();
+            let mut uid_index_lck = self.uid_store.uid_index.lock().unwrap();
+            let mut env_hashes = Vec::with_capacity(ret.len());
+            for (uid, env, modseq) in ret {
+                env_hashes.push(env.hash());
+                max_uid = std::cmp::max(max_uid, uid);
+                hash_index_lck.insert(env.hash(), (uid, mailbox_hash));
+                uid_index_lck.insert((mailbox_hash, uid), env.hash());
+                env_lck.insert(
+                    env.hash(),
+                    CachedEnvelope {
+                        inner: env,
+                        uid,
+                        mailbox_hash,
+                        modsequence: modseq,
+                    },
+                );
+            }
+            self.uid_store
+                .max_uids
+                .lock()
+                .unwrap()
+                .insert(mailbox_hash, max_uid);
+            Ok(Some(env_hashes))
+        }
+
+        fn insert_envelopes(
+            &mut self,
+            mailbox_hash: MailboxHash,
+            fetches: &[FetchResponse<'_>],
+        ) -> Result<()> {
+            debug!(
+                "insert_envelopes mailbox_hash {} len {}",
+                mailbox_hash,
+                fetches.len()
+            );
+            let mut max_uid = self
+                .uid_store
+                .max_uids
+                .lock()
+                .unwrap()
+                .get(&mailbox_hash)
+                .cloned()
+                .unwrap_or_default();
+            if self.mailbox_state(mailbox_hash)?.is_none() {
+                return Err(MeliError::new("Mailbox is not in cache").set_kind(ErrorKind::Bug));
+            }
+            let Self {
+                ref mut connection,
+                ref uid_store,
+                loaded_mailboxes: _,
+            } = self;
+            let tx = connection.transaction()?;
+            for item in fetches {
+                if let FetchResponse {
+                    uid: Some(uid),
+                    message_sequence_number: _,
+                    modseq,
+                    flags: _,
+                    body: _,
+                    references: _,
+                    envelope: Some(envelope),
+                    raw_fetch_value: _,
+                } = item
+                {
+                    max_uid = std::cmp::max(max_uid, *uid);
+                    tx.execute(
+                "INSERT OR REPLACE INTO envelopes (hash, uid, mailbox_hash, modsequence, envelope) VALUES (?1, ?2, ?3, ?4, ?5)",
+                sqlite3::params![envelope.hash() as i64, *uid as Sqlite3UID, mailbox_hash as i64, modseq, &envelope],
+            ).chain_err_summary(|| format!("Could not insert envelope {} {} in header_cache of account {}", envelope.message_id(), envelope.hash(), uid_store.account_name))?;
+                }
+            }
+            tx.commit()?;
+            self.uid_store
+                .max_uids
+                .lock()
+                .unwrap()
+                .insert(mailbox_hash, max_uid);
+            Ok(())
+        }
+
+        fn update(
+            &mut self,
+            mailbox_hash: MailboxHash,
+            refresh_events: &[(UID, RefreshEvent)],
+        ) -> Result<()> {
+            if self.mailbox_state(mailbox_hash)?.is_none() {
+                return Err(MeliError::new("Mailbox is not in cache").set_kind(ErrorKind::Bug));
+            }
+            let Self {
+                ref mut connection,
+                ref uid_store,
+                loaded_mailboxes: _,
+            } = self;
+            let tx = connection.transaction()?;
+            let mut hash_index_lck = uid_store.hash_index.lock().unwrap();
+            for (uid, event) in refresh_events {
+                match &event.kind {
+                    RefreshEventKind::Remove(env_hash) => {
+                        hash_index_lck.remove(&env_hash);
+                        tx.execute(
+                            "DELETE FROM envelopes WHERE mailbox_hash = ?1 AND uid = ?2;",
+                            sqlite3::params![mailbox_hash as i64, *uid as Sqlite3UID],
+                        )
+                        .chain_err_summary(|| {
+                            format!(
+                                "Could not remove envelope {} uid {} from  mailbox {} account {}",
+                                env_hash, *uid, mailbox_hash, uid_store.account_name
+                            )
+                        })?;
+                    }
+                    RefreshEventKind::NewFlags(env_hash, (flags, tags)) => {
+                        let mut stmt = tx.prepare(
+                            "SELECT envelope FROM envelopes WHERE mailbox_hash = ?1 AND uid = ?2;",
+                        )?;
+
+                        let mut ret: Vec<Envelope> = stmt
+                            .query_map(
+                                sqlite3::params![mailbox_hash as i64, *uid as Sqlite3UID],
+                                |row| Ok(row.get(0)?),
+                            )?
+                            .collect::<std::result::Result<_, _>>()?;
+                        if let Some(mut env) = ret.pop() {
+                            env.set_flags(*flags);
+                            env.labels_mut().clear();
+                            env.labels_mut().extend(tags.iter().map(|t| tag_hash!(t)));
+                            tx.execute(
+                    "UPDATE envelopes SET envelope = ?1 WHERE mailbox_hash = ?2 AND uid = ?3;",
+                    sqlite3::params![&env, mailbox_hash as i64, *uid as Sqlite3UID],
+                )
+                                .chain_err_summary(|| {
+                                    format!(
+                                        "Could not update envelope {} uid {} from  mailbox {} account {}",
+                                        env_hash, *uid, mailbox_hash, uid_store.account_name
+                                    )
+                                })?;
+                            uid_store
+                                .envelopes
+                                .lock()
+                                .unwrap()
+                                .entry(*env_hash)
+                                .and_modify(|entry| {
+                                    entry.inner = env;
+                                });
+                        }
+                    }
+                    _ => {}
+                }
+            }
+            tx.commit()?;
+            let new_max_uid = self.max_uid(mailbox_hash).unwrap_or(0);
+            self.uid_store
+                .max_uids
+                .lock()
+                .unwrap()
+                .insert(mailbox_hash, new_max_uid);
+            Ok(())
+        }
+
+        fn find_envelope(
+            &mut self,
+            identifier: std::result::Result<UID, EnvelopeHash>,
+            mailbox_hash: MailboxHash,
+        ) -> Result<Option<CachedEnvelope>> {
+            let mut ret: Vec<(UID, Envelope, Option<ModSequence>)> = match identifier {
+                Ok(uid) => {
+                    let mut stmt = self.connection.prepare(
+                "SELECT uid, envelope, modsequence FROM envelopes WHERE mailbox_hash = ?1 AND uid = ?2;",
+            )?;
+
+                    let x = stmt
+                        .query_map(
+                            sqlite3::params![mailbox_hash as i64, uid as Sqlite3UID],
+                            |row| {
+                                Ok((
+                                    row.get(0).map(|u: Sqlite3UID| u as UID)?,
+                                    row.get(1)?,
+                                    row.get(2)?,
+                                ))
+                            },
+                        )?
+                        .collect::<std::result::Result<_, _>>()?;
+                    x
+                }
+                Err(env_hash) => {
+                    let mut stmt = self.connection.prepare(
+                "SELECT uid, envelope, modsequence FROM envelopes WHERE mailbox_hash = ?1 AND hash = ?2;",
+            )?;
+
+                    let x = stmt
+                        .query_map(
+                            sqlite3::params![mailbox_hash as i64, env_hash as i64],
+                            |row| {
+                                Ok((
+                                    row.get(0).map(|u: Sqlite3UID| u as UID)?,
+                                    row.get(1)?,
+                                    row.get(2)?,
+                                ))
+                            },
+                        )?
+                        .collect::<std::result::Result<_, _>>()?;
+                    x
+                }
+            };
+            if ret.len() != 1 {
+                return Ok(None);
+            }
+            let (uid, inner, modsequence) = ret.pop().unwrap();
+            return Ok(Some(CachedEnvelope {
+                inner,
+                uid,
+                mailbox_hash,
+                modsequence,
+            }));
+        }
+
+        fn rfc822(
+            &mut self,
+            identifier: std::result::Result<UID, EnvelopeHash>,
+            mailbox_hash: MailboxHash,
+        ) -> Result<Option<Vec<u8>>> {
+            let mut ret: Vec<Option<Vec<u8>>> = match identifier {
+                Ok(uid) => {
+                    let mut stmt = self.connection.prepare(
+                        "SELECT rfc822 FROM envelopes WHERE mailbox_hash = ?1 AND uid = ?2;",
+                    )?;
+                    let x = stmt
+                        .query_map(
+                            sqlite3::params![mailbox_hash as i64, uid as Sqlite3UID],
+                            |row| Ok(row.get(0)?),
+                        )?
+                        .collect::<std::result::Result<_, _>>()?;
+                    x
+                }
+                Err(env_hash) => {
+                    let mut stmt = self.connection.prepare(
+                        "SELECT rfc822 FROM envelopes WHERE mailbox_hash = ?1 AND hash = ?2;",
+                    )?;
+                    let x = stmt
+                        .query_map(
+                            sqlite3::params![mailbox_hash as i64, env_hash as i64],
+                            |row| Ok(row.get(0)?),
+                        )?
+                        .collect::<std::result::Result<_, _>>()?;
+                    x
+                }
+            };
+
+            if ret.len() != 1 {
+                return Ok(None);
+            }
+            Ok(ret.pop().unwrap())
+        }
+    }
+}
+
+pub(super) async fn fetch_cached_envs(state: &mut FetchState) -> Result<Option<Vec<Envelope>>> {
+    let FetchState {
+        stage: _,
+        ref mut connection,
+        mailbox_hash,
+        ref uid_store,
+        cache_handle: _,
+    } = state;
+    let mailbox_hash = *mailbox_hash;
+    if !uid_store.keep_offline_cache {
+        return Ok(None);
+    }
+    {
+        let mut conn = connection.lock().await;
+        match conn.load_cache(mailbox_hash).await {
+            None => return Ok(None),
+            Some(Ok(env_hashes)) => {
+                let env_lck = uid_store.envelopes.lock().unwrap();
+                return Ok(Some(
+                    env_hashes
+                        .into_iter()
+                        .filter_map(|env_hash| {
+                            env_lck.get(&env_hash).map(|c_env| c_env.inner.clone())
+                        })
+                        .collect::<Vec<Envelope>>(),
+                ));
+            }
+            Some(Err(err)) => return Err(err),
+        }
+    }
+}
+
+#[cfg(not(feature = "sqlite3"))]
+pub use default_m::*;
+
+#[cfg(not(feature = "sqlite3"))]
+mod default_m {
+    use super::*;
+    #[derive(Debug)]
+    pub struct DefaultCache;
+
+    impl DefaultCache {
+        pub fn get(_uid_store: Arc<UIDStore>) -> Result<Box<dyn ImapCache>> {
+            Ok(Box::new(Self))
+        }
+    }
+
+    impl ImapCache for DefaultCache {
+        fn reset(&mut self) -> Result<()> {
+            Err(MeliError::new("melib is not built with any imap cache").set_kind(ErrorKind::Bug))
+        }
+
+        fn mailbox_state(&mut self, _mailbox_hash: MailboxHash) -> Result<Option<()>> {
+            Err(MeliError::new("melib is not built with any imap cache").set_kind(ErrorKind::Bug))
+        }
+
+        fn clear(
+            &mut self,
+            _mailbox_hash: MailboxHash,
+            _select_response: &SelectResponse,
+        ) -> Result<()> {
+            Err(MeliError::new("melib is not built with any imap cache").set_kind(ErrorKind::Bug))
+        }
+
+        fn envelopes(&mut self, _mailbox_hash: MailboxHash) -> Result<Option<Vec<EnvelopeHash>>> {
+            Err(MeliError::new("melib is not built with any imap cache").set_kind(ErrorKind::Bug))
+        }
+
+        fn insert_envelopes(
+            &mut self,
+            _mailbox_hash: MailboxHash,
+            _fetches: &[FetchResponse<'_>],
+        ) -> Result<()> {
+            Err(MeliError::new("melib is not built with any imap cache").set_kind(ErrorKind::Bug))
+        }
+
+        fn update_mailbox(
+            &mut self,
+            _mailbox_hash: MailboxHash,
+            _select_response: &SelectResponse,
+        ) -> Result<()> {
+            Err(MeliError::new("melib is not built with any imap cache").set_kind(ErrorKind::Bug))
+        }
+
+        fn update(
+            &mut self,
+            _mailbox_hash: MailboxHash,
+            _refresh_events: &[(UID, RefreshEvent)],
+        ) -> Result<()> {
+            Err(MeliError::new("melib is not built with any imap cache").set_kind(ErrorKind::Bug))
+        }
+
+        fn find_envelope(
+            &mut self,
+            _identifier: std::result::Result<UID, EnvelopeHash>,
+            _mailbox_hash: MailboxHash,
+        ) -> Result<Option<CachedEnvelope>> {
+            Err(MeliError::new("melib is not built with any imap cache").set_kind(ErrorKind::Bug))
+        }
+
+        fn rfc822(
+            &mut self,
+            _identifier: std::result::Result<UID, EnvelopeHash>,
+            _mailbox_hash: MailboxHash,
+        ) -> Result<Option<Vec<u8>>> {
+            Err(MeliError::new("melib is not built with any imap cache").set_kind(ErrorKind::Bug))
+        }
+    }
+}

melib/src/backends/imap/cache/sync.rs

@@ -0,0 +1,701 @@
+/*
+ * melib - IMAP
+ *
+ * Copyright 2020 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+
+impl ImapConnection {
+    pub async fn resync(&mut self, mailbox_hash: MailboxHash) -> Result<Option<Vec<Envelope>>> {
+        debug!("resync mailbox_hash {}", mailbox_hash);
+        debug!(&self.sync_policy);
+        if let SyncPolicy::None = self.sync_policy {
+            return Ok(None);
+        }
+
+        #[cfg(not(feature = "sqlite3"))]
+        let mut cache_handle = DefaultCache::get(self.uid_store.clone())?;
+        #[cfg(feature = "sqlite3")]
+        let mut cache_handle = Sqlite3Cache::get(self.uid_store.clone())?;
+        if cache_handle.mailbox_state(mailbox_hash)?.is_none() {
+            return Ok(None);
+        }
+
+        match self.sync_policy {
+            SyncPolicy::None => Ok(None),
+            SyncPolicy::Basic => self.resync_basic(cache_handle, mailbox_hash).await,
+            SyncPolicy::Condstore => self.resync_condstore(cache_handle, mailbox_hash).await,
+            SyncPolicy::CondstoreQresync => {
+                self.resync_condstoreqresync(cache_handle, mailbox_hash)
+                    .await
+            }
+        }
+    }
+
+    pub async fn load_cache(
+        &mut self,
+        mailbox_hash: MailboxHash,
+    ) -> Option<Result<Vec<EnvelopeHash>>> {
+        debug!("load_cache {}", mailbox_hash);
+        #[cfg(not(feature = "sqlite3"))]
+        let mut cache_handle = match DefaultCache::get(self.uid_store.clone()) {
+            Ok(v) => v,
+            Err(err) => return Some(Err(err)),
+        };
+        #[cfg(feature = "sqlite3")]
+        let mut cache_handle = match Sqlite3Cache::get(self.uid_store.clone()) {
+            Ok(v) => v,
+            Err(err) => return Some(Err(err)),
+        };
+        match cache_handle.mailbox_state(mailbox_hash) {
+            Err(err) => return Some(Err(err)),
+            Ok(Some(())) => {}
+            Ok(None) => {
+                return None;
+            }
+        };
+        match cache_handle.envelopes(mailbox_hash) {
+            Ok(Some(envs)) => Some(Ok(envs)),
+            Ok(None) => None,
+            Err(err) => Some(Err(err)),
+        }
+    }
+
+    //rfc4549_Synchronization_Operations_for_Disconnected_IMAP4_Clients
+    pub async fn resync_basic(
+        &mut self,
+        mut cache_handle: Box<dyn ImapCache>,
+        mailbox_hash: MailboxHash,
+    ) -> Result<Option<Vec<Envelope>>> {
+        let mut payload = vec![];
+        debug!("resync_basic");
+        let mut response = Vec::with_capacity(8 * 1024);
+        let cached_uidvalidity = self
+            .uid_store
+            .uidvalidity
+            .lock()
+            .unwrap()
+            .get(&mailbox_hash)
+            .cloned();
+        let cached_max_uid = self
+            .uid_store
+            .max_uids
+            .lock()
+            .unwrap()
+            .get(&mailbox_hash)
+            .cloned();
+        // 3.  tag2 UID FETCH 1:<lastseenuid> FLAGS
+        if cached_uidvalidity.is_none() || cached_max_uid.is_none() {
+            return Ok(None);
+        }
+
+        let current_uidvalidity: UID = cached_uidvalidity.unwrap();
+        let max_uid: UID = cached_max_uid.unwrap();
+        let (mailbox_path, mailbox_exists, unseen) = {
+            let f = &self.uid_store.mailboxes.lock().await[&mailbox_hash];
+            (
+                f.imap_path().to_string(),
+                f.exists.clone(),
+                f.unseen.clone(),
+            )
+        };
+        let mut new_unseen = BTreeSet::default();
+        let select_response = self
+            .select_mailbox(mailbox_hash, &mut response, true)
+            .await?
+            .unwrap();
+        // 1. check UIDVALIDITY. If fail, discard cache and rebuild
+        if select_response.uidvalidity != current_uidvalidity {
+            cache_handle.clear(mailbox_hash, &select_response)?;
+            return Ok(None);
+        }
+        cache_handle.update_mailbox(mailbox_hash, &select_response)?;
+
+        // 2.  tag1 UID FETCH <lastseenuid+1>:* <descriptors>
+        self.send_command(
+            format!(
+                "UID FETCH {}:* (UID FLAGS ENVELOPE BODY.PEEK[HEADER.FIELDS (REFERENCES)] BODYSTRUCTURE)",
+                max_uid + 1
+            )
+            .as_bytes(),
+        )
+        .await?;
+        self.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
+            .await?;
+        debug!(
+            "fetch response is {} bytes and {} lines",
+            response.len(),
+            String::from_utf8_lossy(&response).lines().count()
+        );
+        let (_, mut v, _) = protocol_parser::fetch_responses(&response)?;
+        debug!("responses len is {}", v.len());
+        for FetchResponse {
+            ref uid,
+            ref mut envelope,
+            ref mut flags,
+            ref references,
+            ..
+        } in v.iter_mut()
+        {
+            let uid = uid.unwrap();
+            let env = envelope.as_mut().unwrap();
+            env.set_hash(generate_envelope_hash(&mailbox_path, &uid));
+            if let Some(value) = references {
+                env.set_references(value);
+            }
+            let mut tag_lck = self.uid_store.collection.tag_index.write().unwrap();
+            if let Some((flags, keywords)) = flags {
+                env.set_flags(*flags);
+                if !env.is_seen() {
+                    new_unseen.insert(env.hash());
+                }
+                for f in keywords {
+                    let hash = tag_hash!(f);
+                    if !tag_lck.contains_key(&hash) {
+                        tag_lck.insert(hash, f.to_string());
+                    }
+                    env.labels_mut().push(hash);
+                }
+            }
+        }
+        {
+            cache_handle
+                .insert_envelopes(mailbox_hash, &v)
+                .chain_err_summary(|| {
+                    format!(
+                        "Could not save envelopes in cache for mailbox {}",
+                        mailbox_path
+                    )
+                })?;
+        }
+
+        for FetchResponse {
+            uid,
+            message_sequence_number: _,
+            envelope,
+            ..
+        } in v
+        {
+            let uid = uid.unwrap();
+            let env = envelope.unwrap();
+            /*
+            debug!(
+                "env hash {} {} UID = {} MSN = {}",
+                env.hash(),
+                env.subject(),
+                uid,
+                message_sequence_number
+            );
+            */
+            self.uid_store
+                .hash_index
+                .lock()
+                .unwrap()
+                .insert(env.hash(), (uid, mailbox_hash));
+            self.uid_store
+                .uid_index
+                .lock()
+                .unwrap()
+                .insert((mailbox_hash, uid), env.hash());
+            payload.push((uid, env));
+        }
+        debug!("sending payload for {}", mailbox_hash);
+        let payload_hash_set: BTreeSet<_> =
+            payload.iter().map(|(_, env)| env.hash()).collect::<_>();
+        {
+            let mut unseen_lck = unseen.lock().unwrap();
+            for &seen_env_hash in payload_hash_set.difference(&new_unseen) {
+                unseen_lck.remove(seen_env_hash);
+            }
+
+            unseen_lck.insert_set(new_unseen);
+        }
+        mailbox_exists.lock().unwrap().insert_set(payload_hash_set);
+        // 3.  tag2 UID FETCH 1:<lastseenuid> FLAGS
+        if max_uid == 0 {
+            self.send_command("UID FETCH 1:* FLAGS".as_bytes()).await?;
+        } else {
+            self.send_command(format!("UID FETCH 1:{} FLAGS", max_uid).as_bytes())
+                .await?;
+        }
+        self.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
+            .await?;
+        //1) update cached flags for old messages;
+        //2) find out which old messages got expunged; and
+        //3) build a mapping between message numbers and UIDs (for old messages).
+        let mut valid_envs = BTreeSet::default();
+        let mut env_lck = self.uid_store.envelopes.lock().unwrap();
+        let (_, v, _) = protocol_parser::fetch_responses(&response)?;
+        let mut refresh_events = vec![];
+        for FetchResponse { uid, flags, .. } in v {
+            let uid = uid.unwrap();
+            let env_hash = generate_envelope_hash(&mailbox_path, &uid);
+            valid_envs.insert(env_hash);
+            if !env_lck.contains_key(&env_hash) {
+                return Ok(None);
+            }
+            let (flags, tags) = flags.unwrap();
+            if env_lck[&env_hash].inner.flags() != flags
+                || env_lck[&env_hash].inner.labels()
+                    != &tags
+                        .iter()
+                        .map(|t| tag_hash!(t))
+                        .collect::<SmallVec<[u64; 8]>>()
+            {
+                env_lck.entry(env_hash).and_modify(|entry| {
+                    entry.inner.set_flags(flags);
+                    entry.inner.labels_mut().clear();
+                    entry
+                        .inner
+                        .labels_mut()
+                        .extend(tags.iter().map(|t| tag_hash!(t)));
+                });
+                refresh_events.push((
+                    uid,
+                    RefreshEvent {
+                        mailbox_hash,
+                        account_hash: self.uid_store.account_hash,
+                        kind: RefreshEventKind::NewFlags(env_hash, (flags, tags)),
+                    },
+                ));
+            }
+        }
+        for env_hash in env_lck
+            .iter()
+            .filter_map(|(h, cenv)| {
+                if cenv.mailbox_hash == mailbox_hash {
+                    Some(*h)
+                } else {
+                    None
+                }
+            })
+            .collect::<BTreeSet<EnvelopeHash>>()
+            .difference(&valid_envs)
+        {
+            refresh_events.push((
+                env_lck[env_hash].uid,
+                RefreshEvent {
+                    mailbox_hash,
+                    account_hash: self.uid_store.account_hash,
+                    kind: RefreshEventKind::Remove(*env_hash),
+                },
+            ));
+            env_lck.remove(env_hash);
+        }
+        drop(env_lck);
+        cache_handle.update(mailbox_hash, &refresh_events)?;
+        for (_uid, ev) in refresh_events {
+            self.add_refresh_event(ev);
+        }
+        Ok(Some(payload.into_iter().map(|(_, env)| env).collect()))
+    }
+
+    //rfc4549_Synchronization_Operations_for_Disconnected_IMAP4_Clients
+    //Section 6.1
+    pub async fn resync_condstore(
+        &mut self,
+        mut cache_handle: Box<dyn ImapCache>,
+        mailbox_hash: MailboxHash,
+    ) -> Result<Option<Vec<Envelope>>> {
+        let mut payload = vec![];
+        debug!("resync_condstore");
+        let mut response = Vec::with_capacity(8 * 1024);
+        let cached_uidvalidity = self
+            .uid_store
+            .uidvalidity
+            .lock()
+            .unwrap()
+            .get(&mailbox_hash)
+            .cloned();
+        let cached_max_uid = self
+            .uid_store
+            .max_uids
+            .lock()
+            .unwrap()
+            .get(&mailbox_hash)
+            .cloned();
+        let cached_highestmodseq = self
+            .uid_store
+            .highestmodseqs
+            .lock()
+            .unwrap()
+            .get(&mailbox_hash)
+            .cloned();
+        if cached_uidvalidity.is_none()
+            || cached_max_uid.is_none()
+            || cached_highestmodseq.is_none()
+        {
+            // This means the mailbox is not cached.
+            return Ok(None);
+        }
+        let cached_uidvalidity: UID = cached_uidvalidity.unwrap();
+        let cached_max_uid: UID = cached_max_uid.unwrap();
+        let cached_highestmodseq: std::result::Result<ModSequence, ()> =
+            cached_highestmodseq.unwrap();
+        if cached_highestmodseq.is_err() {
+            // No MODSEQ is available for __this__ mailbox, fallback to basic sync
+            return self.resync_basic(cache_handle, mailbox_hash).await;
+        }
+        let cached_highestmodseq: ModSequence = cached_highestmodseq.unwrap();
+
+        let (mailbox_path, mailbox_exists, unseen) = {
+            let f = &self.uid_store.mailboxes.lock().await[&mailbox_hash];
+            (
+                f.imap_path().to_string(),
+                f.exists.clone(),
+                f.unseen.clone(),
+            )
+        };
+        let mut new_unseen = BTreeSet::default();
+        // 1. check UIDVALIDITY. If fail, discard cache and rebuild
+        let select_response = self
+            .select_mailbox(mailbox_hash, &mut response, true)
+            .await?
+            .unwrap();
+        if select_response.uidvalidity != cached_uidvalidity {
+            // 1a) Check the mailbox UIDVALIDITY (see section 4.1 for more
+            //details) with SELECT/EXAMINE/STATUS.
+            //   If the UIDVALIDITY value returned by the server differs, the
+            //   client MUST
+            //   * empty the local cache of that mailbox;
+            //   * "forget" the cached HIGHESTMODSEQ value for the mailbox;
+            //   * remove any pending "actions" that refer to UIDs in that
+            //     mailbox (note that this doesn't affect actions performed on
+            //     client-generated fake UIDs; see Section 5); and
+            //   * skip steps 1b and 2-II;
+            cache_handle.clear(mailbox_hash, &select_response)?;
+            return Ok(None);
+        }
+        if select_response.highestmodseq.is_none()
+            || select_response.highestmodseq.as_ref().unwrap().is_err()
+        {
+            if select_response.highestmodseq.as_ref().unwrap().is_err() {
+                self.uid_store
+                    .highestmodseqs
+                    .lock()
+                    .unwrap()
+                    .insert(mailbox_hash, Err(()));
+            }
+            return self.resync_basic(cache_handle, mailbox_hash).await;
+        }
+        cache_handle.update_mailbox(mailbox_hash, &select_response)?;
+        let new_highestmodseq = select_response.highestmodseq.unwrap().unwrap();
+        let mut refresh_events = vec![];
+        // 1b) Check the mailbox HIGHESTMODSEQ.
+        //  If the cached value is the same as the one returned by the server, skip fetching
+        //  message flags on step 2-II, i.e., the client only has to find out which messages got
+        //  expunged.
+        if cached_highestmodseq != new_highestmodseq {
+            /* Cache is synced, only figure out which messages got expunged */
+
+            // 2) Fetch the current "descriptors".
+            //   I)  Discover new messages.
+
+            //   II) Discover changes to old messages and flags for new messages
+            //       using
+            //       "FETCH 1:* (FLAGS) (CHANGEDSINCE <cached-value>)" or
+            //       "SEARCH MODSEQ <cached-value>".
+
+            // 2.  tag1 UID FETCH <lastseenuid+1>:* <descriptors>
+            self.send_command(
+                format!(
+                    "UID FETCH {}:* (UID FLAGS ENVELOPE BODY.PEEK[HEADER.FIELDS (REFERENCES)] BODYSTRUCTURE) (CHANGEDSINCE {})",
+                    cached_max_uid + 1,
+                    cached_highestmodseq,
+                )
+                .as_bytes(),
+            )
+            .await?;
+            self.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
+                .await?;
+            debug!(
+                "fetch response is {} bytes and {} lines",
+                response.len(),
+                String::from_utf8_lossy(&response).lines().count()
+            );
+            let (_, mut v, _) = protocol_parser::fetch_responses(&response)?;
+            debug!("responses len is {}", v.len());
+            for FetchResponse {
+                ref uid,
+                ref mut envelope,
+                ref mut flags,
+                ref references,
+                ..
+            } in v.iter_mut()
+            {
+                let uid = uid.unwrap();
+                let env = envelope.as_mut().unwrap();
+                env.set_hash(generate_envelope_hash(&mailbox_path, &uid));
+                if let Some(value) = references {
+                    env.set_references(value);
+                }
+                let mut tag_lck = self.uid_store.collection.tag_index.write().unwrap();
+                if let Some((flags, keywords)) = flags {
+                    env.set_flags(*flags);
+                    if !env.is_seen() {
+                        new_unseen.insert(env.hash());
+                    }
+                    for f in keywords {
+                        let hash = tag_hash!(f);
+                        if !tag_lck.contains_key(&hash) {
+                            tag_lck.insert(hash, f.to_string());
+                        }
+                        env.labels_mut().push(hash);
+                    }
+                }
+            }
+            {
+                cache_handle
+                    .insert_envelopes(mailbox_hash, &v)
+                    .chain_err_summary(|| {
+                        format!(
+                            "Could not save envelopes in cache for mailbox {}",
+                            mailbox_path
+                        )
+                    })?;
+            }
+
+            for FetchResponse { uid, envelope, .. } in v {
+                let uid = uid.unwrap();
+                let env = envelope.unwrap();
+                /*
+                debug!(
+                    "env hash {} {} UID = {} MSN = {}",
+                    env.hash(),
+                    env.subject(),
+                    uid,
+                    message_sequence_number
+                );
+                */
+                self.uid_store
+                    .hash_index
+                    .lock()
+                    .unwrap()
+                    .insert(env.hash(), (uid, mailbox_hash));
+                self.uid_store
+                    .uid_index
+                    .lock()
+                    .unwrap()
+                    .insert((mailbox_hash, uid), env.hash());
+                payload.push((uid, env));
+            }
+            debug!("sending payload for {}", mailbox_hash);
+            let payload_hash_set: BTreeSet<_> =
+                payload.iter().map(|(_, env)| env.hash()).collect::<_>();
+            {
+                let mut unseen_lck = unseen.lock().unwrap();
+                for &seen_env_hash in payload_hash_set.difference(&new_unseen) {
+                    unseen_lck.remove(seen_env_hash);
+                }
+
+                unseen_lck.insert_set(new_unseen);
+            }
+            mailbox_exists.lock().unwrap().insert_set(payload_hash_set);
+            // 3.  tag2 UID FETCH 1:<lastseenuid> FLAGS
+            if cached_max_uid == 0 {
+                self.send_command(
+                    format!(
+                        "UID FETCH 1:* FLAGS (CHANGEDSINCE {})",
+                        cached_highestmodseq
+                    )
+                    .as_bytes(),
+                )
+                .await?;
+            } else {
+                self.send_command(
+                    format!(
+                        "UID FETCH 1:{} FLAGS (CHANGEDSINCE {})",
+                        cached_max_uid, cached_highestmodseq
+                    )
+                    .as_bytes(),
+                )
+                .await?;
+            }
+            self.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
+                .await?;
+            //1) update cached flags for old messages;
+            let mut env_lck = self.uid_store.envelopes.lock().unwrap();
+            let (_, v, _) = protocol_parser::fetch_responses(&response)?;
+            for FetchResponse { uid, flags, .. } in v {
+                let uid = uid.unwrap();
+                let env_hash = generate_envelope_hash(&mailbox_path, &uid);
+                if !env_lck.contains_key(&env_hash) {
+                    return Ok(None);
+                }
+                let (flags, tags) = flags.unwrap();
+                if env_lck[&env_hash].inner.flags() != flags
+                    || env_lck[&env_hash].inner.labels()
+                        != &tags
+                            .iter()
+                            .map(|t| tag_hash!(t))
+                            .collect::<SmallVec<[u64; 8]>>()
+                {
+                    env_lck.entry(env_hash).and_modify(|entry| {
+                        entry.inner.set_flags(flags);
+                        entry.inner.labels_mut().clear();
+                        entry
+                            .inner
+                            .labels_mut()
+                            .extend(tags.iter().map(|t| tag_hash!(t)));
+                    });
+                    refresh_events.push((
+                        uid,
+                        RefreshEvent {
+                            mailbox_hash,
+                            account_hash: self.uid_store.account_hash,
+                            kind: RefreshEventKind::NewFlags(env_hash, (flags, tags)),
+                        },
+                    ));
+                }
+            }
+            self.uid_store
+                .highestmodseqs
+                .lock()
+                .unwrap()
+                .insert(mailbox_hash, Ok(new_highestmodseq));
+        }
+        let mut valid_envs = BTreeSet::default();
+        // This should be UID SEARCH 1:<maxuid> but it's difficult to compare to cached UIDs at the
+        // point of calling this function
+        self.send_command(b"UID SEARCH ALL").await?;
+        self.read_response(&mut response, RequiredResponses::SEARCH)
+            .await?;
+        //1) update cached flags for old messages;
+        let (_, v) = protocol_parser::search_results(response.as_slice())?;
+        for uid in v {
+            valid_envs.insert(generate_envelope_hash(&mailbox_path, &uid));
+        }
+        {
+            let mut env_lck = self.uid_store.envelopes.lock().unwrap();
+            for env_hash in env_lck
+                .iter()
+                .filter_map(|(h, cenv)| {
+                    if cenv.mailbox_hash == mailbox_hash {
+                        Some(*h)
+                    } else {
+                        None
+                    }
+                })
+                .collect::<BTreeSet<EnvelopeHash>>()
+                .difference(&valid_envs)
+            {
+                refresh_events.push((
+                    env_lck[env_hash].uid,
+                    RefreshEvent {
+                        mailbox_hash,
+                        account_hash: self.uid_store.account_hash,
+                        kind: RefreshEventKind::Remove(*env_hash),
+                    },
+                ));
+                env_lck.remove(env_hash);
+            }
+            drop(env_lck);
+        }
+        cache_handle.update(mailbox_hash, &refresh_events)?;
+        for (_uid, ev) in refresh_events {
+            self.add_refresh_event(ev);
+        }
+        Ok(Some(payload.into_iter().map(|(_, env)| env).collect()))
+    }
+
+    //rfc7162_Quick Flag Changes Resynchronization (CONDSTORE)_and Quick Mailbox Resynchronization (QRESYNC)
+    pub async fn resync_condstoreqresync(
+        &mut self,
+        _cache_handle: Box<dyn ImapCache>,
+        _mailbox_hash: MailboxHash,
+    ) -> Result<Option<Vec<Envelope>>> {
+        Ok(None)
+    }
+
+    pub async fn init_mailbox(&mut self, mailbox_hash: MailboxHash) -> Result<SelectResponse> {
+        let mut response = Vec::with_capacity(8 * 1024);
+        let (mailbox_path, mailbox_exists, permissions) = {
+            let f = &self.uid_store.mailboxes.lock().await[&mailbox_hash];
+            (
+                f.imap_path().to_string(),
+                f.exists.clone(),
+                f.permissions.clone(),
+            )
+        };
+
+        /* first SELECT the mailbox to get READ/WRITE permissions (because EXAMINE only
+         * returns READ-ONLY for both cases) */
+        let mut select_response = self
+            .select_mailbox(mailbox_hash, &mut response, true)
+            .await?
+            .unwrap();
+        debug!(
+            "mailbox: {} select_response: {:?}",
+            mailbox_path, select_response
+        );
+        {
+            {
+                let mut uidvalidities = self.uid_store.uidvalidity.lock().unwrap();
+
+                let v = uidvalidities
+                    .entry(mailbox_hash)
+                    .or_insert(select_response.uidvalidity);
+                *v = select_response.uidvalidity;
+            }
+            {
+                if let Some(highestmodseq) = select_response.highestmodseq {
+                    let mut highestmodseqs = self.uid_store.highestmodseqs.lock().unwrap();
+                    let v = highestmodseqs.entry(mailbox_hash).or_insert(highestmodseq);
+                    *v = highestmodseq;
+                }
+            }
+            let mut permissions = permissions.lock().unwrap();
+            permissions.create_messages = !select_response.read_only;
+            permissions.remove_messages = !select_response.read_only;
+            permissions.set_flags = !select_response.read_only;
+            permissions.rename_messages = !select_response.read_only;
+            permissions.delete_messages = !select_response.read_only;
+            {
+                let mut mailbox_exists_lck = mailbox_exists.lock().unwrap();
+                mailbox_exists_lck.clear();
+                mailbox_exists_lck.set_not_yet_seen(select_response.exists);
+            }
+        }
+        if select_response.exists == 0 {
+            return Ok(select_response);
+        }
+        /* reselecting the same mailbox with EXAMINE prevents expunging it */
+        self.examine_mailbox(mailbox_hash, &mut response, true)
+            .await?;
+        if select_response.uidnext == 0 {
+            /* UIDNEXT shouldn't be 0, since exists != 0 at this point */
+            self.send_command(format!("STATUS \"{}\" (UIDNEXT)", mailbox_path).as_bytes())
+                .await?;
+            self.read_response(&mut response, RequiredResponses::STATUS)
+                .await?;
+            let (_, status) = protocol_parser::status_response(response.as_slice())?;
+            if let Some(uidnext) = status.uidnext {
+                if uidnext == 0 {
+                    return Err(MeliError::new(
+                        "IMAP server error: zero UIDNEXT with nonzero exists.",
+                    ));
+                }
+                select_response.uidnext = uidnext;
+            } else {
+                return Err(MeliError::new("IMAP server did not reply with UIDNEXT"));
+            }
+        }
+        Ok(select_response)
+    }
+}

melib/src/backends/imap/folder.rs

@@ -1,70 +0,0 @@
-/*
- * meli - imap module.
- *
- * Copyright 2019 Manos Pitsidianakis
- *
- * This file is part of meli.
- *
- * meli is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * meli is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with meli. If not, see <http://www.gnu.org/licenses/>.
- */
-use crate::backends::{BackendFolder, Folder, FolderHash};
-use std::sync::{Arc, Mutex};
-
-#[derive(Debug, Default, Clone)]
-pub struct ImapFolder {
-    pub(super) hash: FolderHash,
-    pub(super) path: String,
-    pub(super) name: String,
-    pub(super) parent: Option<FolderHash>,
-    pub(super) children: Vec<FolderHash>,
-
-    pub exists: Arc<Mutex<usize>>,
-}
-
-impl BackendFolder for ImapFolder {
-    fn hash(&self) -> FolderHash {
-        self.hash
-    }
-
-    fn name(&self) -> &str {
-        &self.name
-    }
-
-    fn path(&self) -> &str {
-        &self.path
-    }
-
-    fn change_name(&mut self, s: &str) {
-        self.name = s.to_string();
-    }
-
-    fn children(&self) -> &Vec<FolderHash> {
-        &self.children
-    }
-
-    fn clone(&self) -> Folder {
-        Box::new(ImapFolder {
-            hash: self.hash,
-            path: self.path.clone(),
-            name: self.name.clone(),
-            parent: self.parent,
-            children: self.children.clone(),
-            exists: self.exists.clone(),
-        })
-    }
-
-    fn parent(&self) -> Option<FolderHash> {
-        self.parent
-    }
-}

melib/src/backends/imap/mailbox.rs

@@ -0,0 +1,125 @@
+/*
+ * meli - imap module.
+ *
+ * Copyright 2019 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::protocol_parser::SelectResponse;
+use crate::backends::{
+    BackendMailbox, LazyCountSet, Mailbox, MailboxHash, MailboxPermissions, SpecialUsageMailbox,
+};
+use crate::error::*;
+use std::sync::{Arc, Mutex, RwLock};
+
+#[derive(Debug, Default, Clone)]
+pub struct ImapMailbox {
+    pub hash: MailboxHash,
+    pub imap_path: String,
+    pub path: String,
+    pub name: String,
+    pub parent: Option<MailboxHash>,
+    pub children: Vec<MailboxHash>,
+    pub separator: u8,
+    pub usage: Arc<RwLock<SpecialUsageMailbox>>,
+    pub select: Arc<RwLock<Option<SelectResponse>>>,
+    pub no_select: bool,
+    pub is_subscribed: bool,
+
+    pub permissions: Arc<Mutex<MailboxPermissions>>,
+    pub exists: Arc<Mutex<LazyCountSet>>,
+    pub unseen: Arc<Mutex<LazyCountSet>>,
+    pub warm: Arc<Mutex<bool>>,
+}
+
+impl ImapMailbox {
+    pub fn imap_path(&self) -> &str {
+        &self.imap_path
+    }
+
+    /// Establish that mailbox contents have been fetched at least once during this execution
+    #[inline(always)]
+    pub fn set_warm(&self, new_value: bool) {
+        *self.warm.lock().unwrap() = new_value;
+    }
+
+    /// Mailbox contents have been fetched at least once during this execution
+    #[inline(always)]
+    pub fn is_warm(&self) -> bool {
+        *self.warm.lock().unwrap()
+    }
+
+    /// Mailbox contents have not been fetched at all during this execution
+    #[inline(always)]
+    pub fn is_cold(&self) -> bool {
+        !self.is_warm()
+    }
+}
+
+impl BackendMailbox for ImapMailbox {
+    fn hash(&self) -> MailboxHash {
+        self.hash
+    }
+
+    fn name(&self) -> &str {
+        &self.name
+    }
+
+    fn path(&self) -> &str {
+        &self.path
+    }
+
+    fn change_name(&mut self, s: &str) {
+        self.name = s.to_string();
+    }
+
+    fn children(&self) -> &[MailboxHash] {
+        &self.children
+    }
+
+    fn clone(&self) -> Mailbox {
+        Box::new(std::clone::Clone::clone(self))
+    }
+
+    fn special_usage(&self) -> SpecialUsageMailbox {
+        *self.usage.read().unwrap()
+    }
+
+    fn parent(&self) -> Option<MailboxHash> {
+        self.parent
+    }
+
+    fn permissions(&self) -> MailboxPermissions {
+        *self.permissions.lock().unwrap()
+    }
+    fn is_subscribed(&self) -> bool {
+        self.is_subscribed
+    }
+    fn set_is_subscribed(&mut self, new_val: bool) -> Result<()> {
+        self.is_subscribed = new_val;
+        Ok(())
+    }
+
+    fn set_special_usage(&mut self, new_val: SpecialUsageMailbox) -> Result<()> {
+        *self.usage.write()? = new_val;
+        Ok(())
+    }
+
+    fn count(&self) -> Result<(usize, usize)> {
+        Ok((self.unseen.lock()?.len(), self.exists.lock()?.len()))
+    }
+}

melib/src/backends/imap/managesieve.rs

@@ -0,0 +1,156 @@
+/*
+ * meli - managesieve
+ *
+ * Copyright 2020 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::{ImapConnection, ImapProtocol, ImapServerConf, UIDStore};
+use crate::conf::AccountSettings;
+use crate::error::{MeliError, Result};
+use crate::get_conf_val;
+use nom::{
+    branch::alt, bytes::complete::tag, combinator::map, error::ErrorKind,
+    multi::separated_nonempty_list, sequence::separated_pair, IResult,
+};
+use std::str::FromStr;
+use std::sync::{Arc, Mutex};
+use std::time::SystemTime;
+
+pub fn managesieve_capabilities(input: &[u8]) -> Result<Vec<(&[u8], &[u8])>> {
+    let (_, ret) = separated_nonempty_list(
+        tag(b"\r\n"),
+        alt((
+            separated_pair(quoted_raw, tag(b" "), quoted_raw),
+            map(quoted_raw, |q| (q, &b""[..])),
+        )),
+    )(input)?;
+    Ok(ret)
+}
+
+#[test]
+fn test_managesieve_capabilities() {
+    assert_eq!(managesieve_capabilities(b"\"IMPLEMENTATION\" \"Dovecot Pigeonhole\"\r\n\"SIEVE\" \"fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date index ihave duplicate mime foreverypart extracttext\"\r\n\"NOTIFY\" \"mailto\"\r\n\"SASL\" \"PLAIN\"\r\n\"STARTTLS\"\r\n\"VERSION\" \"1.0\"\r\n").unwrap(), vec![
+        (&b"IMPLEMENTATION"[..],&b"Dovecot Pigeonhole"[..]),
+        (&b"SIEVE"[..],&b"fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date index ihave duplicate mime foreverypart extracttext"[..]),
+        (&b"NOTIFY"[..],&b"mailto"[..]),
+        (&b"SASL"[..],&b"PLAIN"[..]),
+        (&b"STARTTLS"[..], &b""[..]),
+        (&b"VERSION"[..],&b"1.0"[..])]
+
+    );
+}
+
+// Return a byte sequence surrounded by "s and decoded if necessary
+pub fn quoted_raw(input: &[u8]) -> IResult<&[u8], &[u8]> {
+    if input.is_empty() || input[0] != b'"' {
+        return Err(nom::Err::Error((input, ErrorKind::Tag)));
+    }
+
+    let mut i = 1;
+    while i < input.len() {
+        if input[i] == b'\"' && input[i - 1] != b'\\' {
+            return Ok((&input[i + 1..], &input[1..i]));
+        }
+        i += 1;
+    }
+
+    Err(nom::Err::Error((input, ErrorKind::Tag)))
+}
+
+pub trait ManageSieve {
+    fn havespace(&mut self) -> Result<()>;
+    fn putscript(&mut self) -> Result<()>;
+
+    fn listscripts(&mut self) -> Result<()>;
+    fn setactive(&mut self) -> Result<()>;
+
+    fn getscript(&mut self) -> Result<()>;
+
+    fn deletescript(&mut self) -> Result<()>;
+    fn renamescript(&mut self) -> Result<()>;
+}
+
+pub fn new_managesieve_connection(
+    account_hash: crate::backends::AccountHash,
+    account_name: String,
+    s: &AccountSettings,
+    event_consumer: crate::backends::BackendEventConsumer,
+) -> Result<ImapConnection> {
+    let server_hostname = get_conf_val!(s["server_hostname"])?;
+    let server_username = get_conf_val!(s["server_username"])?;
+    let server_password = get_conf_val!(s["server_password"])?;
+    let server_port = get_conf_val!(s["server_port"], 4190)?;
+    let danger_accept_invalid_certs: bool = get_conf_val!(s["danger_accept_invalid_certs"], false)?;
+    let timeout = get_conf_val!(s["timeout"], 16_u64)?;
+    let timeout = if timeout == 0 {
+        None
+    } else {
+        Some(std::time::Duration::from_secs(timeout))
+    };
+    let server_conf = ImapServerConf {
+        server_hostname: server_hostname.to_string(),
+        server_username: server_username.to_string(),
+        server_password: server_password.to_string(),
+        server_port,
+        use_starttls: true,
+        use_tls: true,
+        danger_accept_invalid_certs,
+        protocol: ImapProtocol::ManageSieve,
+        timeout,
+    };
+    let uid_store = Arc::new(UIDStore {
+        is_online: Arc::new(Mutex::new((
+            SystemTime::now(),
+            Err(MeliError::new("Account is uninitialised.")),
+        ))),
+        ..UIDStore::new(
+            account_hash,
+            Arc::new(account_name),
+            event_consumer,
+            server_conf.timeout,
+        )
+    });
+    Ok(ImapConnection::new_connection(&server_conf, uid_store))
+}
+
+impl ManageSieve for ImapConnection {
+    fn havespace(&mut self) -> Result<()> {
+        Ok(())
+    }
+    fn putscript(&mut self) -> Result<()> {
+        Ok(())
+    }
+
+    fn listscripts(&mut self) -> Result<()> {
+        Ok(())
+    }
+    fn setactive(&mut self) -> Result<()> {
+        Ok(())
+    }
+
+    fn getscript(&mut self) -> Result<()> {
+        Ok(())
+    }
+
+    fn deletescript(&mut self) -> Result<()> {
+        Ok(())
+    }
+    fn renamescript(&mut self) -> Result<()> {
+        Ok(())
+    }
+}

melib/src/backends/imap/operations.rs

@@ -21,283 +21,152 @@
 
 use super::*;
 
-use crate::backends::BackendOp;
+use crate::backends::*;
 use crate::email::*;
-use crate::error::{MeliError, Result};
-use std::cell::Cell;
-use std::sync::{Arc, Mutex};
+use crate::error::MeliError;
+use std::sync::Arc;
 
 /// `BackendOp` implementor for Imap
 #[derive(Debug, Clone)]
 pub struct ImapOp {
-    uid: usize,
-    bytes: Option<String>,
-    headers: Option<String>,
-    body: Option<String>,
-    folder_path: String,
-    flags: Cell<Option<Flag>>,
-    connection: Arc<Mutex<ImapConnection>>,
-    byte_cache: Arc<Mutex<FnvHashMap<UID, EnvelopeCache>>>,
+    uid: UID,
+    mailbox_hash: MailboxHash,
+    connection: Arc<FutureMutex<ImapConnection>>,
+    uid_store: Arc<UIDStore>,
 }
 
 impl ImapOp {
     pub fn new(
-        uid: usize,
-        folder_path: String,
-        connection: Arc<Mutex<ImapConnection>>,
-        byte_cache: Arc<Mutex<FnvHashMap<UID, EnvelopeCache>>>,
+        uid: UID,
+        mailbox_hash: MailboxHash,
+        connection: Arc<FutureMutex<ImapConnection>>,
+        uid_store: Arc<UIDStore>,
     ) -> Self {
         ImapOp {
             uid,
             connection,
-            bytes: None,
-            headers: None,
-            body: None,
-            folder_path,
-            flags: Cell::new(None),
-            byte_cache,
+            mailbox_hash,
+            uid_store,
         }
     }
 }
 
 impl BackendOp for ImapOp {
-    fn description(&self) -> String {
-        unimplemented!();
-    }
-
-    fn as_bytes(&mut self) -> Result<&[u8]> {
-        if self.bytes.is_none() {
-            let mut bytes_cache = self.byte_cache.lock()?;
-            let cache = bytes_cache.entry(self.uid).or_default();
-            if cache.bytes.is_some() {
-                self.bytes = cache.bytes.clone();
-            } else {
-                let mut response = String::with_capacity(8 * 1024);
+    fn as_bytes(&mut self) -> ResultFuture<Vec<u8>> {
+        let connection = self.connection.clone();
+        let mailbox_hash = self.mailbox_hash;
+        let uid = self.uid;
+        let uid_store = self.uid_store.clone();
+        Ok(Box::pin(async move {
+            let exists_in_cache = {
+                let mut bytes_cache = uid_store.byte_cache.lock()?;
+                let cache = bytes_cache.entry(uid).or_default();
+                cache.bytes.is_some()
+            };
+            if !exists_in_cache {
+                let mut response = Vec::with_capacity(8 * 1024);
                 {
-                    let mut conn = self.connection.lock().unwrap();
-                    conn.send_command(format!("SELECT {}", self.folder_path).as_bytes())?;
-                    conn.read_response(&mut response)?;
-                    conn.send_command(format!("UID FETCH {} (FLAGS RFC822)", self.uid).as_bytes())?;
-                    conn.read_response(&mut response)?;
+                    let mut conn = timeout(uid_store.timeout, connection.lock()).await?;
+                    conn.connect().await?;
+                    conn.examine_mailbox(mailbox_hash, &mut response, false)
+                        .await?;
+                    conn.send_command(format!("UID FETCH {} (FLAGS RFC822)", uid).as_bytes())
+                        .await?;
+                    conn.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
+                        .await?;
                 }
                 debug!(
                     "fetch response is {} bytes and {} lines",
                     response.len(),
-                    response.lines().collect::<Vec<&str>>().len()
+                    String::from_utf8_lossy(&response).lines().count()
                 );
-                match protocol_parser::uid_fetch_response(response.as_bytes())
-                    .to_full_result()
-                    .map_err(MeliError::from)
-                {
-                    Ok(v) => {
-                        if v.len() != 1 {
-                            debug!("responses len is {}", v.len());
-                            /* TODO: Trigger cache invalidation here. */
-                            return Err(MeliError::new(format!(
-                                "message with UID {} was not found",
-                                self.uid
-                            )));
-                        }
-                        let (uid, flags, b) = v[0];
-                        assert_eq!(uid, self.uid);
-                        if flags.is_some() {
-                            self.flags.set(flags);
-                            cache.flags = flags;
-                        }
-                        cache.bytes = Some(unsafe { std::str::from_utf8_unchecked(b).to_string() });
-                    }
-                    Err(e) => return Err(e),
+                let mut results = protocol_parser::fetch_responses(&response)?.1;
+                if results.len() != 1 {
+                    return Err(MeliError::new(format!(
+                        "Invalid/unexpected response: {:?}",
+                        response
+                    ))
+                    .set_summary(format!("message with UID {} was not found?", uid)));
                 }
-
-                self.bytes = cache.bytes.clone();
+                let FetchResponse {
+                    uid: _uid,
+                    flags: _flags,
+                    body,
+                    ..
+                } = results.pop().unwrap();
+                let _uid = _uid.unwrap();
+                assert_eq!(_uid, uid);
+                assert!(body.is_some());
+                let mut bytes_cache = uid_store.byte_cache.lock()?;
+                let cache = bytes_cache.entry(uid).or_default();
+                if let Some((_flags, _)) = _flags {
+                    //flags.lock().await.set(Some(_flags));
+                    cache.flags = Some(_flags);
+                }
+                cache.bytes = Some(body.unwrap().to_vec());
             }
-        }
-        Ok(self.bytes.as_ref().unwrap().as_bytes())
+            let mut bytes_cache = uid_store.byte_cache.lock()?;
+            let cache = bytes_cache.entry(uid).or_default();
+            let ret = cache.bytes.clone().unwrap();
+            Ok(ret)
+        }))
     }
 
-    fn fetch_headers(&mut self) -> Result<&[u8]> {
-        if self.bytes.is_some() {
-            let result =
-                parser::headers_raw(self.bytes.as_ref().unwrap().as_bytes()).to_full_result()?;
-            return Ok(result);
-        }
-        if self.headers.is_none() {
-            let mut bytes_cache = self.byte_cache.lock()?;
-            let cache = bytes_cache.entry(self.uid).or_default();
-            if cache.headers.is_some() {
-                self.headers = cache.headers.clone();
-            } else {
-                let mut response = String::with_capacity(8 * 1024);
-                let mut conn = self.connection.lock().unwrap();
-                conn.send_command(
-                    format!("UID FETCH {} (FLAGS RFC822.HEADER)", self.uid).as_bytes(),
-                )?;
-                conn.read_response(&mut response)?;
+    fn fetch_flags(&self) -> ResultFuture<Flag> {
+        let mut response = Vec::with_capacity(8 * 1024);
+        let connection = self.connection.clone();
+        let mailbox_hash = self.mailbox_hash;
+        let uid = self.uid;
+        let uid_store = self.uid_store.clone();
+
+        Ok(Box::pin(async move {
+            let exists_in_cache = {
+                let mut bytes_cache = uid_store.byte_cache.lock()?;
+                let cache = bytes_cache.entry(uid).or_default();
+                cache.flags.is_some()
+            };
+            if !exists_in_cache {
+                let mut conn = connection.lock().await;
+                conn.connect().await?;
+                conn.examine_mailbox(mailbox_hash, &mut response, false)
+                    .await?;
+                conn.send_command(format!("UID FETCH {} FLAGS", uid).as_bytes())
+                    .await?;
+                conn.read_response(&mut response, RequiredResponses::FETCH_REQUIRED)
+                    .await?;
                 debug!(
                     "fetch response is {} bytes and {} lines",
                     response.len(),
-                    response.lines().collect::<Vec<&str>>().len()
+                    String::from_utf8_lossy(&response).lines().count()
                 );
-                match protocol_parser::uid_fetch_response(response.as_bytes())
-                    .to_full_result()
-                    .map_err(MeliError::from)
-                {
-                    Ok(v) => {
-                        if v.len() != 1 {
-                            debug!("responses len is {}", v.len());
-                            /* TODO: Trigger cache invalidation here. */
-                            return Err(MeliError::new(format!(
-                                "message with UID {} was not found",
-                                self.uid
-                            )));
-                        }
-                        let (uid, flags, b) = v[0];
-                        assert_eq!(uid, self.uid);
-                        if flags.is_some() {
-                            self.flags.set(flags);
-                            cache.flags = flags;
-                        }
-                        cache.headers =
-                            Some(unsafe { std::str::from_utf8_unchecked(b).to_string() });
-                    }
-                    Err(e) => return Err(e),
-                }
-                self.headers = cache.headers.clone();
-            }
-        }
-        Ok(self.headers.as_ref().unwrap().as_bytes())
-    }
-
-    fn fetch_body(&mut self) -> Result<&[u8]> {
-        if self.bytes.is_some() {
-            let result =
-                parser::body_raw(self.bytes.as_ref().unwrap().as_bytes()).to_full_result()?;
-            return Ok(result);
-        }
-        if self.body.is_none() {
-            let mut bytes_cache = self.byte_cache.lock()?;
-            let cache = bytes_cache.entry(self.uid).or_default();
-            if cache.body.is_some() {
-                self.body = cache.body.clone();
-            } else {
-                let mut response = String::with_capacity(8 * 1024);
-                let mut conn = self.connection.lock().unwrap();
-                conn.send_command(
-                    format!("UID FETCH {} (FLAGS RFC822.TEXT)", self.uid).as_bytes(),
-                )?;
-                conn.read_response(&mut response)?;
-                debug!(
-                    "fetch response is {} bytes and {} lines",
-                    response.len(),
-                    response.lines().collect::<Vec<&str>>().len()
-                );
-                match protocol_parser::uid_fetch_response(response.as_bytes())
-                    .to_full_result()
-                    .map_err(MeliError::from)
-                {
-                    Ok(v) => {
-                        if v.len() != 1 {
-                            debug!("responses len is {}", v.len());
-                            /* TODO: Trigger cache invalidation here. */
-                            return Err(MeliError::new(format!(
-                                "message with UID {} was not found",
-                                self.uid
-                            )));
-                        }
-                        let (uid, flags, b) = v[0];
-                        assert_eq!(uid, self.uid);
-                        if flags.is_some() {
-                            self.flags.set(flags);
-                        }
-                        cache.body = Some(unsafe { std::str::from_utf8_unchecked(b).to_string() });
-                    }
-                    Err(e) => return Err(e),
-                }
-                self.body = cache.body.clone();
-            }
-        }
-        Ok(self.body.as_ref().unwrap().as_bytes())
-    }
-
-    fn fetch_flags(&self) -> Flag {
-        if self.flags.get().is_some() {
-            return self.flags.get().unwrap();
-        }
-        let mut bytes_cache = self.byte_cache.lock().unwrap();
-        let cache = bytes_cache.entry(self.uid).or_default();
-        if cache.flags.is_some() {
-            self.flags.set(cache.flags);
-        } else {
-            let mut response = String::with_capacity(8 * 1024);
-            let mut conn = self.connection.lock().unwrap();
-            conn.send_command(format!("UID FETCH {} FLAGS", self.uid).as_bytes())
-                .unwrap();
-            conn.read_response(&mut response).unwrap();
-            debug!(
-                "fetch response is {} bytes and {} lines",
-                response.len(),
-                response.lines().collect::<Vec<&str>>().len()
-            );
-            match protocol_parser::uid_fetch_response(response.as_bytes())
-                .to_full_result()
-                .map_err(MeliError::from)
-            {
-                Ok(v) => {
-                    if v.len() != 1 {
-                        debug!("responses len is {}", v.len());
-                        /* TODO: Trigger cache invalidation here. */
-                        panic!(format!("message with UID {} was not found", self.uid));
-                    }
-                    let (uid, flags, _) = v[0];
-                    assert_eq!(uid, self.uid);
-                    if flags.is_some() {
-                        cache.flags = flags;
-                        self.flags.set(flags);
-                    }
-                }
-                Err(e) => Err(e).unwrap(),
-            }
-        }
-        self.flags.get().unwrap()
-    }
-
-    fn set_flag(&mut self, _envelope: &mut Envelope, flag: Flag) -> Result<()> {
-        let mut response = String::with_capacity(8 * 1024);
-        let mut conn = self.connection.lock().unwrap();
-        conn.send_command(format!("SELECT \"{}\"", &self.folder_path,).as_bytes())?;
-        conn.read_response(&mut response)?;
-        debug!(&response);
-        conn.send_command(
-            format!(
-                "UID STORE {} FLAGS.SILENT ({})",
-                self.uid,
-                flags_to_imap_list!(flag)
-            )
-            .as_bytes(),
-        )?;
-        conn.read_response(&mut response)?;
-        debug!(&response);
-        match protocol_parser::uid_fetch_response(response.as_bytes())
-            .to_full_result()
-            .map_err(MeliError::from)
-        {
-            Ok(v) => {
-                if v.len() == 1 {
+                let v = protocol_parser::uid_fetch_flags_responses(&response)
+                    .map(|(_, v)| v)
+                    .map_err(MeliError::from)?;
+                if v.len() != 1 {
                     debug!("responses len is {}", v.len());
-                    let (uid, flags, _) = v[0];
-                    assert_eq!(uid, self.uid);
-                    if flags.is_some() {
-                        self.flags.set(flags);
-                    }
+                    debug!(String::from_utf8_lossy(&response));
+                    /* TODO: Trigger cache invalidation here. */
+                    debug!("message with UID {} was not found", uid);
+                    return Err(MeliError::new(format!(
+                        "Invalid/unexpected response: {:?}",
+                        response
+                    ))
+                    .set_summary(format!("message with UID {} was not found?", uid)));
                 }
+                let (_uid, (_flags, _)) = v[0];
+                assert_eq!(uid, uid);
+                let mut bytes_cache = uid_store.byte_cache.lock()?;
+                let cache = bytes_cache.entry(uid).or_default();
+                cache.flags = Some(_flags);
             }
-            Err(e) => Err(e).unwrap(),
-        }
-        conn.send_command(format!("EXAMINE \"{}\"", &self.folder_path,).as_bytes())?;
-        conn.read_response(&mut response)?;
-        let mut bytes_cache = self.byte_cache.lock()?;
-        let cache = bytes_cache.entry(self.uid).or_default();
-        cache.flags = Some(flag);
-        Ok(())
+            {
+                let val = {
+                    let mut bytes_cache = uid_store.byte_cache.lock()?;
+                    let cache = bytes_cache.entry(uid).or_default();
+                    cache.flags
+                };
+                Ok(val.unwrap())
+            }
+        }))
     }
 }

melib/src/backends/imap/tokens.rs

@@ -1,571 +0,0 @@
-use std::fmt;
-use std::num::NonZeroUsize;
-
-trait Join {
-    fn join(&self, sep: char) -> String;
-}
-
-impl<T> Join for [T]
-where
-    T: fmt::Display,
-{
-    fn join(&self, sep: char) -> String {
-        if self.is_empty() {
-            String::from("")
-        } else if self.len() == 1 {
-            format!("{}", self[0])
-        } else {
-            format!("{}{}{}", self[0], sep, self[1..].join(sep))
-        }
-    }
-}
-
-struct Search {
-    charset: Option<String>,
-    search_keys: Vec<SearchKey>,
-}
-
-impl fmt::Display for Search {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "SEARCH{} {}",
-            if let Some(ch) = self.charset.as_ref() {
-                format!(" CHARSET {}", ch)
-            } else {
-                format!("")
-            },
-            self.search_keys.join(' ')
-        )
-    }
-}
-
-enum SearchKey {
-    All,
-    Answered,
-    Bcc(String),
-    Before(String),
-    Body(String),
-    Cc(String),
-    Deleted,
-    Flagged,
-    From(String),
-    Keyword(FlagKeyword),
-    New,
-    Old,
-    On(String),
-    Recent,
-    Seen,
-    Since(String),
-    Subject(String),
-    Text(String),
-    To(String),
-    Unanswered,
-    Undeleted,
-    Unflagged,
-    Unkeyword(FlagKeyword),
-    Unseen,
-    Draft,
-    Header(String, String), //HeaderFldName
-    Larger(u64),
-    Not(Box<SearchKey>),
-    Or(Box<SearchKey>, Box<SearchKey>),
-    SentBefore(String), //Date
-    SentOn(String),     //Date
-    SentSince(String),  //Date
-    Smaller(u64),
-    Uid(SequenceSet),
-    Undraft,
-    SequenceSet(SequenceSet),
-    And(Vec<SearchKey>),
-}
-impl fmt::Display for SearchKey {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "{}",
-            match self {
-                SearchKey::All => format!("ALL"),
-                SearchKey::Answered => format!("ANSWERED"),
-                SearchKey::Bcc(ref s) => format!("BCC {}", s),
-                SearchKey::Before(ref s) => format!("BEFORE {}", s),
-                SearchKey::Body(ref s) => format!("BODY {}", s),
-                SearchKey::Cc(ref s) => format!("CC {}", s),
-                SearchKey::Deleted => format!("DELETED"),
-                SearchKey::Flagged => format!("FLAGGED"),
-                SearchKey::From(ref s) => format!("FROM {}", s),
-                SearchKey::Keyword(ref s) => format!("KEYWORD {}", s),
-                SearchKey::New => format!("NEW"),
-                SearchKey::Old => format!("OLD"),
-                SearchKey::On(ref s) => format!("ON {}", s),
-                SearchKey::Recent => format!("RECENT"),
-                SearchKey::Seen => format!("SEEN"),
-                SearchKey::Since(ref s) => format!("SINCE {}", s),
-                SearchKey::Subject(ref s) => format!("SUBJECT {}", s),
-                SearchKey::Text(ref s) => format!("TEXT {}", s),
-                SearchKey::To(ref s) => format!("TO {}", s),
-                SearchKey::Unanswered => format!("UNANSWERED"),
-                SearchKey::Undeleted => format!("UNDELETED"),
-                SearchKey::Unflagged => format!("UNFLAGGED"),
-                SearchKey::Unkeyword(ref s) => format!("UNKEYWORD {}", s),
-                SearchKey::Unseen => format!("UNSEEN"),
-                SearchKey::Draft => format!("DRAFT"),
-                SearchKey::Header(ref name, ref value) => format!("HEADER {} {}", name, value),
-                SearchKey::Larger(ref s) => format!("LARGER {}", s),
-                SearchKey::Not(ref s) => format!("NOT {}", s),
-                SearchKey::Or(ref a, ref b) => format!("OR {} {}", a, b),
-                SearchKey::SentBefore(ref s) => format!("SENTBEFORE {}", s),
-                SearchKey::SentOn(ref s) => format!("SENTON {}", s),
-                SearchKey::SentSince(ref s) => format!("SENTSINCE {}", s),
-                SearchKey::Smaller(ref s) => format!("SMALLER {}", s),
-                SearchKey::Uid(ref s) => format!("UID {}", s),
-                SearchKey::Undraft => format!("UNDRAFT"),
-                SearchKey::SequenceSet(ref s) => format!("SEQUENCESET {}", s),
-                SearchKey::And(ref s) => format!("({})", s.join(' ')),
-            }
-        )
-    }
-}
-
-struct Delete {
-    mailbox: Mailbox,
-}
-
-impl fmt::Display for Delete {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "DELETE {}", self.mailbox)
-    }
-}
-
-struct Examine {
-    mailbox: Mailbox,
-}
-
-impl fmt::Display for Examine {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "EXAMINE {}", self.mailbox)
-    }
-}
-
-struct Select {
-    mailbox: Mailbox,
-}
-
-impl fmt::Display for Select {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "SELECT {}", self.mailbox)
-    }
-}
-
-struct List {
-    mailbox: Mailbox,
-    list: String,
-}
-
-impl fmt::Display for List {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "LIST {} \"{}\"",
-            if self.mailbox.is_empty() {
-                format!("\"\"")
-            } else {
-                format!("{}", self.mailbox)
-            },
-            self.list.as_str()
-        )
-    }
-}
-
-struct Lsub {
-    mailbox: Mailbox,
-    list: String,
-}
-
-impl fmt::Display for Lsub {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "LSUB {} \"{}\"", self.mailbox, self.list)
-    }
-}
-
-enum StatusAttribute {
-    Messages,
-    Recent,
-    UidNext,
-    UidValidity,
-    Unseen,
-}
-
-impl fmt::Display for StatusAttribute {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "{}",
-            match self {
-                StatusAttribute::Messages => "MESSAGES",
-                StatusAttribute::Recent => "RECENT",
-                StatusAttribute::UidNext => "UIDNEXT",
-                StatusAttribute::UidValidity => "UIDVALIDITY",
-                StatusAttribute::Unseen => "UNSEEN",
-            }
-        )
-    }
-}
-
-struct Status {
-    mailbox: Mailbox,
-    status_attributes: Vec<StatusAttribute>,
-}
-
-impl fmt::Display for Status {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "STATUS {} ({})",
-            self.mailbox,
-            self.status_attributes.join(' ')
-        )
-    }
-}
-
-struct Store {
-    sequence_set: SequenceSet,
-    //store_att_flags:
-}
-
-impl fmt::Display for Store {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        unimplemented!()
-        //write!(f, "STORE {}", self.sequence_set)
-    }
-}
-
-struct Unsubscribe {
-    mailbox: Mailbox,
-}
-
-impl fmt::Display for Unsubscribe {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "UNSUBSCRIBE {}", self.mailbox)
-    }
-}
-
-struct Subscribe {
-    mailbox: Mailbox,
-}
-
-impl fmt::Display for Subscribe {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "SUBSCRIBE {}", self.mailbox)
-    }
-}
-
-struct Copy {
-    sequence_set: SequenceSet,
-    mailbox: Mailbox,
-}
-
-impl fmt::Display for Copy {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "COPY {} {}", self.sequence_set, self.mailbox)
-    }
-}
-
-struct Create {
-    mailbox: Mailbox,
-}
-
-impl fmt::Display for Create {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "CREATE {}", self.mailbox)
-    }
-}
-
-struct Rename {
-    from: Mailbox,
-    to: Mailbox,
-}
-
-impl fmt::Display for Rename {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "RENAME {} {}", self.from, self.to)
-    }
-}
-
-struct Append {
-    mailbox: Mailbox,
-    flag_list: Vec<Flag>,
-    date_time: Option<String>,
-    literal: String,
-}
-
-impl fmt::Display for Append {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "APPEND {}{}{} {}",
-            self.mailbox,
-            if self.flag_list.is_empty() {
-                String::from("")
-            } else {
-                format!(" {}", self.flag_list.join(' '))
-            },
-            if let Some(date_time) = self.date_time.as_ref() {
-                format!(" {}", date_time)
-            } else {
-                String::from("")
-            },
-            self.literal.as_str()
-        )
-    }
-}
-
-struct Fetch {
-    sequence_set: SequenceSet,
-}
-
-impl fmt::Display for Fetch {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "FETCH {}", self.sequence_set)
-    }
-}
-
-enum Flag {
-    Answered,
-    Flagged,
-    Deleted,
-    Seen,
-    Draft,
-    /*atom */
-    X(String),
-}
-
-impl fmt::Display for Flag {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "\\{}",
-            match self {
-                Flag::Answered => "Answered",
-                Flag::Flagged => "Flagged",
-                Flag::Deleted => "Deleted",
-                Flag::Seen => "Seen",
-                Flag::Draft => "Draft",
-                Flag::X(ref c) => c.as_str(),
-            }
-        )
-    }
-}
-
-enum Uid {
-    Copy(Copy),
-    Fetch(Fetch),
-    Search(Search),
-    Store(Store),
-}
-
-impl fmt::Display for Uid {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "UID {}",
-            match self {
-                Uid::Copy(ref c) => format!("{}", c),
-                Uid::Fetch(ref c) => format!("{}", c),
-                Uid::Search(ref c) => format!("{}", c),
-                Uid::Store(ref c) => format!("{}", c),
-            }
-        )
-    }
-}
-
-enum CommandSelect {
-    Check,
-    Close,
-    Expunge,
-    Copy(Copy),
-    Fetch(Fetch),
-    Store(Store),
-    Uid(Uid),
-    Search(Search),
-}
-
-impl fmt::Display for CommandSelect {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "{}",
-            match self {
-                CommandSelect::Check => format!("CHECK"),
-                CommandSelect::Close => format!("CLOSE"),
-                CommandSelect::Expunge => format!("EXPUNGE"),
-                CommandSelect::Copy(ref c) => format!("{}", c),
-                CommandSelect::Fetch(ref c) => format!("{}", c),
-                CommandSelect::Store(ref c) => format!("{}", c),
-                CommandSelect::Uid(ref c) => format!("{}", c),
-                CommandSelect::Search(ref c) => format!("{}", c),
-            }
-        )
-    }
-}
-
-/// Valid in all states
-enum CommandAny {
-    Capability,
-    Logout,
-    Noop,
-    XCommand(String),
-}
-
-impl fmt::Display for CommandAny {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "{}",
-            match self {
-                CommandAny::Capability => format!("CAPABILITY"),
-                CommandAny::Logout => format!("LOGOUT"),
-                CommandAny::Noop => format!("NOOP"),
-                CommandAny::XCommand(ref x) => format!("{}", x),
-            }
-        )
-    }
-}
-
-enum CommandAuth {
-    Append(Append),
-    Create(Create),
-    Delete(Delete),
-    Examine(Examine),
-    List(List),
-    Lsub(Lsub),
-    Rename(Rename),
-    Select(Select),
-    Status(Status),
-    Subscribe(Subscribe),
-    Unsubscribe(Unsubscribe),
-}
-
-impl fmt::Display for CommandAuth {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "{}",
-            match self {
-                CommandAuth::Append(ref c) => c.to_string(),
-                CommandAuth::Create(ref c) => c.to_string(),
-                CommandAuth::Delete(ref c) => c.to_string(),
-                CommandAuth::Examine(ref c) => c.to_string(),
-                CommandAuth::List(ref c) => c.to_string(),
-                CommandAuth::Lsub(ref c) => c.to_string(),
-                CommandAuth::Rename(ref c) => c.to_string(),
-                CommandAuth::Select(ref c) => c.to_string(),
-                CommandAuth::Status(ref c) => c.to_string(),
-                CommandAuth::Subscribe(ref c) => c.to_string(),
-                CommandAuth::Unsubscribe(ref c) => c.to_string(),
-            }
-        )
-    }
-}
-
-enum CommandNonAuth {
-    Login(String, String),
-    Authenticate(String, String),
-    StartTls,
-}
-
-impl fmt::Display for CommandNonAuth {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            CommandNonAuth::Login(ref userid, ref password) => {
-                write!(f, "LOGIN \"{}\" \"{}\"", userid, password)
-            }
-            CommandNonAuth::Authenticate(ref auth_type, ref base64) => {
-                write!(f, "AUTHENTICATE \"{}\" \"{}\"", auth_type, base64)
-            }
-            CommandNonAuth::StartTls => write!(f, "STARTTLS"),
-        }
-    }
-}
-
-enum Command {
-    Any(CommandAny),
-    Auth(CommandAuth),
-    NonAuth(CommandNonAuth),
-    Select(CommandSelect),
-}
-impl fmt::Display for Command {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            Command::Any(c) => write!(f, "{}", c),
-            Command::Auth(c) => write!(f, "{}", c),
-            Command::NonAuth(c) => write!(f, "{}", c),
-            Command::Select(c) => write!(f, "{}", c),
-        }
-    }
-}
-
-pub(super) struct ImapCommand {
-    tag: usize,
-    command: Command,
-}
-
-impl fmt::Display for ImapCommand {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{} {}\r\n", self.tag, self.command)
-    }
-}
-
-enum SeqNumber {
-    MsgNumber(NonZeroUsize),
-    UID(NonZeroUsize),
-    /** "*" represents the largest number in use.  In
-    the case of message sequence numbers, it is the number of messages in a
-    non-empty mailbox.  In the case of unique identifiers, it is the unique
-    identifier of the last message in the mailbox or, if the mailbox is empty, the
-    mailbox's current UIDNEXT value **/
-    Largest,
-}
-
-impl fmt::Display for SeqNumber {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            SeqNumber::MsgNumber(n) => write!(f, "{}", n),
-            SeqNumber::UID(u) => write!(f, "{}", u),
-            SeqNumber::Largest => write!(f, "*"),
-        }
-    }
-}
-
-struct SeqRange(SeqNumber, SeqNumber);
-impl fmt::Display for SeqRange {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}:{}", self.0, self.1)
-    }
-}
-
-struct SequenceSet {
-    numbers: Vec<SeqNumber>,
-    ranges: Vec<SeqRange>,
-}
-impl fmt::Display for SequenceSet {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "{}{}",
-            if self.numbers.is_empty() {
-                String::from("")
-            } else {
-                self.numbers.join(',')
-            },
-            if self.ranges.is_empty() {
-                String::from("")
-            } else {
-                self.ranges.join(',')
-            }
-        )
-    }
-}
-
-type Mailbox = String;
-type FlagKeyword = String;

melib/src/backends/imap/untagged.rs

@@ -0,0 +1,529 @@
+/*
+ * meli - imap
+ *
+ * Copyright 2020 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::{ImapConnection, MailboxSelection, UID};
+use crate::backends::imap::protocol_parser::{
+    generate_envelope_hash, FetchResponse, ImapLineSplit, RequiredResponses, UntaggedResponse,
+};
+use crate::backends::BackendMailbox;
+use crate::backends::{
+    RefreshEvent,
+    RefreshEventKind::{self, *},
+};
+use crate::error::*;
+use std::convert::TryInto;
+
+impl ImapConnection {
+    pub async fn process_untagged(&mut self, line: &[u8]) -> Result<bool> {
+        macro_rules! try_fail {
+            ($mailbox_hash: expr, $($result:expr $(,)*)+) => {
+                $(if let Err(err) = $result {
+                    self.uid_store.is_online.lock().unwrap().1 = Err(err.clone());
+                    debug!("failure: {}", err.to_string());
+                    self.add_refresh_event(RefreshEvent {
+                        account_hash: self.uid_store.account_hash,
+                        mailbox_hash: $mailbox_hash,
+                        kind: RefreshEventKind::Failure(err.clone()),
+                    });
+                    Err(err)
+                } else { Ok(()) }?;)+
+            };
+        }
+        let mailbox_hash = match self.stream.as_ref()?.current_mailbox {
+            MailboxSelection::Select(h) | MailboxSelection::Examine(h) => h,
+            MailboxSelection::None => return Ok(false),
+        };
+        let mailbox =
+            std::clone::Clone::clone(&self.uid_store.mailboxes.lock().await[&mailbox_hash]);
+
+        #[cfg(not(feature = "sqlite3"))]
+        let mut cache_handle = super::cache::DefaultCache::get(self.uid_store.clone())?;
+        #[cfg(feature = "sqlite3")]
+        let mut cache_handle = super::cache::Sqlite3Cache::get(self.uid_store.clone())?;
+        let mut response = Vec::with_capacity(8 * 1024);
+        let untagged_response =
+            match super::protocol_parser::untagged_responses(line).map(|(_, v, _)| v) {
+                Ok(None) | Err(_) => {
+                    return Ok(false);
+                }
+                Ok(Some(r)) => r,
+            };
+        match untagged_response {
+            UntaggedResponse::Bye { reason } => {
+                self.uid_store.is_online.lock().unwrap().1 = Err(reason.into());
+            }
+            UntaggedResponse::Expunge(n) => {
+                if self
+                    .uid_store
+                    .msn_index
+                    .lock()
+                    .unwrap()
+                    .get(&mailbox_hash)
+                    .map(|i| i.len() < TryInto::<usize>::try_into(n).unwrap())
+                    .unwrap_or(true)
+                {
+                    debug!(
+                        "Received expunge {} but mailbox msn index is {:?}",
+                        n,
+                        self.uid_store.msn_index.lock().unwrap().get(&mailbox_hash)
+                    );
+                    self.send_command("UID SEARCH 1:*".as_bytes()).await?;
+                    self.read_response(&mut response, RequiredResponses::SEARCH)
+                        .await?;
+                    let results = super::protocol_parser::search_results(&response)?
+                        .1
+                        .into_iter()
+                        .collect::<std::collections::BTreeSet<UID>>();
+                    {
+                        let mut lck = self.uid_store.msn_index.lock().unwrap();
+                        let msn_index = lck.entry(mailbox_hash).or_default();
+                        msn_index.clear();
+                        msn_index.extend(
+                            super::protocol_parser::search_results(&response)?
+                                .1
+                                .into_iter(),
+                        );
+                    }
+                    let mut events = vec![];
+                    for (deleted_uid, deleted_hash) in self
+                        .uid_store
+                        .uid_index
+                        .lock()
+                        .unwrap()
+                        .iter()
+                        .filter(|((mailbox_hash_, u), _)| {
+                            *mailbox_hash_ == mailbox_hash && !results.contains(u)
+                        })
+                        .map(|((_, uid), hash)| (*uid, *hash))
+                        .collect::<Vec<(UID, crate::email::EnvelopeHash)>>()
+                    {
+                        mailbox.exists.lock().unwrap().remove(deleted_hash);
+                        mailbox.unseen.lock().unwrap().remove(deleted_hash);
+                        self.uid_store
+                            .uid_index
+                            .lock()
+                            .unwrap()
+                            .remove(&(mailbox_hash, deleted_uid));
+                        self.uid_store
+                            .hash_index
+                            .lock()
+                            .unwrap()
+                            .remove(&deleted_hash);
+                        events.push((
+                            deleted_uid,
+                            RefreshEvent {
+                                account_hash: self.uid_store.account_hash,
+                                mailbox_hash,
+                                kind: Remove(deleted_hash),
+                            },
+                        ));
+                    }
+                    if self.uid_store.keep_offline_cache {
+                        cache_handle.update(mailbox_hash, &events)?;
+                    }
+                    for (_, event) in events {
+                        self.add_refresh_event(event);
+                    }
+
+                    return Ok(true);
+                }
+                let deleted_uid = self
+                    .uid_store
+                    .msn_index
+                    .lock()
+                    .unwrap()
+                    .entry(mailbox_hash)
+                    .or_default()
+                    .remove(TryInto::<usize>::try_into(n).unwrap().saturating_sub(1));
+                debug!("expunge {}, UID = {}", n, deleted_uid);
+                let deleted_hash: crate::email::EnvelopeHash = match self
+                    .uid_store
+                    .uid_index
+                    .lock()
+                    .unwrap()
+                    .remove(&(mailbox_hash, deleted_uid))
+                {
+                    Some(v) => v,
+                    None => return Ok(true),
+                };
+                mailbox.exists.lock().unwrap().remove(deleted_hash);
+                mailbox.unseen.lock().unwrap().remove(deleted_hash);
+                self.uid_store
+                    .hash_index
+                    .lock()
+                    .unwrap()
+                    .remove(&deleted_hash);
+                let mut event: [(UID, RefreshEvent); 1] = [(
+                    deleted_uid,
+                    RefreshEvent {
+                        account_hash: self.uid_store.account_hash,
+                        mailbox_hash,
+                        kind: Remove(deleted_hash),
+                    },
+                )];
+                if self.uid_store.keep_offline_cache {
+                    cache_handle.update(mailbox_hash, &event)?;
+                }
+                self.add_refresh_event(std::mem::replace(
+                    &mut event[0].1,
+                    RefreshEvent {
+                        account_hash: self.uid_store.account_hash,
+                        mailbox_hash,
+                        kind: Rescan,
+                    },
+                ));
+            }
+            UntaggedResponse::Exists(n) => {
+                debug!("exists {}", n);
+                try_fail!(
+                    mailbox_hash,
+                    self.send_command(format!("FETCH {} (UID FLAGS ENVELOPE BODY.PEEK[HEADER.FIELDS (REFERENCES)] BODYSTRUCTURE)", n).as_bytes()).await
+                    self.read_response(&mut response, RequiredResponses::FETCH_REQUIRED).await
+                );
+                let mut v = match super::protocol_parser::fetch_responses(&response) {
+                    Ok((_, v, _)) => v,
+                    Err(err) => {
+                        debug!(
+                            "Error when parsing FETCH response after untagged exists {:?}",
+                            err
+                        );
+                        return Ok(true);
+                    }
+                };
+                debug!("responses len is {}", v.len());
+                for FetchResponse {
+                    ref uid,
+                    ref mut envelope,
+                    ref mut flags,
+                    ref references,
+                    ..
+                } in &mut v
+                {
+                    if uid.is_none() || flags.is_none() || envelope.is_none() {
+                        continue;
+                    }
+                    let uid = uid.unwrap();
+                    let env = envelope.as_mut().unwrap();
+                    env.set_hash(generate_envelope_hash(&mailbox.imap_path(), &uid));
+                    if let Some(value) = references {
+                        env.set_references(value);
+                    }
+                    let mut tag_lck = self.uid_store.collection.tag_index.write().unwrap();
+                    if let Some((flags, keywords)) = flags {
+                        env.set_flags(*flags);
+                        if !env.is_seen() {
+                            mailbox.unseen.lock().unwrap().insert_new(env.hash());
+                        }
+                        for f in keywords {
+                            let hash = tag_hash!(f);
+                            if !tag_lck.contains_key(&hash) {
+                                tag_lck.insert(hash, f.to_string());
+                            }
+                            env.labels_mut().push(hash);
+                        }
+                    }
+                    mailbox.exists.lock().unwrap().insert_new(env.hash());
+                    if !self
+                        .uid_store
+                        .uid_index
+                        .lock()
+                        .unwrap()
+                        .contains_key(&(mailbox_hash, uid))
+                    {
+                        self.uid_store
+                            .msn_index
+                            .lock()
+                            .unwrap()
+                            .entry(mailbox_hash)
+                            .or_default()
+                            .push(uid);
+                    }
+                    self.uid_store
+                        .hash_index
+                        .lock()
+                        .unwrap()
+                        .insert(env.hash(), (uid, mailbox_hash));
+                    self.uid_store
+                        .uid_index
+                        .lock()
+                        .unwrap()
+                        .insert((mailbox_hash, uid), env.hash());
+                    debug!(
+                        "Create event {} {} {}",
+                        env.hash(),
+                        env.subject(),
+                        mailbox.path(),
+                    );
+                }
+                if self.uid_store.keep_offline_cache {
+                    if let Err(err) = cache_handle
+                        .insert_envelopes(mailbox_hash, &v)
+                        .chain_err_summary(|| {
+                            format!(
+                                "Could not save envelopes in cache for mailbox {}",
+                                &mailbox.imap_path()
+                            )
+                        })
+                    {
+                        crate::log(err.to_string(), crate::INFO);
+                    }
+                }
+                for response in v {
+                    if let FetchResponse {
+                        envelope: Some(envelope),
+                        ..
+                    } = response
+                    {
+                        self.add_refresh_event(RefreshEvent {
+                            account_hash: self.uid_store.account_hash,
+                            mailbox_hash,
+                            kind: Create(Box::new(envelope)),
+                        });
+                    }
+                }
+            }
+            UntaggedResponse::Recent(_) => {
+                try_fail!(
+                    mailbox_hash,
+                    self.send_command(b"UID SEARCH RECENT").await
+                    self.read_response(&mut response, RequiredResponses::SEARCH).await
+                );
+                match super::protocol_parser::search_results_raw(&response)
+                    .map(|(_, v)| v)
+                    .map_err(MeliError::from)
+                {
+                    Ok(&[]) => {
+                        debug!("UID SEARCH RECENT returned no results");
+                    }
+                    Ok(v) => {
+                        let command = {
+                            let mut iter = v.split(u8::is_ascii_whitespace);
+                            let first = iter.next().unwrap_or(v);
+                            let mut accum = format!("{}", to_str!(first).trim());
+                            for ms in iter {
+                                accum = format!("{},{}", accum, to_str!(ms).trim());
+                            }
+                            format!("UID FETCH {} (UID FLAGS ENVELOPE BODY.PEEK[HEADER.FIELDS (REFERENCES)] BODYSTRUCTURE)", accum)
+                        };
+                        try_fail!(
+                            mailbox_hash,
+                            self.send_command(command.as_bytes()).await
+                            self.read_response(&mut response, RequiredResponses::FETCH_REQUIRED).await
+                        );
+                        let mut v = match super::protocol_parser::fetch_responses(&response) {
+                            Ok((_, v, _)) => v,
+                            Err(err) => {
+                                debug!(
+                                    "Error when parsing FETCH response after untagged recent {:?}",
+                                    err
+                                );
+                                return Ok(true);
+                            }
+                        };
+                        debug!("responses len is {}", v.len());
+                        for FetchResponse {
+                            ref uid,
+                            ref mut envelope,
+                            ref mut flags,
+                            ref references,
+                            ..
+                        } in &mut v
+                        {
+                            if uid.is_none() || flags.is_none() || envelope.is_none() {
+                                continue;
+                            }
+                            let uid = uid.unwrap();
+                            let env = envelope.as_mut().unwrap();
+                            env.set_hash(generate_envelope_hash(&mailbox.imap_path(), &uid));
+                            if let Some(value) = references {
+                                env.set_references(value);
+                            }
+                            let mut tag_lck = self.uid_store.collection.tag_index.write().unwrap();
+                            if let Some((flags, keywords)) = flags {
+                                env.set_flags(*flags);
+                                if !env.is_seen() {
+                                    mailbox.unseen.lock().unwrap().insert_new(env.hash());
+                                }
+                                for f in keywords {
+                                    let hash = tag_hash!(f);
+                                    if !tag_lck.contains_key(&hash) {
+                                        tag_lck.insert(hash, f.to_string());
+                                    }
+                                    env.labels_mut().push(hash);
+                                }
+                            }
+                            mailbox.exists.lock().unwrap().insert_new(env.hash());
+                        }
+                        if self.uid_store.keep_offline_cache {
+                            if let Err(err) = cache_handle
+                                .insert_envelopes(mailbox_hash, &v)
+                                .chain_err_summary(|| {
+                                    format!(
+                                        "Could not save envelopes in cache for mailbox {}",
+                                        &mailbox.imap_path()
+                                    )
+                                })
+                            {
+                                crate::log(err.to_string(), crate::INFO);
+                            }
+                        }
+                        for response in v {
+                            if let FetchResponse {
+                                envelope: Some(envelope),
+                                uid: Some(uid),
+                                ..
+                            } = response
+                            {
+                                if !self
+                                    .uid_store
+                                    .uid_index
+                                    .lock()
+                                    .unwrap()
+                                    .contains_key(&(mailbox_hash, uid))
+                                {
+                                    self.uid_store
+                                        .msn_index
+                                        .lock()
+                                        .unwrap()
+                                        .entry(mailbox_hash)
+                                        .or_default()
+                                        .push(uid);
+                                }
+                                self.uid_store
+                                    .hash_index
+                                    .lock()
+                                    .unwrap()
+                                    .insert(envelope.hash(), (uid, mailbox_hash));
+                                self.uid_store
+                                    .uid_index
+                                    .lock()
+                                    .unwrap()
+                                    .insert((mailbox_hash, uid), envelope.hash());
+                                debug!(
+                                    "Create event {} {} {}",
+                                    envelope.hash(),
+                                    envelope.subject(),
+                                    mailbox.path(),
+                                );
+                                self.add_refresh_event(RefreshEvent {
+                                    account_hash: self.uid_store.account_hash,
+                                    mailbox_hash,
+                                    kind: Create(Box::new(envelope)),
+                                });
+                            }
+                        }
+                    }
+                    Err(e) => {
+                        debug!(
+                            "UID SEARCH RECENT err: {}\nresp: {}",
+                            e.to_string(),
+                            to_str!(&response)
+                        );
+                    }
+                }
+            }
+            UntaggedResponse::Fetch(FetchResponse {
+                uid,
+                message_sequence_number: msg_seq,
+                modseq,
+                flags,
+                body: _,
+                references: _,
+                envelope: _,
+                raw_fetch_value: _,
+            }) => {
+                if let Some(flags) = flags {
+                    let uid = if let Some(uid) = uid {
+                        uid
+                    } else {
+                        try_fail!(
+                            mailbox_hash,
+                            self.send_command(format!("UID SEARCH {}", msg_seq).as_bytes())
+                                .await,
+                            self.read_response(&mut response, RequiredResponses::SEARCH)
+                                .await,
+                        );
+                        match super::protocol_parser::search_results(
+                            response.split_rn().next().unwrap_or(b""),
+                        )
+                        .map(|(_, v)| v)
+                        {
+                            Ok(mut v) if v.len() == 1 => v.pop().unwrap(),
+                            Ok(_) => {
+                                return Ok(false);
+                            }
+                            Err(e) => {
+                                debug!("SEARCH error failed: {}", e);
+                                debug!(to_str!(&response));
+                                return Ok(false);
+                            }
+                        }
+                    };
+                    debug!("fetch uid {} {:?}", uid, flags);
+                    if let Some(env_hash) = {
+                        let temp = self
+                            .uid_store
+                            .uid_index
+                            .lock()
+                            .unwrap()
+                            .get(&(mailbox_hash, uid))
+                            .copied();
+                        temp
+                    } {
+                        if !flags.0.intersects(crate::email::Flag::SEEN) {
+                            mailbox.unseen.lock().unwrap().insert_new(env_hash);
+                        } else {
+                            mailbox.unseen.lock().unwrap().remove(env_hash);
+                        }
+                        mailbox.exists.lock().unwrap().insert_new(env_hash);
+                        if let Some(modseq) = modseq {
+                            self.uid_store
+                                .modseq
+                                .lock()
+                                .unwrap()
+                                .insert(env_hash, modseq);
+                        }
+                        let mut event: [(UID, RefreshEvent); 1] = [(
+                            uid,
+                            RefreshEvent {
+                                account_hash: self.uid_store.account_hash,
+                                mailbox_hash,
+                                kind: NewFlags(env_hash, flags),
+                            },
+                        )];
+                        if self.uid_store.keep_offline_cache {
+                            cache_handle.update(mailbox_hash, &event)?;
+                        }
+                        self.add_refresh_event(std::mem::replace(
+                            &mut event[0].1,
+                            RefreshEvent {
+                                account_hash: self.uid_store.account_hash,
+                                mailbox_hash,
+                                kind: Rescan,
+                            },
+                        ));
+                    };
+                }
+            }
+        }
+        Ok(true)
+    }
+}

melib/src/backends/jmap.rs

@@ -0,0 +1,874 @@
+/*
+ * meli - jmap module.
+ *
+ * Copyright 2019 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use crate::backends::*;
+use crate::conf::AccountSettings;
+use crate::email::*;
+use crate::error::{MeliError, Result};
+use crate::Collection;
+use futures::lock::Mutex as FutureMutex;
+use isahc::config::RedirectPolicy;
+use isahc::prelude::HttpClient;
+use isahc::ResponseExt;
+use serde_json::Value;
+use std::collections::{hash_map::DefaultHasher, HashMap, HashSet};
+use std::convert::TryFrom;
+use std::hash::{Hash, Hasher};
+use std::str::FromStr;
+use std::sync::{Arc, Mutex, RwLock};
+use std::time::Instant;
+
+macro_rules! tag_hash {
+    ($t:ident) => {{
+        let mut hasher = DefaultHasher::default();
+        $t.hash(&mut hasher);
+        hasher.finish()
+    }};
+    ($t:literal) => {{
+        let mut hasher = DefaultHasher::default();
+        $t.hash(&mut hasher);
+        hasher.finish()
+    }};
+}
+
+#[macro_export]
+macro_rules! _impl {
+        ($(#[$outer:meta])*$field:ident : $t:ty) => {
+            $(#[$outer])*
+            pub fn $field(mut self, new_val: $t) -> Self {
+                self.$field = new_val;
+                self
+            }
+        };
+        (get_mut $(#[$outer:meta])*$method:ident, $field:ident : $t:ty) => {
+            $(#[$outer])*
+            pub fn $method(&mut self) -> &mut $t {
+                &mut self.$field
+            }
+        };
+        (get $(#[$outer:meta])*$method:ident, $field:ident : $t:ty) => {
+            $(#[$outer])*
+            pub fn $method(&self) -> &$t {
+                &self.$field
+            }
+        }
+    }
+
+pub mod operations;
+use operations::*;
+
+pub mod connection;
+use connection::*;
+
+pub mod protocol;
+use protocol::*;
+
+pub mod rfc8620;
+use rfc8620::*;
+
+pub mod objects;
+use objects::*;
+
+pub mod mailbox;
+use mailbox::*;
+
+#[derive(Debug, Default)]
+pub struct EnvelopeCache {
+    bytes: Option<String>,
+    headers: Option<String>,
+    body: Option<String>,
+    flags: Option<Flag>,
+}
+
+#[derive(Debug, Clone)]
+pub struct JmapServerConf {
+    pub server_hostname: String,
+    pub server_username: String,
+    pub server_password: String,
+    pub server_port: u16,
+    pub danger_accept_invalid_certs: bool,
+}
+
+macro_rules! get_conf_val {
+    ($s:ident[$var:literal]) => {
+        $s.extra.get($var).ok_or_else(|| {
+            MeliError::new(format!(
+                "Configuration error ({}): JMAP connection requires the field `{}` set",
+                $s.name.as_str(),
+                $var
+            ))
+        })
+    };
+    ($s:ident[$var:literal], $default:expr) => {
+        $s.extra
+            .get($var)
+            .map(|v| {
+                <_>::from_str(v).map_err(|e| {
+                    MeliError::new(format!(
+                        "Configuration error ({}): Invalid value for field `{}`: {}\n{}",
+                        $s.name.as_str(),
+                        $var,
+                        v,
+                        e
+                    ))
+                })
+            })
+            .unwrap_or_else(|| Ok($default))
+    };
+}
+
+impl JmapServerConf {
+    pub fn new(s: &AccountSettings) -> Result<Self> {
+        Ok(JmapServerConf {
+            server_hostname: get_conf_val!(s["server_hostname"])?.to_string(),
+            server_username: get_conf_val!(s["server_username"])?.to_string(),
+            server_password: get_conf_val!(s["server_password"])?.to_string(),
+            server_port: get_conf_val!(s["server_port"], 443)?,
+            danger_accept_invalid_certs: get_conf_val!(s["danger_accept_invalid_certs"], false)?,
+        })
+    }
+}
+
+macro_rules! get_conf_val {
+    ($s:ident[$var:literal]) => {
+        $s.extra.get($var).ok_or_else(|| {
+            MeliError::new(format!(
+                "Configuration error ({}): JMAP connection requires the field `{}` set",
+                $s.name.as_str(),
+                $var
+            ))
+        })
+    };
+    ($s:ident[$var:literal], $default:expr) => {
+        $s.extra
+            .get($var)
+            .map(|v| {
+                <_>::from_str(v).map_err(|e| {
+                    MeliError::new(format!(
+                        "Configuration error ({}): Invalid value for field `{}`: {}\n{}",
+                        $s.name.as_str(),
+                        $var,
+                        v,
+                        e
+                    ))
+                })
+            })
+            .unwrap_or_else(|| Ok($default))
+    };
+}
+
+#[derive(Debug)]
+pub struct Store {
+    pub account_name: Arc<String>,
+    pub account_hash: AccountHash,
+    pub account_id: Arc<Mutex<Id<Account>>>,
+    pub byte_cache: Arc<Mutex<HashMap<EnvelopeHash, EnvelopeCache>>>,
+    pub id_store: Arc<Mutex<HashMap<EnvelopeHash, Id<EmailObject>>>>,
+    pub reverse_id_store: Arc<Mutex<HashMap<Id<EmailObject>, EnvelopeHash>>>,
+    pub blob_id_store: Arc<Mutex<HashMap<EnvelopeHash, Id<BlobObject>>>>,
+    pub collection: Collection,
+    pub mailboxes: Arc<RwLock<HashMap<MailboxHash, JmapMailbox>>>,
+    pub mailboxes_index: Arc<RwLock<HashMap<MailboxHash, HashSet<EnvelopeHash>>>>,
+    pub mailbox_state: Arc<Mutex<State<MailboxObject>>>,
+    pub online_status: Arc<FutureMutex<(Instant, Result<()>)>>,
+    pub is_subscribed: Arc<IsSubscribedFn>,
+    pub event_consumer: BackendEventConsumer,
+}
+
+impl Store {
+    pub fn add_envelope(&self, obj: EmailObject) -> Envelope {
+        let mut tag_lck = self.collection.tag_index.write().unwrap();
+        let tags = obj
+            .keywords()
+            .keys()
+            .map(|tag| {
+                let tag_hash = {
+                    let mut hasher = DefaultHasher::default();
+                    tag.hash(&mut hasher);
+                    hasher.finish()
+                };
+                if !tag_lck.contains_key(&tag_hash) {
+                    tag_lck.insert(tag_hash, tag.to_string());
+                }
+                tag_hash
+            })
+            .collect::<SmallVec<[u64; 1024]>>();
+        let id = obj.id.clone();
+        let mailbox_ids = obj.mailbox_ids.clone();
+        let blob_id = obj.blob_id.clone();
+        drop(tag_lck);
+        let mut ret: Envelope = obj.into();
+
+        debug_assert_eq!(tag_hash!("$draft"), 6613915297903591176);
+        debug_assert_eq!(tag_hash!("$seen"), 1683863812294339685);
+        debug_assert_eq!(tag_hash!("$flagged"), 2714010747478170100);
+        debug_assert_eq!(tag_hash!("$answered"), 8940855303929342213);
+        debug_assert_eq!(tag_hash!("$junk"), 2656839745430720464);
+        debug_assert_eq!(tag_hash!("$notjunk"), 4091323799684325059);
+        let mut id_store_lck = self.id_store.lock().unwrap();
+        let mut reverse_id_store_lck = self.reverse_id_store.lock().unwrap();
+        let mut blob_id_store_lck = self.blob_id_store.lock().unwrap();
+        let mailboxes_lck = self.mailboxes.read().unwrap();
+        let mut mailboxes_index_lck = self.mailboxes_index.write().unwrap();
+        for (mailbox_id, _) in mailbox_ids {
+            if let Some((mailbox_hash, _)) = mailboxes_lck.iter().find(|(_, m)| m.id == mailbox_id)
+            {
+                mailboxes_index_lck
+                    .entry(*mailbox_hash)
+                    .or_default()
+                    .insert(ret.hash());
+            }
+        }
+        reverse_id_store_lck.insert(id.clone(), ret.hash());
+        id_store_lck.insert(ret.hash(), id);
+        blob_id_store_lck.insert(ret.hash(), blob_id);
+        for t in tags {
+            match t {
+                6613915297903591176 => {
+                    ret.set_flags(ret.flags() | Flag::DRAFT);
+                }
+                1683863812294339685 => {
+                    ret.set_flags(ret.flags() | Flag::SEEN);
+                }
+                2714010747478170100 => {
+                    ret.set_flags(ret.flags() | Flag::FLAGGED);
+                }
+                8940855303929342213 => {
+                    ret.set_flags(ret.flags() | Flag::REPLIED);
+                }
+                2656839745430720464 | 4091323799684325059 => { /* ignore */ }
+                _ => ret.labels_mut().push(t),
+            }
+        }
+        ret
+    }
+
+    pub fn remove_envelope(
+        &self,
+        obj_id: Id<EmailObject>,
+    ) -> Option<(EnvelopeHash, SmallVec<[MailboxHash; 8]>)> {
+        let env_hash = self.reverse_id_store.lock().unwrap().remove(&obj_id)?;
+        self.id_store.lock().unwrap().remove(&env_hash);
+        self.blob_id_store.lock().unwrap().remove(&env_hash);
+        self.byte_cache.lock().unwrap().remove(&env_hash);
+        let mut mailbox_hashes = SmallVec::new();
+        for (k, set) in self.mailboxes_index.write().unwrap().iter_mut() {
+            if set.remove(&env_hash) {
+                mailbox_hashes.push(*k);
+            }
+        }
+        Some((env_hash, mailbox_hashes))
+    }
+}
+
+#[derive(Debug)]
+pub struct JmapType {
+    server_conf: JmapServerConf,
+    connection: Arc<FutureMutex<JmapConnection>>,
+    store: Arc<Store>,
+}
+
+impl MailBackend for JmapType {
+    fn capabilities(&self) -> MailBackendCapabilities {
+        const CAPABILITIES: MailBackendCapabilities = MailBackendCapabilities {
+            is_async: true,
+            is_remote: true,
+            supports_search: true,
+            extensions: None,
+            supports_tags: true,
+            supports_submission: false,
+        };
+        CAPABILITIES
+    }
+
+    fn is_online(&self) -> ResultFuture<()> {
+        let online = self.store.online_status.clone();
+        Ok(Box::pin(async move {
+            //match timeout(std::time::Duration::from_secs(3), connection.lock()).await {
+            let online_lck = online.lock().await;
+            if online_lck.1.is_err()
+                && Instant::now().duration_since(online_lck.0) >= std::time::Duration::new(2, 0)
+            {
+                //let _ = self.mailboxes();
+            }
+            online_lck.1.clone()
+        }))
+    }
+
+    fn fetch(
+        &mut self,
+        mailbox_hash: MailboxHash,
+    ) -> Result<Pin<Box<dyn Stream<Item = Result<Vec<Envelope>>> + Send + 'static>>> {
+        let store = self.store.clone();
+        let connection = self.connection.clone();
+        Ok(Box::pin(async_stream::try_stream! {
+            let mut conn = connection.lock().await;
+            conn.connect().await?;
+            let res = protocol::fetch(
+                &conn,
+                &store,
+                mailbox_hash,
+            ).await?;
+            yield res;
+        }))
+    }
+
+    fn refresh(&mut self, mailbox_hash: MailboxHash) -> ResultFuture<()> {
+        let connection = self.connection.clone();
+        Ok(Box::pin(async move {
+            let mut conn = connection.lock().await;
+            conn.connect().await?;
+            conn.email_changes(mailbox_hash).await?;
+            Ok(())
+        }))
+    }
+
+    fn watch(&self) -> ResultFuture<()> {
+        let connection = self.connection.clone();
+        let store = self.store.clone();
+        Ok(Box::pin(async move {
+            {
+                let mut conn = connection.lock().await;
+                conn.connect().await?;
+            }
+            loop {
+                {
+                    let mailbox_hashes = {
+                        store
+                            .mailboxes
+                            .read()
+                            .unwrap()
+                            .keys()
+                            .cloned()
+                            .collect::<SmallVec<[MailboxHash; 16]>>()
+                    };
+                    let conn = connection.lock().await;
+                    for mailbox_hash in mailbox_hashes {
+                        conn.email_changes(mailbox_hash).await?;
+                    }
+                }
+                crate::connections::sleep(std::time::Duration::from_secs(60)).await;
+            }
+        }))
+    }
+
+    fn mailboxes(&self) -> ResultFuture<HashMap<MailboxHash, Mailbox>> {
+        let store = self.store.clone();
+        let connection = self.connection.clone();
+        Ok(Box::pin(async move {
+            let mut conn = connection.lock().await;
+            conn.connect().await?;
+            if store.mailboxes.read().unwrap().is_empty() {
+                let new_mailboxes = debug!(protocol::get_mailboxes(&conn).await)?;
+                *store.mailboxes.write().unwrap() = new_mailboxes;
+            }
+
+            let ret = store
+                .mailboxes
+                .read()
+                .unwrap()
+                .iter()
+                .filter(|(_, f)| f.is_subscribed)
+                .map(|(&h, f)| (h, BackendMailbox::clone(f) as Mailbox))
+                .collect();
+
+            Ok(ret)
+        }))
+    }
+
+    fn operation(&self, hash: EnvelopeHash) -> Result<Box<dyn BackendOp>> {
+        Ok(Box::new(JmapOp::new(
+            hash,
+            self.connection.clone(),
+            self.store.clone(),
+        )))
+    }
+
+    fn save(
+        &self,
+        bytes: Vec<u8>,
+        mailbox_hash: MailboxHash,
+        _flags: Option<Flag>,
+    ) -> ResultFuture<()> {
+        let store = self.store.clone();
+        let connection = self.connection.clone();
+        Ok(Box::pin(async move {
+            let mut conn = connection.lock().await;
+            conn.connect().await?;
+            /*
+             * 1. upload binary blob, get blobId
+             * 2. Email/import
+             */
+            let (api_url, upload_url) = {
+                let lck = conn.session.lock().unwrap();
+                (lck.api_url.clone(), lck.upload_url.clone())
+            };
+            let mut res = conn
+                .client
+                .post_async(
+                    &upload_request_format(upload_url.as_str(), &conn.mail_account_id()),
+                    bytes,
+                )
+                .await?;
+
+            let mailbox_id: Id<MailboxObject> = {
+                let mailboxes_lck = store.mailboxes.read().unwrap();
+                if let Some(mailbox) = mailboxes_lck.get(&mailbox_hash) {
+                    mailbox.id.clone()
+                } else {
+                    return Err(MeliError::new(format!(
+                        "Mailbox with hash {} not found",
+                        mailbox_hash
+                    )));
+                }
+            };
+            let res_text = res.text_async().await?;
+
+            let upload_response: UploadResponse = serde_json::from_str(&res_text)?;
+            let mut req = Request::new(conn.request_no.clone());
+            let creation_id: Id<EmailObject> = "1".to_string().into();
+            let mut email_imports = HashMap::default();
+            let mut mailbox_ids = HashMap::default();
+            mailbox_ids.insert(mailbox_id, true);
+            email_imports.insert(
+                creation_id.clone(),
+                EmailImport::new()
+                    .blob_id(upload_response.blob_id)
+                    .mailbox_ids(mailbox_ids),
+            );
+
+            let import_call: ImportCall = ImportCall::new()
+                .account_id(conn.mail_account_id().clone())
+                .emails(email_imports);
+
+            req.add_call(&import_call);
+            let mut res = conn
+                .client
+                .post_async(api_url.as_str(), serde_json::to_string(&req)?)
+                .await?;
+            let res_text = res.text_async().await?;
+
+            let mut v: MethodResponse = serde_json::from_str(&res_text)?;
+            let m = ImportResponse::try_from(v.method_responses.remove(0)).or_else(|err| {
+                let ierr: Result<ImportError> =
+                    serde_json::from_str(&res_text).map_err(|err| err.into());
+                if let Ok(err) = ierr {
+                    Err(MeliError::new(format!("Could not save message: {:?}", err)))
+                } else {
+                    Err(err.into())
+                }
+            })?;
+
+            if let Some(err) = m.not_created.get(&creation_id) {
+                return Err(MeliError::new(format!("Could not save message: {:?}", err)));
+            }
+            Ok(())
+        }))
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn as_any_mut(&mut self) -> &mut dyn Any {
+        self
+    }
+
+    fn collection(&self) -> Collection {
+        self.store.collection.clone()
+    }
+
+    fn search(
+        &self,
+        q: crate::search::Query,
+        mailbox_hash: Option<MailboxHash>,
+    ) -> ResultFuture<SmallVec<[EnvelopeHash; 512]>> {
+        let store = self.store.clone();
+        let connection = self.connection.clone();
+        let filter = if let Some(mailbox_hash) = mailbox_hash {
+            let mailbox_id = self.store.mailboxes.read().unwrap()[&mailbox_hash]
+                .id
+                .clone();
+
+            let mut f = Filter::Condition(
+                EmailFilterCondition::new()
+                    .in_mailbox(Some(mailbox_id))
+                    .into(),
+            );
+            f &= Filter::<EmailFilterCondition, EmailObject>::from(q);
+            f
+        } else {
+            Filter::<EmailFilterCondition, EmailObject>::from(q)
+        };
+
+        Ok(Box::pin(async move {
+            let mut conn = connection.lock().await;
+            conn.connect().await?;
+            let email_call: EmailQuery = EmailQuery::new(
+                Query::new()
+                    .account_id(conn.mail_account_id().clone())
+                    .filter(Some(filter))
+                    .position(0),
+            )
+            .collapse_threads(false);
+
+            let mut req = Request::new(conn.request_no.clone());
+            req.add_call(&email_call);
+            let api_url = conn.session.lock().unwrap().api_url.clone();
+
+            let mut res = conn
+                .client
+                .post_async(api_url.as_str(), serde_json::to_string(&req)?)
+                .await?;
+
+            let res_text = res.text_async().await?;
+            let mut v: MethodResponse = serde_json::from_str(&res_text).unwrap();
+            *store.online_status.lock().await = (std::time::Instant::now(), Ok(()));
+            let m = QueryResponse::<EmailObject>::try_from(v.method_responses.remove(0))?;
+            let QueryResponse::<EmailObject> { ids, .. } = m;
+            let ret = ids.into_iter().map(|id| id.into_hash()).collect();
+            Ok(ret)
+        }))
+    }
+
+    fn rename_mailbox(
+        &mut self,
+        _mailbox_hash: MailboxHash,
+        _new_path: String,
+    ) -> ResultFuture<Mailbox> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn create_mailbox(
+        &mut self,
+        _path: String,
+    ) -> ResultFuture<(MailboxHash, HashMap<MailboxHash, Mailbox>)> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn copy_messages(
+        &mut self,
+        env_hashes: EnvelopeHashBatch,
+        source_mailbox_hash: MailboxHash,
+        destination_mailbox_hash: MailboxHash,
+        move_: bool,
+    ) -> ResultFuture<()> {
+        let store = self.store.clone();
+        let connection = self.connection.clone();
+        Ok(Box::pin(async move {
+            let (source_mailbox_id, destination_mailbox_id) = {
+                let mailboxes_lck = store.mailboxes.read().unwrap();
+                if !mailboxes_lck.contains_key(&source_mailbox_hash) {
+                    return Err(MeliError::new(format!(
+                        "Could not find source mailbox with hash {}",
+                        source_mailbox_hash
+                    )));
+                }
+                if !mailboxes_lck.contains_key(&destination_mailbox_hash) {
+                    return Err(MeliError::new(format!(
+                        "Could not find destination mailbox with hash {}",
+                        destination_mailbox_hash
+                    )));
+                }
+
+                (
+                    mailboxes_lck[&source_mailbox_hash].id.clone(),
+                    mailboxes_lck[&destination_mailbox_hash].id.clone(),
+                )
+            };
+            let mut update_map: HashMap<Id<EmailObject>, Value> = HashMap::default();
+            let mut ids: Vec<Id<EmailObject>> = Vec::with_capacity(env_hashes.rest.len() + 1);
+            let mut id_map: HashMap<Id<EmailObject>, EnvelopeHash> = HashMap::default();
+            let mut update_keywords: HashMap<String, Value> = HashMap::default();
+            update_keywords.insert(
+                format!("mailboxIds/{}", &destination_mailbox_id),
+                serde_json::json!(true),
+            );
+            if move_ {
+                update_keywords.insert(
+                    format!("mailboxIds/{}", &source_mailbox_id),
+                    serde_json::json!(null),
+                );
+            }
+            {
+                for env_hash in env_hashes.iter() {
+                    if let Some(id) = store.id_store.lock().unwrap().get(&env_hash) {
+                        ids.push(id.clone());
+                        id_map.insert(id.clone(), env_hash);
+                        update_map.insert(id.clone(), serde_json::json!(update_keywords.clone()));
+                    }
+                }
+            }
+            let conn = connection.lock().await;
+            let api_url = conn.session.lock().unwrap().api_url.clone();
+
+            let email_set_call: EmailSet = EmailSet::new(
+                Set::<EmailObject>::new()
+                    .account_id(conn.mail_account_id().clone())
+                    .update(Some(update_map)),
+            );
+
+            let mut req = Request::new(conn.request_no.clone());
+            let _prev_seq = req.add_call(&email_set_call);
+
+            let mut res = conn
+                .client
+                .post_async(api_url.as_str(), serde_json::to_string(&req)?)
+                .await?;
+
+            let res_text = res.text_async().await?;
+
+            let mut v: MethodResponse = serde_json::from_str(&res_text).unwrap();
+            *store.online_status.lock().await = (std::time::Instant::now(), Ok(()));
+            let m = SetResponse::<EmailObject>::try_from(v.method_responses.remove(0))?;
+            if let Some(ids) = m.not_updated {
+                if !ids.is_empty() {
+                    return Err(MeliError::new(format!(
+                        "Could not update ids: {}",
+                        ids.into_iter()
+                            .map(|err| err.to_string())
+                            .collect::<Vec<String>>()
+                            .join(",")
+                    )));
+                }
+            }
+            Ok(())
+        }))
+    }
+
+    fn set_flags(
+        &mut self,
+        env_hashes: EnvelopeHashBatch,
+        mailbox_hash: MailboxHash,
+        flags: SmallVec<[(std::result::Result<Flag, String>, bool); 8]>,
+    ) -> ResultFuture<()> {
+        let store = self.store.clone();
+        let connection = self.connection.clone();
+        Ok(Box::pin(async move {
+            let mut update_map: HashMap<Id<EmailObject>, Value> = HashMap::default();
+            let mut ids: Vec<Id<EmailObject>> = Vec::with_capacity(env_hashes.rest.len() + 1);
+            let mut id_map: HashMap<Id<EmailObject>, EnvelopeHash> = HashMap::default();
+            let mut update_keywords: HashMap<String, Value> = HashMap::default();
+            for (flag, value) in flags.iter() {
+                match flag {
+                    Ok(f) => {
+                        update_keywords.insert(
+                            format!(
+                                "keywords/{}",
+                                match *f {
+                                    Flag::DRAFT => "$draft",
+                                    Flag::FLAGGED => "$flagged",
+                                    Flag::SEEN => "$seen",
+                                    Flag::REPLIED => "$answered",
+                                    Flag::TRASHED => "$junk",
+                                    Flag::PASSED => "$passed",
+                                    _ => continue, //FIXME
+                                }
+                            ),
+                            if *value {
+                                serde_json::json!(true)
+                            } else {
+                                serde_json::json!(null)
+                            },
+                        );
+                    }
+                    Err(t) => {
+                        update_keywords.insert(
+                            format!("keywords/{}", t),
+                            if *value {
+                                serde_json::json!(true)
+                            } else {
+                                serde_json::json!(null)
+                            },
+                        );
+                    }
+                }
+            }
+            {
+                for hash in env_hashes.iter() {
+                    if let Some(id) = store.id_store.lock().unwrap().get(&hash) {
+                        ids.push(id.clone());
+                        id_map.insert(id.clone(), hash);
+                        update_map.insert(id.clone(), serde_json::json!(update_keywords.clone()));
+                    }
+                }
+            }
+            let conn = connection.lock().await;
+
+            let email_set_call: EmailSet = EmailSet::new(
+                Set::<EmailObject>::new()
+                    .account_id(conn.mail_account_id().clone())
+                    .update(Some(update_map)),
+            );
+
+            let mut req = Request::new(conn.request_no.clone());
+            req.add_call(&email_set_call);
+            let email_call: EmailGet = EmailGet::new(
+                Get::new()
+                    .ids(Some(JmapArgument::Value(ids)))
+                    .account_id(conn.mail_account_id().clone())
+                    .properties(Some(vec!["keywords".to_string()])),
+            );
+
+            req.add_call(&email_call);
+            let api_url = conn.session.lock().unwrap().api_url.clone();
+            //debug!(serde_json::to_string(&req)?);
+            let mut res = conn
+                .client
+                .post_async(api_url.as_str(), serde_json::to_string(&req)?)
+                .await?;
+
+            let res_text = res.text_async().await?;
+            /*
+             *{"methodResponses":[["Email/set",{"notUpdated":null,"notDestroyed":null,"oldState":"86","newState":"87","accountId":"u148940c7","updated":{"M045926eed54b11423918f392":{"id":"M045926eed54b11423918f392"}},"created":null,"destroyed":null,"notCreated":null},"m3"]],"sessionState":"cyrus-0;p-5;vfs-0"}
+             */
+            //debug!("res_text = {}", &res_text);
+            let mut v: MethodResponse = serde_json::from_str(&res_text).unwrap();
+            *store.online_status.lock().await = (std::time::Instant::now(), Ok(()));
+            let m = SetResponse::<EmailObject>::try_from(v.method_responses.remove(0))?;
+            if let Some(ids) = m.not_updated {
+                return Err(MeliError::new(
+                    ids.into_iter()
+                        .map(|err| err.to_string())
+                        .collect::<Vec<String>>()
+                        .join(","),
+                ));
+            }
+
+            {
+                let mut tag_index_lck = store.collection.tag_index.write().unwrap();
+                for (flag, value) in flags.iter() {
+                    match flag {
+                        Ok(_) => {}
+                        Err(t) => {
+                            if *value {
+                                tag_index_lck.insert(tag_hash!(t), t.clone());
+                            }
+                        }
+                    }
+                }
+                drop(tag_index_lck);
+            }
+            let e = GetResponse::<EmailObject>::try_from(v.method_responses.pop().unwrap())?;
+            let GetResponse::<EmailObject> { list, state, .. } = e;
+            {
+                let (is_empty, is_equal) = {
+                    let mailboxes_lck = conn.store.mailboxes.read().unwrap();
+                    mailboxes_lck
+                        .get(&mailbox_hash)
+                        .map(|mbox| {
+                            let current_state_lck = mbox.email_state.lock().unwrap();
+                            (
+                                current_state_lck.is_some(),
+                                current_state_lck.as_ref() != Some(&state),
+                            )
+                        })
+                        .unwrap_or((true, true))
+                };
+                if is_empty {
+                    let mut mailboxes_lck = conn.store.mailboxes.write().unwrap();
+                    debug!("{:?}: inserting state {}", EmailObject::NAME, &state);
+                    mailboxes_lck.entry(mailbox_hash).and_modify(|mbox| {
+                        *mbox.email_state.lock().unwrap() = Some(state);
+                    });
+                } else if !is_equal {
+                    conn.email_changes(mailbox_hash).await?;
+                }
+            }
+            debug!(&list);
+            for envobj in list {
+                let env_hash = id_map[&envobj.id];
+                conn.add_refresh_event(RefreshEvent {
+                    account_hash: store.account_hash,
+                    mailbox_hash,
+                    kind: RefreshEventKind::NewFlags(
+                        env_hash,
+                        protocol::keywords_to_flags(envobj.keywords().keys().cloned().collect()),
+                    ),
+                });
+            }
+            Ok(())
+        }))
+    }
+
+    fn delete_messages(
+        &mut self,
+        _env_hashes: EnvelopeHashBatch,
+        _mailbox_hash: MailboxHash,
+    ) -> ResultFuture<()> {
+        Err(MeliError::new("Unimplemented."))
+    }
+}
+
+impl JmapType {
+    pub fn new(
+        s: &AccountSettings,
+        is_subscribed: Box<dyn Fn(&str) -> bool + Send + Sync>,
+        event_consumer: BackendEventConsumer,
+    ) -> Result<Box<dyn MailBackend>> {
+        let online_status = Arc::new(FutureMutex::new((
+            std::time::Instant::now(),
+            Err(MeliError::new("Account is uninitialised.")),
+        )));
+        let server_conf = JmapServerConf::new(s)?;
+
+        let account_hash = {
+            let mut hasher = DefaultHasher::new();
+            hasher.write(s.name.as_bytes());
+            hasher.finish()
+        };
+        let store = Arc::new(Store {
+            account_name: Arc::new(s.name.clone()),
+            account_hash,
+            account_id: Arc::new(Mutex::new(Id::new())),
+            online_status,
+            event_consumer,
+            is_subscribed: Arc::new(IsSubscribedFn(is_subscribed)),
+            collection: Collection::default(),
+
+            byte_cache: Default::default(),
+            id_store: Default::default(),
+            reverse_id_store: Default::default(),
+            blob_id_store: Default::default(),
+            mailboxes: Default::default(),
+            mailboxes_index: Default::default(),
+            mailbox_state: Default::default(),
+        });
+
+        Ok(Box::new(JmapType {
+            connection: Arc::new(FutureMutex::new(JmapConnection::new(
+                &server_conf,
+                store.clone(),
+            )?)),
+            store,
+            server_conf,
+        }))
+    }
+
+    pub fn validate_config(s: &AccountSettings) -> Result<()> {
+        get_conf_val!(s["server_hostname"])?;
+        get_conf_val!(s["server_username"])?;
+        get_conf_val!(s["server_password"])?;
+        get_conf_val!(s["server_port"], 443)?;
+        get_conf_val!(s["danger_accept_invalid_certs"], false)?;
+        Ok(())
+    }
+}

melib/src/backends/jmap/connection.rs

@@ -0,0 +1,342 @@
+/*
+ * meli - jmap module.
+ *
+ * Copyright 2019 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+use isahc::config::Configurable;
+
+#[derive(Debug)]
+pub struct JmapConnection {
+    pub session: Arc<Mutex<JmapSession>>,
+    pub request_no: Arc<Mutex<usize>>,
+    pub client: Arc<HttpClient>,
+    pub server_conf: JmapServerConf,
+    pub store: Arc<Store>,
+}
+
+impl JmapConnection {
+    pub fn new(server_conf: &JmapServerConf, store: Arc<Store>) -> Result<Self> {
+        let client = HttpClient::builder()
+            .timeout(std::time::Duration::from_secs(10))
+            .redirect_policy(RedirectPolicy::Limit(10))
+            .authentication(isahc::auth::Authentication::basic())
+            .credentials(isahc::auth::Credentials::new(
+                &server_conf.server_username,
+                &server_conf.server_password,
+            ))
+            .build()?;
+        let server_conf = server_conf.clone();
+        Ok(JmapConnection {
+            session: Arc::new(Mutex::new(Default::default())),
+            request_no: Arc::new(Mutex::new(0)),
+            client: Arc::new(client),
+            server_conf,
+            store,
+        })
+    }
+
+    pub async fn connect(&mut self) -> Result<()> {
+        if self.store.online_status.lock().await.1.is_ok() {
+            return Ok(());
+        }
+        let mut jmap_session_resource_url =
+            if self.server_conf.server_hostname.starts_with("https://") {
+                self.server_conf.server_hostname.to_string()
+            } else {
+                format!("https://{}", &self.server_conf.server_hostname)
+            };
+        if self.server_conf.server_port != 443 {
+            jmap_session_resource_url.push(':');
+            jmap_session_resource_url.push_str(&self.server_conf.server_port.to_string());
+        }
+        jmap_session_resource_url.push_str("/.well-known/jmap");
+
+        let mut req = self.client.get_async(&jmap_session_resource_url).await?;
+        let res_text = req.text_async().await?;
+
+        let session: JmapSession = match serde_json::from_str(&res_text) {
+            Err(err) => {
+                let err = MeliError::new(format!("Could not connect to JMAP server endpoint for {}. Is your server hostname setting correct? (i.e. \"jmap.mailserver.org\") (Note: only session resource discovery via /.well-known/jmap is supported. DNS SRV records are not suppported.)\nReply from server: {}", &self.server_conf.server_hostname, &res_text)).set_source(Some(Arc::new(err)));
+                *self.store.online_status.lock().await = (Instant::now(), Err(err.clone()));
+                return Err(err);
+            }
+            Ok(s) => s,
+        };
+        if !session
+            .capabilities
+            .contains_key("urn:ietf:params:jmap:core")
+        {
+            let err = MeliError::new(format!("Server {} did not return JMAP Core capability (urn:ietf:params:jmap:core). Returned capabilities were: {}", &self.server_conf.server_hostname, session.capabilities.keys().map(String::as_str).collect::<Vec<&str>>().join(", ")));
+            *self.store.online_status.lock().await = (Instant::now(), Err(err.clone()));
+            return Err(err);
+        }
+        if !session
+            .capabilities
+            .contains_key("urn:ietf:params:jmap:mail")
+        {
+            let err = MeliError::new(format!("Server {} does not support JMAP Mail capability (urn:ietf:params:jmap:mail). Returned capabilities were: {}", &self.server_conf.server_hostname, session.capabilities.keys().map(String::as_str).collect::<Vec<&str>>().join(", ")));
+            *self.store.online_status.lock().await = (Instant::now(), Err(err.clone()));
+            return Err(err);
+        }
+
+        *self.store.online_status.lock().await = (Instant::now(), Ok(()));
+        *self.session.lock().unwrap() = session;
+        Ok(())
+    }
+
+    pub fn mail_account_id(&self) -> Id<Account> {
+        self.session.lock().unwrap().primary_accounts["urn:ietf:params:jmap:mail"].clone()
+    }
+
+    pub fn add_refresh_event(&self, event: RefreshEvent) {
+        (self.store.event_consumer)(self.store.account_hash, BackendEvent::Refresh(event));
+    }
+
+    pub async fn email_changes(&self, mailbox_hash: MailboxHash) -> Result<()> {
+        let mut current_state: State<EmailObject> = if let Some(s) = self
+            .store
+            .mailboxes
+            .read()
+            .unwrap()
+            .get(&mailbox_hash)
+            .and_then(|mbox| mbox.email_state.lock().unwrap().clone())
+        {
+            s
+        } else {
+            return Ok(());
+        };
+        loop {
+            let email_changes_call: EmailChanges = EmailChanges::new(
+                Changes::<EmailObject>::new()
+                    .account_id(self.mail_account_id().clone())
+                    .since_state(current_state.clone()),
+            );
+
+            let mut req = Request::new(self.request_no.clone());
+            let prev_seq = req.add_call(&email_changes_call);
+            let email_get_call: EmailGet = EmailGet::new(
+                Get::new()
+                    .ids(Some(JmapArgument::reference(
+                        prev_seq,
+                        ResultField::<EmailChanges, EmailObject>::new("created"),
+                    )))
+                    .account_id(self.mail_account_id().clone()),
+            );
+
+            req.add_call(&email_get_call);
+            if let Some(mailbox) = self.store.mailboxes.read().unwrap().get(&mailbox_hash) {
+                if let Some(email_query_state) = mailbox.email_query_state.lock().unwrap().clone() {
+                    let email_query_changes_call = EmailQueryChanges::new(
+                        QueryChanges::new(self.mail_account_id().clone(), email_query_state)
+                            .filter(Some(Filter::Condition(
+                                EmailFilterCondition::new()
+                                    .in_mailbox(Some(mailbox.id.clone()))
+                                    .into(),
+                            ))),
+                    );
+                    let seq_no = req.add_call(&email_query_changes_call);
+                    let email_get_call: EmailGet = EmailGet::new(
+                        Get::new()
+                            .ids(Some(JmapArgument::reference(
+                                seq_no,
+                                ResultField::<EmailQueryChanges, EmailObject>::new("removed"),
+                            )))
+                            .account_id(self.mail_account_id().clone())
+                            .properties(Some(vec![
+                                "keywords".to_string(),
+                                "mailboxIds".to_string(),
+                            ])),
+                    );
+                    req.add_call(&email_get_call);
+                } else {
+                    return Ok(());
+                }
+            } else {
+                return Ok(());
+            }
+            let api_url = self.session.lock().unwrap().api_url.clone();
+            let mut res = self
+                .client
+                .post_async(api_url.as_str(), serde_json::to_string(&req)?)
+                .await?;
+
+            let res_text = res.text_async().await?;
+            debug!(&res_text);
+            let mut v: MethodResponse = serde_json::from_str(&res_text).unwrap();
+            let changes_response =
+                ChangesResponse::<EmailObject>::try_from(v.method_responses.remove(0))?;
+            if changes_response.new_state == current_state {
+                return Ok(());
+            }
+            let get_response = GetResponse::<EmailObject>::try_from(v.method_responses.remove(0))?;
+
+            {
+                /* process get response */
+                let GetResponse::<EmailObject> { list, .. } = get_response;
+
+                let mut mailbox_hashes: Vec<SmallVec<[MailboxHash; 8]>> =
+                    Vec::with_capacity(list.len());
+                for envobj in &list {
+                    let v = self
+                        .store
+                        .mailboxes
+                        .read()
+                        .unwrap()
+                        .iter()
+                        .filter(|(_, m)| envobj.mailbox_ids.contains_key(&m.id))
+                        .map(|(k, _)| *k)
+                        .collect::<SmallVec<[MailboxHash; 8]>>();
+                    mailbox_hashes.push(v);
+                }
+                for (env, mailbox_hashes) in list
+                    .into_iter()
+                    .map(|obj| self.store.add_envelope(obj))
+                    .zip(mailbox_hashes)
+                {
+                    for mailbox_hash in mailbox_hashes.iter().skip(1).cloned() {
+                        let mut mailboxes_lck = self.store.mailboxes.write().unwrap();
+                        mailboxes_lck.entry(mailbox_hash).and_modify(|mbox| {
+                            if !env.is_seen() {
+                                mbox.unread_emails.lock().unwrap().insert_new(env.hash());
+                            }
+                            mbox.total_emails.lock().unwrap().insert_new(env.hash());
+                        });
+                        self.add_refresh_event(RefreshEvent {
+                            account_hash: self.store.account_hash,
+                            mailbox_hash,
+                            kind: RefreshEventKind::Create(Box::new(env.clone())),
+                        });
+                    }
+                    if let Some(mailbox_hash) = mailbox_hashes.first().cloned() {
+                        let mut mailboxes_lck = self.store.mailboxes.write().unwrap();
+                        mailboxes_lck.entry(mailbox_hash).and_modify(|mbox| {
+                            if !env.is_seen() {
+                                mbox.unread_emails.lock().unwrap().insert_new(env.hash());
+                            }
+                            mbox.total_emails.lock().unwrap().insert_new(env.hash());
+                        });
+                        self.add_refresh_event(RefreshEvent {
+                            account_hash: self.store.account_hash,
+                            mailbox_hash,
+                            kind: RefreshEventKind::Create(Box::new(env)),
+                        });
+                    }
+                }
+            }
+            let reverse_id_store_lck = self.store.reverse_id_store.lock().unwrap();
+            let response = v.method_responses.remove(0);
+            match EmailQueryChangesResponse::try_from(response) {
+                Ok(EmailQueryChangesResponse {
+                    collapse_threads: _,
+                    query_changes_response:
+                        QueryChangesResponse {
+                            account_id: _,
+                            old_query_state,
+                            new_query_state,
+                            total: _,
+                            removed,
+                            added,
+                        },
+                }) if old_query_state != new_query_state => {
+                    self.store
+                        .mailboxes
+                        .write()
+                        .unwrap()
+                        .entry(mailbox_hash)
+                        .and_modify(|mbox| {
+                            *mbox.email_query_state.lock().unwrap() = Some(new_query_state);
+                        });
+                    /*  If the "filter" or "sort" includes a mutable property, the server
+                    MUST include all Foos in the current results for which this
+                    property may have changed.  The position of these may have moved
+                    in the results, so they must be reinserted by the client to ensure
+                    its query cache is correct.  */
+                    for email_obj_id in removed
+                        .into_iter()
+                        .filter(|id| !added.iter().any(|item| item.id == *id))
+                    {
+                        if let Some(env_hash) = reverse_id_store_lck.get(&email_obj_id) {
+                            let mut mailboxes_lck = self.store.mailboxes.write().unwrap();
+                            mailboxes_lck.entry(mailbox_hash).and_modify(|mbox| {
+                                mbox.unread_emails.lock().unwrap().remove(*env_hash);
+                                mbox.total_emails.lock().unwrap().insert_new(*env_hash);
+                            });
+                            self.add_refresh_event(RefreshEvent {
+                                account_hash: self.store.account_hash,
+                                mailbox_hash,
+                                kind: RefreshEventKind::Remove(*env_hash),
+                            });
+                        }
+                    }
+                    for AddedItem {
+                        id: _email_obj_id,
+                        index: _,
+                    } in added
+                    {
+                        // FIXME
+                    }
+                }
+                Ok(_) => {}
+                Err(err) => {
+                    debug!(mailbox_hash);
+                    debug!(err);
+                }
+            }
+            let GetResponse::<EmailObject> { list, .. } =
+                GetResponse::<EmailObject>::try_from(v.method_responses.remove(0))?;
+            let mut mailboxes_lck = self.store.mailboxes.write().unwrap();
+            for envobj in list {
+                if let Some(env_hash) = reverse_id_store_lck.get(&envobj.id) {
+                    let new_flags =
+                        protocol::keywords_to_flags(envobj.keywords().keys().cloned().collect());
+                    mailboxes_lck.entry(mailbox_hash).and_modify(|mbox| {
+                        if new_flags.0.contains(Flag::SEEN) {
+                            mbox.unread_emails.lock().unwrap().remove(*env_hash);
+                        } else {
+                            mbox.unread_emails.lock().unwrap().insert_new(*env_hash);
+                        }
+                    });
+                    self.add_refresh_event(RefreshEvent {
+                        account_hash: self.store.account_hash,
+                        mailbox_hash,
+                        kind: RefreshEventKind::NewFlags(*env_hash, new_flags),
+                    });
+                }
+            }
+            drop(mailboxes_lck);
+            if changes_response.has_more_changes {
+                current_state = changes_response.new_state;
+            } else {
+                self.store
+                    .mailboxes
+                    .write()
+                    .unwrap()
+                    .entry(mailbox_hash)
+                    .and_modify(|mbox| {
+                        *mbox.email_state.lock().unwrap() = Some(changes_response.new_state);
+                    });
+
+                break;
+            }
+        }
+
+        Ok(())
+    }
+}

melib/src/backends/jmap/mailbox.rs

@@ -0,0 +1,118 @@
+/*
+ * meli - jmap module.
+ *
+ * Copyright 2019 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+use crate::backends::{LazyCountSet, MailboxPermissions, SpecialUsageMailbox};
+use std::sync::{Arc, Mutex, RwLock};
+
+#[derive(Debug, Clone)]
+pub struct JmapMailbox {
+    pub name: String,
+    pub path: String,
+    pub hash: MailboxHash,
+    pub children: Vec<MailboxHash>,
+    pub id: Id<MailboxObject>,
+    pub is_subscribed: bool,
+    pub my_rights: JmapRights,
+    pub parent_id: Option<Id<MailboxObject>>,
+    pub parent_hash: Option<MailboxHash>,
+    pub role: Option<String>,
+    pub sort_order: u64,
+    pub total_emails: Arc<Mutex<LazyCountSet>>,
+    pub total_threads: u64,
+    pub unread_emails: Arc<Mutex<LazyCountSet>>,
+    pub unread_threads: u64,
+    pub usage: Arc<RwLock<SpecialUsageMailbox>>,
+    pub email_state: Arc<Mutex<Option<State<EmailObject>>>>,
+    pub email_query_state: Arc<Mutex<Option<String>>>,
+}
+
+impl BackendMailbox for JmapMailbox {
+    fn hash(&self) -> MailboxHash {
+        self.hash
+    }
+
+    fn name(&self) -> &str {
+        &self.name
+    }
+
+    fn path(&self) -> &str {
+        &self.path
+    }
+
+    fn change_name(&mut self, _s: &str) {}
+
+    fn clone(&self) -> Mailbox {
+        Box::new(std::clone::Clone::clone(self))
+    }
+
+    fn children(&self) -> &[MailboxHash] {
+        &self.children
+    }
+
+    fn parent(&self) -> Option<MailboxHash> {
+        self.parent_hash
+    }
+
+    fn permissions(&self) -> MailboxPermissions {
+        MailboxPermissions::default()
+    }
+
+    fn special_usage(&self) -> SpecialUsageMailbox {
+        match self.role.as_ref().map(String::as_str) {
+            Some("inbox") => SpecialUsageMailbox::Inbox,
+            Some("archive") => SpecialUsageMailbox::Archive,
+            Some("junk") => SpecialUsageMailbox::Junk,
+            Some("trash") => SpecialUsageMailbox::Trash,
+            Some("drafts") => SpecialUsageMailbox::Drafts,
+            Some("sent") => SpecialUsageMailbox::Sent,
+            Some(other) => {
+                debug!(
+                    "unknown JMAP mailbox role for mailbox {}: {}",
+                    self.path(),
+                    other
+                );
+                SpecialUsageMailbox::Normal
+            }
+            None => SpecialUsageMailbox::Normal,
+        }
+    }
+    fn is_subscribed(&self) -> bool {
+        self.is_subscribed
+    }
+    fn set_is_subscribed(&mut self, new_val: bool) -> Result<()> {
+        self.is_subscribed = new_val;
+        // FIXME: jmap subscribe
+        Ok(())
+    }
+
+    fn set_special_usage(&mut self, new_val: SpecialUsageMailbox) -> Result<()> {
+        *self.usage.write()? = new_val;
+        Ok(())
+    }
+
+    fn count(&self) -> Result<(usize, usize)> {
+        Ok((
+            self.unread_emails.lock()?.len(),
+            self.total_emails.lock()?.len(),
+        ))
+    }
+}

melib/src/backends/jmap/objects.rs

@@ -1,5 +1,5 @@
 /*
- * meli - conf module
+ * meli - jmap module.
  *
  * Copyright 2019 Manos Pitsidianakis
  *
@@ -19,12 +19,10 @@
  * along with meli. If not, see <http://www.gnu.org/licenses/>.
  */
 
-/// Settings for writing and sending new e-mail
-#[derive(Debug, Serialize, Deserialize, Clone, Default)]
-pub struct ComposingSettings {
-    /// A command to pipe new emails to
-    /// Required
-    pub mailer_cmd: String,
-    /// Command to launch editor. Can have arguments. Draft filename is given as the last argument. If it's missing, the environment variable $EDITOR is looked up.
-    pub editor_cmd: Option<String>,
-}
+use super::*;
+
+mod email;
+pub use email::*;
+
+mod mailbox;
+pub use mailbox::*;

melib/src/backends/jmap/objects/email.rs

@@ -0,0 +1,849 @@
+/*
+ * meli - jmap module.
+ *
+ * Copyright 2019 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+use crate::backends::jmap::rfc8620::bool_false;
+use crate::email::address::{Address, MailboxAddress};
+use core::marker::PhantomData;
+use serde::de::{Deserialize, Deserializer};
+use serde_json::value::RawValue;
+use serde_json::Value;
+use std::collections::hash_map::DefaultHasher;
+use std::collections::HashMap;
+use std::hash::Hasher;
+
+mod import;
+pub use import::*;
+
+#[derive(Debug)]
+pub struct ThreadObject;
+
+impl Object for ThreadObject {
+    const NAME: &'static str = "Thread";
+}
+
+impl Id<EmailObject> {
+    pub fn into_hash(&self) -> EnvelopeHash {
+        let mut h = DefaultHasher::new();
+        h.write(self.inner.as_bytes());
+        h.finish()
+    }
+}
+
+// 4.1.1.
+// Metadata
+// These properties represent metadata about the message in the mail
+// store and are not derived from parsing the message itself.
+//
+//   o  id: "Id" (immutable; server-set)
+//
+//      The id of the Email object.  Note that this is the JMAP object id,
+//      NOT the Message-ID header field value of the message [RFC5322].
+//
+//   o  blobId: "Id" (immutable; server-set)
+//
+//      The id representing the raw octets of the message [RFC5322] for
+//      this Email.  This may be used to download the raw original message
+//      or to attach it directly to another Email, etc.
+//
+//   o  threadId: "Id" (immutable; server-set)
+//
+//      The id of the Thread to which this Email belongs.
+//
+//   o  mailboxIds: "Id[Boolean]"
+//
+//      The set of Mailbox ids this Email belongs to.  An Email in the
+//      mail store MUST belong to one or more Mailboxes at all times
+//      (until it is destroyed).  The set is represented as an object,
+//      with each key being a Mailbox id.  The value for each key in the
+//      object MUST be true.
+//
+//   o  keywords: "String[Boolean]" (default: {})
+//
+//      A set of keywords that apply to the Email.  The set is represented
+//      as an object, with the keys being the keywords.  The value for
+//      each key in the object MUST be true.
+//
+//      Keywords are shared with IMAP.  The six system keywords from IMAP
+//      get special treatment.  The following four keywords have their
+//      first character changed from "\" in IMAP to "$" in JMAP and have
+//      particular semantic meaning:
+//
+//      *  "$draft": The Email is a draft the user is composing.
+//
+//      *  "$seen": The Email has been read.
+//
+//      *  "$flagged": The Email has been flagged for urgent/special
+//         attention.
+//
+//      *  "$answered": The Email has been replied to.
+//
+//      The IMAP "\Recent" keyword is not exposed via JMAP.  The IMAP
+//      "\Deleted" keyword is also not present: IMAP uses a delete+expunge
+//      model, which JMAP does not.  Any message with the "\Deleted"
+//      keyword MUST NOT be visible via JMAP (and so are not counted in
+//      the "totalEmails", "unreadEmails", "totalThreads", and
+//      "unreadThreads" Mailbox properties).
+//
+//      Users may add arbitrary keywords to an Email.  For compatibility
+//      with IMAP, a keyword is a case-insensitive string of 1-255
+//      characters in the ASCII subset %x21-%x7e (excludes control chars
+//      and space), and it MUST NOT include any of these characters:
+//
+//                              ( ) { ] % * " \
+//
+//      Because JSON is case sensitive, servers MUST return keywords in
+//      lowercase.
+//
+//      The IANA "IMAP and JMAP Keywords" registry at
+//      <https://www.iana.org/assignments/imap-jmap-keywords/> as
+//      established in [RFC5788] assigns semantic meaning to some other
+//      keywords in common use.  New keywords may be established here in
+//      the future.  In particular, note:
+//
+//      *  "$forwarded": The Email has been forwarded.
+//
+//      *  "$phishing": The Email is highly likely to be phishing.
+//         Clients SHOULD warn users to take care when viewing this Email
+//         and disable links and attachments.
+//
+//      *  "$junk": The Email is definitely spam.  Clients SHOULD set this
+//         flag when users report spam to help train automated spam-
+//         detection systems.
+//
+//      *  "$notjunk": The Email is definitely not spam.  Clients SHOULD
+//         set this flag when users indicate an Email is legitimate, to
+//         help train automated spam-detection systems.
+//
+//   o  size: "UnsignedInt" (immutable; server-set)
+//
+//      The size, in octets, of the raw data for the message [RFC5322] (as
+//      referenced by the "blobId", i.e., the number of octets in the file
+//      the user would download).
+//
+//   o  receivedAt: "UTCDate" (immutable; default: time of creation on
+//      server)
+//
+//      The date the Email was received by the message store.  This is the
+//      "internal date" in IMAP [RFC3501]./
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct EmailObject {
+    #[serde(default)]
+    pub id: Id<EmailObject>,
+    #[serde(default)]
+    pub blob_id: Id<BlobObject>,
+    #[serde(default)]
+    pub mailbox_ids: HashMap<Id<MailboxObject>, bool>,
+    #[serde(default)]
+    pub size: u64,
+    #[serde(default)]
+    pub received_at: String,
+    #[serde(default)]
+    pub message_id: Vec<String>,
+    #[serde(default)]
+    pub to: Option<SmallVec<[EmailAddress; 1]>>,
+    #[serde(default)]
+    pub bcc: Option<Vec<EmailAddress>>,
+    #[serde(default)]
+    pub reply_to: Option<Vec<EmailAddress>>,
+    #[serde(default)]
+    pub cc: Option<SmallVec<[EmailAddress; 1]>>,
+    #[serde(default)]
+    pub sender: Option<Vec<EmailAddress>>,
+    #[serde(default)]
+    pub from: Option<SmallVec<[EmailAddress; 1]>>,
+    #[serde(default)]
+    pub in_reply_to: Option<Vec<String>>,
+    #[serde(default)]
+    pub references: Option<Vec<String>>,
+    #[serde(default)]
+    pub keywords: HashMap<String, bool>,
+    #[serde(default)]
+    pub attached_emails: Option<Id<BlobObject>>,
+    #[serde(default)]
+    pub attachments: Vec<Value>,
+    #[serde(default)]
+    pub has_attachment: bool,
+    #[serde(default)]
+    #[serde(deserialize_with = "deserialize_header")]
+    pub headers: HashMap<String, String>,
+    #[serde(default)]
+    pub html_body: Vec<HtmlBody>,
+    #[serde(default)]
+    pub preview: Option<String>,
+    #[serde(default)]
+    pub sent_at: Option<String>,
+    #[serde(default)]
+    pub subject: Option<String>,
+    #[serde(default)]
+    pub text_body: Vec<TextBody>,
+    #[serde(default)]
+    pub thread_id: Id<ThreadObject>,
+    #[serde(flatten)]
+    pub extra: HashMap<String, Value>,
+}
+
+impl EmailObject {
+    _impl!(get keywords, keywords: HashMap<String, bool>);
+}
+
+#[derive(Deserialize, Serialize, Debug, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct Header {
+    pub name: String,
+    pub value: String,
+}
+
+fn deserialize_header<'de, D>(
+    deserializer: D,
+) -> std::result::Result<HashMap<String, String>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let v = <Vec<Header>>::deserialize(deserializer)?;
+    Ok(v.into_iter().map(|t| (t.name, t.value)).collect())
+}
+
+#[derive(Deserialize, Serialize, Debug, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct EmailAddress {
+    pub email: String,
+    pub name: Option<String>,
+}
+
+impl Into<crate::email::Address> for EmailAddress {
+    fn into(self) -> crate::email::Address {
+        let Self { email, mut name } = self;
+        crate::make_address!((name.take().unwrap_or_default()), email)
+    }
+}
+
+impl std::fmt::Display for EmailAddress {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        if self.name.is_some() {
+            write!(f, "{} <{}>", self.name.as_ref().unwrap(), &self.email)
+        } else {
+            write!(f, "{}", &self.email)
+        }
+    }
+}
+
+impl std::convert::From<EmailObject> for crate::Envelope {
+    fn from(mut t: EmailObject) -> crate::Envelope {
+        let mut env = crate::Envelope::new(0);
+        if let Ok(d) = crate::email::parser::dates::rfc5322_date(env.date_as_str().as_bytes()) {
+            env.set_datetime(d);
+        }
+        if let Some(ref mut sent_at) = t.sent_at {
+            let unix =
+                crate::datetime::rfc3339_to_timestamp(sent_at.as_bytes().to_vec()).unwrap_or(0);
+            env.set_datetime(unix);
+            env.set_date(std::mem::replace(sent_at, String::new()).as_bytes());
+        }
+
+        if let Some(v) = t.message_id.get(0) {
+            env.set_message_id(v.as_bytes());
+        }
+        if let Some(ref in_reply_to) = t.in_reply_to {
+            env.set_in_reply_to(in_reply_to[0].as_bytes());
+            if let Some(in_reply_to) = env.in_reply_to().cloned() {
+                env.push_references(in_reply_to);
+            }
+        }
+        if let Some(v) = t.headers.get("References") {
+            env.set_references(v.as_bytes());
+        }
+        if let Some(v) = t.headers.get("Date") {
+            env.set_date(v.as_bytes());
+            if let Ok(d) = crate::email::parser::dates::rfc5322_date(v.as_bytes()) {
+                env.set_datetime(d);
+            }
+        } else if let Ok(d) = crate::email::parser::dates::rfc5322_date(t.received_at.as_bytes()) {
+            env.set_datetime(d);
+        }
+        env.set_has_attachments(t.has_attachment);
+        if let Some(ref mut subject) = t.subject {
+            env.set_subject(std::mem::replace(subject, String::new()).into_bytes());
+        }
+
+        if let Some(ref mut from) = t.from {
+            env.set_from(
+                std::mem::replace(from, SmallVec::new())
+                    .into_iter()
+                    .map(|addr| addr.into())
+                    .collect::<SmallVec<[crate::email::Address; 1]>>(),
+            );
+        }
+        if let Some(ref mut to) = t.to {
+            env.set_to(
+                std::mem::replace(to, SmallVec::new())
+                    .into_iter()
+                    .map(|addr| addr.into())
+                    .collect::<SmallVec<[crate::email::Address; 1]>>(),
+            );
+        }
+
+        if let Some(ref mut cc) = t.cc {
+            env.set_cc(
+                std::mem::replace(cc, SmallVec::new())
+                    .into_iter()
+                    .map(|addr| addr.into())
+                    .collect::<SmallVec<[crate::email::Address; 1]>>(),
+            );
+        }
+
+        if let Some(ref mut bcc) = t.bcc {
+            env.set_bcc(
+                std::mem::replace(bcc, Vec::new())
+                    .into_iter()
+                    .map(|addr| addr.into())
+                    .collect::<Vec<crate::email::Address>>(),
+            );
+        }
+
+        if let Some(ref r) = env.references {
+            if let Some(pos) = r.refs.iter().position(|r| r == env.message_id()) {
+                env.references.as_mut().unwrap().refs.remove(pos);
+            }
+        }
+
+        env.set_hash(t.id.into_hash());
+        env
+    }
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct HtmlBody {
+    pub blob_id: Id<BlobObject>,
+    #[serde(default)]
+    pub charset: String,
+    #[serde(default)]
+    pub cid: Option<String>,
+    #[serde(default)]
+    pub disposition: Option<String>,
+    #[serde(default)]
+    pub headers: Value,
+    #[serde(default)]
+    pub language: Option<Vec<String>>,
+    #[serde(default)]
+    pub location: Option<String>,
+    #[serde(default)]
+    pub name: Option<String>,
+    #[serde(default)]
+    pub part_id: Option<String>,
+    pub size: u64,
+    #[serde(alias = "type")]
+    pub content_type: String,
+    #[serde(default)]
+    pub sub_parts: Vec<Value>,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct TextBody {
+    pub blob_id: Id<BlobObject>,
+    #[serde(default)]
+    pub charset: String,
+    #[serde(default)]
+    pub cid: Option<String>,
+    #[serde(default)]
+    pub disposition: Option<String>,
+    #[serde(default)]
+    pub headers: Value,
+    #[serde(default)]
+    pub language: Option<Vec<String>>,
+    #[serde(default)]
+    pub location: Option<String>,
+    #[serde(default)]
+    pub name: Option<String>,
+    #[serde(default)]
+    pub part_id: Option<String>,
+    pub size: u64,
+    #[serde(alias = "type")]
+    pub content_type: String,
+    #[serde(default)]
+    pub sub_parts: Vec<Value>,
+}
+
+impl Object for EmailObject {
+    const NAME: &'static str = "Email";
+}
+
+#[derive(Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct EmailQuery {
+    #[serde(flatten)]
+    pub query_call: Query<Filter<EmailFilterCondition, EmailObject>, EmailObject>,
+    //pub filter: EmailFilterCondition, /* "inMailboxes": [ mailbox.id ] },*/
+    pub collapse_threads: bool,
+}
+
+impl Method<EmailObject> for EmailQuery {
+    const NAME: &'static str = "Email/query";
+}
+
+impl EmailQuery {
+    pub const RESULT_FIELD_IDS: ResultField<EmailQuery, EmailObject> =
+        ResultField::<EmailQuery, EmailObject> {
+            field: "/ids",
+            _ph: PhantomData,
+        };
+
+    pub fn new(query_call: Query<Filter<EmailFilterCondition, EmailObject>, EmailObject>) -> Self {
+        EmailQuery {
+            query_call,
+            collapse_threads: false,
+        }
+    }
+
+    _impl!(collapse_threads: bool);
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct EmailGet {
+    #[serde(flatten)]
+    pub get_call: Get<EmailObject>,
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub body_properties: Vec<String>,
+    #[serde(default = "bool_false")]
+    pub fetch_text_body_values: bool,
+    #[serde(default = "bool_false")]
+    #[serde(rename = "fetchHTMLBodyValues")]
+    pub fetch_html_body_values: bool,
+    #[serde(default = "bool_false")]
+    pub fetch_all_body_values: bool,
+    #[serde(default)]
+    #[serde(skip_serializing_if = "u64_zero")]
+    pub max_body_value_bytes: u64,
+}
+
+impl Method<EmailObject> for EmailGet {
+    const NAME: &'static str = "Email/get";
+}
+
+impl EmailGet {
+    pub fn new(get_call: Get<EmailObject>) -> Self {
+        EmailGet {
+            get_call,
+            body_properties: Vec::new(),
+            fetch_text_body_values: false,
+            fetch_html_body_values: false,
+            fetch_all_body_values: false,
+            max_body_value_bytes: 0,
+        }
+    }
+
+    _impl!(body_properties: Vec<String>);
+    _impl!(fetch_text_body_values: bool);
+    _impl!(fetch_html_body_values: bool);
+    _impl!(fetch_all_body_values: bool);
+    _impl!(max_body_value_bytes: u64);
+}
+
+#[derive(Serialize, Deserialize, Default, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct EmailFilterCondition {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub in_mailbox: Option<Id<MailboxObject>>,
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub in_mailbox_other_than: Vec<Id<MailboxObject>>,
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub before: UtcDate,
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub after: UtcDate,
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub min_size: Option<u64>,
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub max_size: Option<u64>,
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub all_in_thread_have_keyword: String,
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub some_in_thread_have_keyword: String,
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub none_in_thread_have_keyword: String,
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub has_keyword: String,
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub not_keyword: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub has_attachment: Option<bool>,
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub text: String,
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub from: String,
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub to: String,
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub cc: String,
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub bcc: String,
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub subject: String,
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub body: String,
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub header: Vec<Value>,
+}
+
+impl EmailFilterCondition {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    _impl!(in_mailbox: Option<Id<MailboxObject>>);
+    _impl!(in_mailbox_other_than: Vec<Id<MailboxObject>>);
+    _impl!(before: UtcDate);
+    _impl!(after: UtcDate);
+    _impl!(min_size: Option<u64>);
+    _impl!(max_size: Option<u64>);
+    _impl!(all_in_thread_have_keyword: String);
+    _impl!(some_in_thread_have_keyword: String);
+    _impl!(none_in_thread_have_keyword: String);
+    _impl!(has_keyword: String);
+    _impl!(not_keyword: String);
+    _impl!(has_attachment: Option<bool>);
+    _impl!(text: String);
+    _impl!(from: String);
+    _impl!(to: String);
+    _impl!(cc: String);
+    _impl!(bcc: String);
+    _impl!(subject: String);
+    _impl!(body: String);
+    _impl!(header: Vec<Value>);
+}
+
+impl FilterTrait<EmailObject> for EmailFilterCondition {}
+
+impl From<EmailFilterCondition> for FilterCondition<EmailFilterCondition, EmailObject> {
+    fn from(val: EmailFilterCondition) -> FilterCondition<EmailFilterCondition, EmailObject> {
+        FilterCondition {
+            cond: val,
+            _ph: PhantomData,
+        }
+    }
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub enum MessageProperty {
+    ThreadId,
+    MailboxIds,
+    Keywords,
+    Size,
+    ReceivedAt,
+    IsUnread,
+    IsFlagged,
+    IsAnswered,
+    IsDraft,
+    HasAttachment,
+    From,
+    To,
+    Cc,
+    Bcc,
+    ReplyTo,
+    Subject,
+    SentAt,
+    Preview,
+    Id,
+    BlobId,
+    MessageId,
+    InReplyTo,
+    Sender,
+}
+
+impl From<crate::search::Query> for Filter<EmailFilterCondition, EmailObject> {
+    fn from(val: crate::search::Query) -> Self {
+        let mut ret = Filter::Condition(EmailFilterCondition::new().into());
+        fn rec(q: &crate::search::Query, f: &mut Filter<EmailFilterCondition, EmailObject>) {
+            use crate::datetime::{timestamp_to_string, RFC3339_FMT};
+            use crate::search::Query::*;
+            match q {
+                Subject(t) => {
+                    *f = Filter::Condition(EmailFilterCondition::new().subject(t.clone()).into());
+                }
+                From(t) => {
+                    *f = Filter::Condition(EmailFilterCondition::new().from(t.clone()).into());
+                }
+                To(t) => {
+                    *f = Filter::Condition(EmailFilterCondition::new().to(t.clone()).into());
+                }
+                Cc(t) => {
+                    *f = Filter::Condition(EmailFilterCondition::new().cc(t.clone()).into());
+                }
+                Bcc(t) => {
+                    *f = Filter::Condition(EmailFilterCondition::new().bcc(t.clone()).into());
+                }
+                AllText(t) => {
+                    *f = Filter::Condition(EmailFilterCondition::new().text(t.clone()).into());
+                }
+                Body(t) => {
+                    *f = Filter::Condition(EmailFilterCondition::new().body(t.clone()).into());
+                }
+                Before(t) => {
+                    *f = Filter::Condition(
+                        EmailFilterCondition::new()
+                            .before(timestamp_to_string(*t, Some(RFC3339_FMT), true))
+                            .into(),
+                    );
+                }
+                After(t) => {
+                    *f = Filter::Condition(
+                        EmailFilterCondition::new()
+                            .after(timestamp_to_string(*t, Some(RFC3339_FMT), true))
+                            .into(),
+                    );
+                }
+                Between(a, b) => {
+                    *f = Filter::Condition(
+                        EmailFilterCondition::new()
+                            .after(timestamp_to_string(*a, Some(RFC3339_FMT), true))
+                            .into(),
+                    );
+                    *f &= Filter::Condition(
+                        EmailFilterCondition::new()
+                            .before(timestamp_to_string(*b, Some(RFC3339_FMT), true))
+                            .into(),
+                    );
+                }
+                On(t) => {
+                    rec(&Between(*t, *t), f);
+                }
+                InReplyTo(ref s) => {
+                    *f = Filter::Condition(
+                        EmailFilterCondition::new()
+                            .header(vec!["In-Reply-To".to_string().into(), s.to_string().into()])
+                            .into(),
+                    );
+                }
+                References(ref s) => {
+                    *f = Filter::Condition(
+                        EmailFilterCondition::new()
+                            .header(vec!["References".to_string().into(), s.to_string().into()])
+                            .into(),
+                    );
+                }
+                AllAddresses(_) => {
+                    //TODO
+                }
+                Flags(v) => {
+                    fn flag_to_filter(f: &str) -> Filter<EmailFilterCondition, EmailObject> {
+                        match f {
+                            "draft" => Filter::Condition(
+                                EmailFilterCondition::new()
+                                    .has_keyword("$draft".to_string())
+                                    .into(),
+                            ),
+                            "flagged" => Filter::Condition(
+                                EmailFilterCondition::new()
+                                    .has_keyword("$flagged".to_string())
+                                    .into(),
+                            ),
+                            "seen" | "read" => Filter::Condition(
+                                EmailFilterCondition::new()
+                                    .has_keyword("$seen".to_string())
+                                    .into(),
+                            ),
+                            "unseen" | "unread" => Filter::Condition(
+                                EmailFilterCondition::new()
+                                    .not_keyword("$seen".to_string())
+                                    .into(),
+                            ),
+                            "answered" => Filter::Condition(
+                                EmailFilterCondition::new()
+                                    .has_keyword("$answered".to_string())
+                                    .into(),
+                            ),
+                            "unanswered" => Filter::Condition(
+                                EmailFilterCondition::new()
+                                    .not_keyword("$answered".to_string())
+                                    .into(),
+                            ),
+                            keyword => Filter::Condition(
+                                EmailFilterCondition::new()
+                                    .not_keyword(keyword.to_string())
+                                    .into(),
+                            ),
+                        }
+                    }
+                    let mut accum = if let Some(first) = v.first() {
+                        flag_to_filter(first.as_str())
+                    } else {
+                        Filter::Condition(EmailFilterCondition::new().into())
+                    };
+                    for f in v.iter().skip(1) {
+                        accum &= flag_to_filter(f.as_str());
+                    }
+                    *f = accum;
+                }
+                HasAttachment => {
+                    *f = Filter::Condition(
+                        EmailFilterCondition::new()
+                            .has_attachment(Some(true))
+                            .into(),
+                    );
+                }
+                And(q1, q2) => {
+                    let mut rhs = Filter::Condition(EmailFilterCondition::new().into());
+                    let mut lhs = Filter::Condition(EmailFilterCondition::new().into());
+                    rec(q1, &mut rhs);
+                    rec(q2, &mut lhs);
+                    rhs &= lhs;
+                    *f = rhs;
+                }
+                Or(q1, q2) => {
+                    let mut rhs = Filter::Condition(EmailFilterCondition::new().into());
+                    let mut lhs = Filter::Condition(EmailFilterCondition::new().into());
+                    rec(q1, &mut rhs);
+                    rec(q2, &mut lhs);
+                    rhs |= lhs;
+                    *f = rhs;
+                }
+                Not(q) => {
+                    let mut qhs = Filter::Condition(EmailFilterCondition::new().into());
+                    rec(q, &mut qhs);
+                    *f = !qhs;
+                }
+            }
+        }
+        rec(&val, &mut ret);
+        ret
+    }
+}
+
+#[test]
+fn test_jmap_query() {
+    use std::sync::{Arc, Mutex};
+    let q: crate::search::Query = crate::search::Query::try_from(
+        "subject:wah or (from:Manos and (subject:foo or subject:bar))",
+    )
+    .unwrap();
+    let f: Filter<EmailFilterCondition, EmailObject> = Filter::from(q);
+    assert_eq!(
+        r#"{"operator":"OR","conditions":[{"subject":"wah"},{"operator":"AND","conditions":[{"from":"Manos"},{"operator":"OR","conditions":[{"subject":"foo"},{"subject":"bar"}]}]}]}"#,
+        serde_json::to_string(&f).unwrap().as_str()
+    );
+    let filter = {
+        let mailbox_id = "mailbox_id".to_string();
+
+        let mut r = Filter::Condition(
+            EmailFilterCondition::new()
+                .in_mailbox(Some(mailbox_id.into()))
+                .into(),
+        );
+        r &= f;
+        r
+    };
+
+    let email_call: EmailQuery = EmailQuery::new(
+        Query::new()
+            .account_id("account_id".to_string().into())
+            .filter(Some(filter))
+            .position(0),
+    )
+    .collapse_threads(false);
+
+    let request_no = Arc::new(Mutex::new(0));
+    let mut req = Request::new(request_no.clone());
+    req.add_call(&email_call);
+
+    assert_eq!(
+        r#"{"using":["urn:ietf:params:jmap:core","urn:ietf:params:jmap:mail"],"methodCalls":[["Email/query",{"accountId":"account_id","calculateTotal":false,"collapseThreads":false,"filter":{"conditions":[{"inMailbox":"mailbox_id"},{"conditions":[{"subject":"wah"},{"conditions":[{"from":"Manos"},{"conditions":[{"subject":"foo"},{"subject":"bar"}],"operator":"OR"}],"operator":"AND"}],"operator":"OR"}],"operator":"AND"},"position":0,"sort":null},"m0"]]}"#,
+        serde_json::to_string(&req).unwrap().as_str()
+    );
+    assert_eq!(*request_no.lock().unwrap(), 1);
+}
+
+#[derive(Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct EmailSet {
+    #[serde(flatten)]
+    pub set_call: Set<EmailObject>,
+}
+
+impl Method<EmailObject> for EmailSet {
+    const NAME: &'static str = "Email/set";
+}
+
+impl EmailSet {
+    pub fn new(set_call: Set<EmailObject>) -> Self {
+        EmailSet { set_call }
+    }
+}
+
+#[derive(Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct EmailChanges {
+    #[serde(flatten)]
+    pub changes_call: Changes<EmailObject>,
+}
+
+impl Method<EmailObject> for EmailChanges {
+    const NAME: &'static str = "Email/changes";
+}
+
+impl EmailChanges {
+    pub fn new(changes_call: Changes<EmailObject>) -> Self {
+        EmailChanges { changes_call }
+    }
+}
+
+#[derive(Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct EmailQueryChanges {
+    #[serde(flatten)]
+    pub query_changes_call: QueryChanges<Filter<EmailFilterCondition, EmailObject>, EmailObject>,
+}
+
+impl Method<EmailObject> for EmailQueryChanges {
+    const NAME: &'static str = "Email/queryChanges";
+}
+
+impl EmailQueryChanges {
+    pub fn new(
+        query_changes_call: QueryChanges<Filter<EmailFilterCondition, EmailObject>, EmailObject>,
+    ) -> Self {
+        EmailQueryChanges { query_changes_call }
+    }
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct EmailQueryChangesResponse {
+    ///o  The "collapseThreads" argument that was used with "Email/query".
+    #[serde(default = "bool_false")]
+    pub collapse_threads: bool,
+    #[serde(flatten)]
+    pub query_changes_response: QueryChangesResponse<EmailObject>,
+}
+
+impl std::convert::TryFrom<&RawValue> for EmailQueryChangesResponse {
+    type Error = crate::error::MeliError;
+    fn try_from(t: &RawValue) -> Result<EmailQueryChangesResponse> {
+        let res: (String, EmailQueryChangesResponse, String) = serde_json::from_str(t.get())?;
+        assert_eq!(&res.0, "Email/queryChanges");
+        Ok(res.1)
+    }
+}

melib/src/backends/jmap/objects/email/import.rs

@@ -0,0 +1,200 @@
+/*
+ * meli -
+ *
+ * Copyright  Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+use serde_json::value::RawValue;
+
+/// #`import`
+///
+///    Objects of type `Foo` are imported via a call to `Foo/import`.
+///
+///    It takes the following arguments:
+///
+///    - `account_id`: "Id"
+///
+///       The id of the account to use.
+///
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct ImportCall {
+    ///accountId: "Id"
+    ///The id of the account to use.
+    pub account_id: Id<Account>,
+    ///ifInState: "String|null"
+    ///This is a state string as returned by the "Email/get" method.  If
+    ///supplied, the string must match the current state of the account
+    ///referenced by the accountId; otherwise, the method will be aborted
+    ///and a "stateMismatch" error returned.  If null, any changes will
+    ///be applied to the current state.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub if_in_state: Option<State<EmailObject>>,
+    ///o  emails: "Id[EmailImport]"
+    ///A map of creation id (client specified) to EmailImport objects.
+    pub emails: HashMap<Id<EmailObject>, EmailImport>,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct EmailImport {
+    ///o  blobId: "Id"
+    ///The id of the blob containing the raw message [RFC5322].
+    pub blob_id: Id<BlobObject>,
+    ///o  mailboxIds: "Id[Boolean]"
+    ///The ids of the Mailboxes to assign this Email to.  At least one
+    ///Mailbox MUST be given.
+    pub mailbox_ids: HashMap<Id<MailboxObject>, bool>,
+    ///o  keywords: "String[Boolean]" (default: {})
+    ///The keywords to apply to the Email.
+    pub keywords: HashMap<String, bool>,
+
+    ///o  receivedAt: "UTCDate" (default: time of most recent Received
+    ///header, or time of import on server if none)
+    ///The "receivedAt" date to set on the Email.
+    pub received_at: Option<String>,
+}
+
+impl ImportCall {
+    pub fn new() -> Self {
+        Self {
+            account_id: Id::new(),
+            if_in_state: None,
+            emails: HashMap::default(),
+        }
+    }
+
+    _impl!(
+        ///   -  accountId: "Id"
+        ///
+        ///      The id of the account to use.
+        ///
+        account_id: Id<Account>
+    );
+    _impl!(if_in_state: Option<State<EmailObject>>);
+    _impl!(emails: HashMap<Id<EmailObject>, EmailImport>);
+}
+
+impl Method<EmailObject> for ImportCall {
+    const NAME: &'static str = "Email/import";
+}
+
+impl EmailImport {
+    pub fn new() -> Self {
+        Self {
+            blob_id: Id::new(),
+            mailbox_ids: HashMap::default(),
+            keywords: HashMap::default(),
+            received_at: None,
+        }
+    }
+
+    _impl!(blob_id: Id<BlobObject>);
+    _impl!(mailbox_ids: HashMap<Id<MailboxObject>, bool>);
+    _impl!(keywords: HashMap<String, bool>);
+    _impl!(received_at: Option<String>);
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+#[serde(tag = "type")]
+pub enum ImportError {
+    ///The server MAY forbid two Email objects with the same exact content
+    ///   [RFC5322], or even just with the same Message-ID [RFC5322], to
+    ///   coexist within an account.  In this case, it MUST reject attempts to
+    ///   import an Email considered to be a duplicate with an "alreadyExists"
+    ///   SetError.
+    AlreadyExists {
+        description: Option<String>,
+        /// An "existingId" property of type "Id" MUST be included on
+        ///the SetError object with the id of the existing Email.  If duplicates
+        ///are allowed, the newly created Email object MUST have a separate id
+        ///and independent mutable properties to the existing object.
+        existing_id: Id<EmailObject>,
+    },
+    ///If the "blobId", "mailboxIds", or "keywords" properties are invalid
+    ///(e.g., missing, wrong type, id not found), the server MUST reject the
+    ///import with an "invalidProperties" SetError.
+    InvalidProperties {
+        description: Option<String>,
+        properties: Vec<String>,
+    },
+    ///If the Email cannot be imported because it would take the account
+    ///over quota, the import should be rejected with an "overQuota"
+    ///SetError.
+    OverQuota { description: Option<String> },
+    ///If the blob referenced is not a valid message [RFC5322], the server
+    ///MAY modify the message to fix errors (such as removing NUL octets or
+    ///fixing invalid headers).  If it does this, the "blobId" on the
+    ///response MUST represent the new representation and therefore be
+    ///different to the "blobId" on the EmailImport object.  Alternatively,
+    ///the server MAY reject the import with an "invalidEmail" SetError.
+    InvalidEmail { description: Option<String> },
+    ///An "ifInState" argument was supplied, and it does not match the current state.
+    StateMismatch,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct ImportResponse {
+    ///o  accountId: "Id"
+    ///The id of the account used for this call.
+    pub account_id: Id<Account>,
+
+    ///o  oldState: "String|null"
+    ///The state string that would have been returned by "Email/get" on
+    ///this account before making the requested changes, or null if the
+    ///server doesn't know what the previous state string was.
+    pub old_state: Option<State<EmailObject>>,
+
+    ///o  newState: "String"
+    ///The state string that will now be returned by "Email/get" on this
+    ///account.
+    pub new_state: Option<State<EmailObject>>,
+
+    ///o  created: "Id[Email]|null"
+    ///A map of the creation id to an object containing the "id",
+    ///"blobId", "threadId", and "size" properties for each successfully
+    ///imported Email, or null if none.
+    pub created: HashMap<Id<EmailObject>, ImportEmailResult>,
+
+    ///o  notCreated: "Id[SetError]|null"
+    ///A map of the creation id to a SetError object for each Email that
+    ///failed to be created, or null if all successful.  The possible
+    ///errors are defined above.
+    pub not_created: HashMap<Id<EmailObject>, ImportError>,
+}
+
+impl std::convert::TryFrom<&RawValue> for ImportResponse {
+    type Error = crate::error::MeliError;
+    fn try_from(t: &RawValue) -> Result<ImportResponse> {
+        let res: (String, ImportResponse, String) = serde_json::from_str(t.get())?;
+        assert_eq!(&res.0, &ImportCall::NAME);
+        Ok(res.1)
+    }
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct ImportEmailResult {
+    pub id: Id<EmailObject>,
+    pub blob_id: Id<BlobObject>,
+    pub thread_id: Id<ThreadObject>,
+    pub size: usize,
+}

melib/src/backends/jmap/objects/mailbox.rs

@@ -0,0 +1,79 @@
+/*
+ * meli - jmap module.
+ *
+ * Copyright 2019 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+
+impl Id<MailboxObject> {
+    pub fn into_hash(&self) -> MailboxHash {
+        let mut h = DefaultHasher::new();
+        h.write(self.inner.as_bytes());
+        h.finish()
+    }
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct MailboxObject {
+    pub id: Id<MailboxObject>,
+    pub is_subscribed: bool,
+    pub my_rights: JmapRights,
+    pub name: String,
+    pub parent_id: Option<Id<MailboxObject>>,
+    pub role: Option<String>,
+    pub sort_order: u64,
+    pub total_emails: u64,
+    pub total_threads: u64,
+    pub unread_emails: u64,
+    pub unread_threads: u64,
+}
+
+impl Object for MailboxObject {
+    const NAME: &'static str = "Mailbox";
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct JmapRights {
+    pub may_add_items: bool,
+    pub may_create_child: bool,
+    pub may_delete: bool,
+    pub may_read_items: bool,
+    pub may_remove_items: bool,
+    pub may_rename: bool,
+    pub may_set_keywords: bool,
+    pub may_set_seen: bool,
+    pub may_submit: bool,
+}
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct MailboxGet {
+    #[serde(flatten)]
+    pub get_call: Get<MailboxObject>,
+}
+impl MailboxGet {
+    pub fn new(get_call: Get<MailboxObject>) -> Self {
+        MailboxGet { get_call }
+    }
+}
+
+impl Method<MailboxObject> for MailboxGet {
+    const NAME: &'static str = "Mailbox/get";
+}

melib/src/backends/jmap/operations.rs

@@ -0,0 +1,90 @@
+/*
+ * meli - jmap module.
+ *
+ * Copyright 2017 - 2019 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+use std::sync::Arc;
+
+/// `BackendOp` implementor for Imap
+#[derive(Debug, Clone)]
+pub struct JmapOp {
+    hash: EnvelopeHash,
+    connection: Arc<FutureMutex<JmapConnection>>,
+    store: Arc<Store>,
+}
+
+impl JmapOp {
+    pub fn new(
+        hash: EnvelopeHash,
+        connection: Arc<FutureMutex<JmapConnection>>,
+        store: Arc<Store>,
+    ) -> Self {
+        JmapOp {
+            hash,
+            connection,
+            store,
+        }
+    }
+}
+
+impl BackendOp for JmapOp {
+    fn as_bytes(&mut self) -> ResultFuture<Vec<u8>> {
+        {
+            let byte_lck = self.store.byte_cache.lock().unwrap();
+            if byte_lck.contains_key(&self.hash) && byte_lck[&self.hash].bytes.is_some() {
+                let ret = byte_lck[&self.hash].bytes.clone().unwrap();
+                return Ok(Box::pin(async move { Ok(ret.into_bytes()) }));
+            }
+        }
+        let store = self.store.clone();
+        let hash = self.hash;
+        let connection = self.connection.clone();
+        Ok(Box::pin(async move {
+            let blob_id = store.blob_id_store.lock().unwrap()[&hash].clone();
+            let mut conn = connection.lock().await;
+            conn.connect().await?;
+            let download_url = conn.session.lock().unwrap().download_url.clone();
+            let mut res = conn
+                .client
+                .get_async(&download_request_format(
+                    download_url.as_str(),
+                    &conn.mail_account_id(),
+                    &blob_id,
+                    None,
+                ))
+                .await?;
+
+            let res_text = res.text_async().await?;
+
+            store
+                .byte_cache
+                .lock()
+                .unwrap()
+                .entry(hash)
+                .or_default()
+                .bytes = Some(res_text.clone());
+            Ok(res_text.into_bytes())
+        }))
+    }
+
+    fn fetch_flags(&self) -> ResultFuture<Flag> {
+        Ok(Box::pin(async { Ok(Flag::default()) }))
+    }
+}

melib/src/backends/jmap/protocol.rs

@@ -0,0 +1,348 @@
+/*
+ * meli - jmap module.
+ *
+ * Copyright 2019 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::mailbox::JmapMailbox;
+use super::*;
+use serde::Serialize;
+use serde_json::{json, Value};
+use std::convert::{TryFrom, TryInto};
+
+pub type UtcDate = String;
+
+use super::rfc8620::Object;
+
+macro_rules! get_request_no {
+    ($lock:expr) => {{
+        let mut lck = $lock.lock().unwrap();
+        let ret = *lck;
+        *lck += 1;
+        ret
+    }};
+}
+
+pub trait Response<OBJ: Object> {
+    const NAME: &'static str;
+}
+
+pub trait Method<OBJ: Object>: Serialize {
+    const NAME: &'static str;
+}
+
+static USING: &[&str] = &["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"];
+
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Request {
+    using: &'static [&'static str],
+    /* Why is this Value instead of Box<dyn Method<_>>? The Method trait cannot be made into a
+     * Trait object because its serialize() will be generic. */
+    method_calls: Vec<Value>,
+
+    #[serde(skip)]
+    request_no: Arc<Mutex<usize>>,
+}
+
+impl Request {
+    pub fn new(request_no: Arc<Mutex<usize>>) -> Self {
+        Request {
+            using: USING,
+            method_calls: Vec::new(),
+            request_no,
+        }
+    }
+
+    pub fn add_call<M: Method<O>, O: Object>(&mut self, call: &M) -> usize {
+        let seq = get_request_no!(self.request_no);
+        self.method_calls
+            .push(serde_json::to_value((M::NAME, call, &format!("m{}", seq))).unwrap());
+        seq
+    }
+}
+
+#[derive(Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct JsonResponse<'a> {
+    #[serde(borrow)]
+    method_responses: Vec<MethodResponse<'a>>,
+}
+
+pub async fn get_mailboxes(conn: &JmapConnection) -> Result<HashMap<MailboxHash, JmapMailbox>> {
+    let seq = get_request_no!(conn.request_no);
+    let api_url = conn.session.lock().unwrap().api_url.clone();
+    let mut res = conn
+        .client
+        .post_async(
+            api_url.as_str(),
+            serde_json::to_string(&json!({
+                "using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"],
+                "methodCalls": [["Mailbox/get", {
+                "accountId": conn.mail_account_id()
+                },
+                 format!("#m{}",seq).as_str()]],
+            }))?,
+        )
+        .await?;
+
+    let res_text = res.text_async().await?;
+    let mut v: MethodResponse = serde_json::from_str(&res_text).unwrap();
+    *conn.store.online_status.lock().await = (std::time::Instant::now(), Ok(()));
+    let m = GetResponse::<MailboxObject>::try_from(v.method_responses.remove(0))?;
+    let GetResponse::<MailboxObject> {
+        list, account_id, ..
+    } = m;
+    *conn.store.account_id.lock().unwrap() = account_id;
+    let mut ret: HashMap<MailboxHash, JmapMailbox> = list
+        .into_iter()
+        .map(|r| {
+            let MailboxObject {
+                id,
+                is_subscribed,
+                my_rights,
+                name,
+                parent_id,
+                role,
+                sort_order,
+                total_emails,
+                total_threads,
+                unread_emails,
+                unread_threads,
+            } = r;
+            let mut total_emails_set = LazyCountSet::default();
+            total_emails_set.set_not_yet_seen(total_emails.try_into().unwrap_or(0));
+            let total_emails = total_emails_set;
+            let mut unread_emails_set = LazyCountSet::default();
+            unread_emails_set.set_not_yet_seen(unread_emails.try_into().unwrap_or(0));
+            let unread_emails = unread_emails_set;
+            let hash = id.into_hash();
+            let parent_hash = parent_id.clone().map(|id| id.into_hash());
+            (
+                hash,
+                JmapMailbox {
+                    name: name.clone(),
+                    hash,
+                    path: name,
+                    children: Vec::new(),
+                    id,
+                    is_subscribed,
+                    my_rights,
+                    parent_id,
+                    parent_hash,
+                    role,
+                    usage: Default::default(),
+                    sort_order,
+                    total_emails: Arc::new(Mutex::new(total_emails)),
+                    total_threads,
+                    unread_emails: Arc::new(Mutex::new(unread_emails)),
+                    unread_threads,
+                    email_state: Arc::new(Mutex::new(None)),
+                    email_query_state: Arc::new(Mutex::new(None)),
+                },
+            )
+        })
+        .collect();
+    for key in ret.keys().cloned().collect::<SmallVec<[MailboxHash; 24]>>() {
+        if let Some(parent_hash) = ret[&key].parent_hash.clone() {
+            ret.entry(parent_hash).and_modify(|e| e.children.push(key));
+        }
+    }
+    Ok(ret)
+}
+
+pub async fn get_message_list(
+    conn: &JmapConnection,
+    mailbox: &JmapMailbox,
+) -> Result<Vec<Id<EmailObject>>> {
+    let email_call: EmailQuery = EmailQuery::new(
+        Query::new()
+            .account_id(conn.mail_account_id().clone())
+            .filter(Some(Filter::Condition(
+                EmailFilterCondition::new()
+                    .in_mailbox(Some(mailbox.id.clone()))
+                    .into(),
+            )))
+            .position(0),
+    )
+    .collapse_threads(false);
+
+    let mut req = Request::new(conn.request_no.clone());
+    req.add_call(&email_call);
+
+    let api_url = conn.session.lock().unwrap().api_url.clone();
+    let mut res = conn
+        .client
+        .post_async(api_url.as_str(), serde_json::to_string(&req)?)
+        .await?;
+
+    let res_text = res.text_async().await?;
+    let mut v: MethodResponse = serde_json::from_str(&res_text).unwrap();
+    *conn.store.online_status.lock().await = (std::time::Instant::now(), Ok(()));
+    let m = QueryResponse::<EmailObject>::try_from(v.method_responses.remove(0))?;
+    let QueryResponse::<EmailObject> { ids, .. } = m;
+    Ok(ids)
+}
+
+/*
+pub async fn get_message(conn: &JmapConnection, ids: &[String]) -> Result<Vec<Envelope>> {
+    let email_call: EmailGet = EmailGet::new(
+        Get::new()
+            .ids(Some(JmapArgument::value(ids.to_vec())))
+            .account_id(conn.mail_account_id().to_string()),
+    );
+
+    let mut req = Request::new(conn.request_no.clone());
+    req.add_call(&email_call);
+    let mut res = conn
+        .client
+        .post_async(&conn.session.api_url, serde_json::to_string(&req)?)
+        .await?;
+
+    let res_text = res.text_async().await?;
+    let mut v: MethodResponse = serde_json::from_str(&res_text).unwrap();
+    let e = GetResponse::<EmailObject>::try_from(v.method_responses.remove(0))?;
+    let GetResponse::<EmailObject> { list, .. } = e;
+    Ok(list
+        .into_iter()
+        .map(std::convert::Into::into)
+        .collect::<Vec<Envelope>>())
+}
+*/
+
+pub async fn fetch(
+    conn: &JmapConnection,
+    store: &Store,
+    mailbox_hash: MailboxHash,
+) -> Result<Vec<Envelope>> {
+    let mailbox_id = store.mailboxes.read().unwrap()[&mailbox_hash].id.clone();
+    let email_query_call: EmailQuery = EmailQuery::new(
+        Query::new()
+            .account_id(conn.mail_account_id().clone())
+            .filter(Some(Filter::Condition(
+                EmailFilterCondition::new()
+                    .in_mailbox(Some(mailbox_id))
+                    .into(),
+            )))
+            .position(0),
+    )
+    .collapse_threads(false);
+
+    let mut req = Request::new(conn.request_no.clone());
+    let prev_seq = req.add_call(&email_query_call);
+
+    let email_call: EmailGet = EmailGet::new(
+        Get::new()
+            .ids(Some(JmapArgument::reference(
+                prev_seq,
+                EmailQuery::RESULT_FIELD_IDS,
+            )))
+            .account_id(conn.mail_account_id().clone()),
+    );
+
+    req.add_call(&email_call);
+
+    let api_url = conn.session.lock().unwrap().api_url.clone();
+    let mut res = conn
+        .client
+        .post_async(api_url.as_str(), serde_json::to_string(&req)?)
+        .await?;
+
+    let res_text = res.text_async().await?;
+
+    let mut v: MethodResponse = serde_json::from_str(&res_text).unwrap();
+    let e = GetResponse::<EmailObject>::try_from(v.method_responses.pop().unwrap())?;
+    let query_response = QueryResponse::<EmailObject>::try_from(v.method_responses.pop().unwrap())?;
+    store
+        .mailboxes
+        .write()
+        .unwrap()
+        .entry(mailbox_hash)
+        .and_modify(|mbox| {
+            *mbox.email_query_state.lock().unwrap() = Some(query_response.query_state);
+        });
+    let GetResponse::<EmailObject> { list, state, .. } = e;
+    {
+        let (is_empty, is_equal) = {
+            let mailboxes_lck = conn.store.mailboxes.read().unwrap();
+            mailboxes_lck
+                .get(&mailbox_hash)
+                .map(|mbox| {
+                    let current_state_lck = mbox.email_state.lock().unwrap();
+                    (
+                        current_state_lck.is_none(),
+                        current_state_lck.as_ref() != Some(&state),
+                    )
+                })
+                .unwrap_or((true, true))
+        };
+        if is_empty {
+            let mut mailboxes_lck = conn.store.mailboxes.write().unwrap();
+            debug!("{:?}: inserting state {}", EmailObject::NAME, &state);
+            mailboxes_lck.entry(mailbox_hash).and_modify(|mbox| {
+                *mbox.email_state.lock().unwrap() = Some(state);
+            });
+        } else if !is_equal {
+            conn.email_changes(mailbox_hash).await?;
+        }
+    }
+    let mut total = BTreeSet::default();
+    let mut unread = BTreeSet::default();
+    let mut ret = Vec::with_capacity(list.len());
+    for obj in list {
+        let env = store.add_envelope(obj);
+        total.insert(env.hash());
+        if !env.is_seen() {
+            unread.insert(env.hash());
+        }
+        ret.push(env);
+    }
+    let mut mailboxes_lck = store.mailboxes.write().unwrap();
+    mailboxes_lck.entry(mailbox_hash).and_modify(|mbox| {
+        mbox.total_emails.lock().unwrap().insert_existing_set(total);
+        mbox.unread_emails
+            .lock()
+            .unwrap()
+            .insert_existing_set(unread);
+    });
+    Ok(ret)
+}
+
+pub fn keywords_to_flags(keywords: Vec<String>) -> (Flag, Vec<String>) {
+    let mut f = Flag::default();
+    let mut tags = vec![];
+    for k in keywords {
+        match k.as_str() {
+            "$draft" => {
+                f |= Flag::DRAFT;
+            }
+            "$seen" => {
+                f |= Flag::SEEN;
+            }
+            "$flagged" => {
+                f |= Flag::FLAGGED;
+            }
+            "$answered" => {
+                f |= Flag::REPLIED;
+            }
+            "$junk" | "$notjunk" => { /* ignore */ }
+            _ => tags.push(k),
+        }
+    }
+    (f, tags)
+}

melib/src/backends/jmap/rfc8620/argument.rs

@@ -0,0 +1,53 @@
+/*
+ * meli - jmap module.
+ *
+ * Copyright 2019 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use crate::backends::jmap::protocol::Method;
+use crate::backends::jmap::rfc8620::Object;
+use crate::backends::jmap::rfc8620::ResultField;
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub enum JmapArgument<T> {
+    Value(T),
+    ResultReference {
+        result_of: String,
+        name: String,
+        path: String,
+    },
+}
+
+impl<T> JmapArgument<T> {
+    pub fn value(v: T) -> Self {
+        JmapArgument::Value(v)
+    }
+
+    pub fn reference<M, OBJ>(result_of: usize, path: ResultField<M, OBJ>) -> Self
+    where
+        M: Method<OBJ>,
+        OBJ: Object,
+    {
+        JmapArgument::ResultReference {
+            result_of: format!("m{}", result_of),
+            name: M::NAME.to_string(),
+            path: path.field.to_string(),
+        }
+    }
+}

melib/src/backends/jmap/rfc8620/comparator.rs

@@ -0,0 +1,53 @@
+/*
+ * meli - jmap module.
+ *
+ * Copyright 2019 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+
+#[derive(Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct Comparator<OBJ: Object> {
+    property: String,
+    #[serde(default = "bool_true")]
+    is_ascending: bool,
+    //FIXME
+    collation: Option<String>,
+    //#[serde(flatten)]
+    additional_properties: Vec<String>,
+
+    _ph: PhantomData<fn() -> OBJ>,
+}
+
+impl<OBJ: Object> Comparator<OBJ> {
+    pub fn new() -> Self {
+        Self {
+            property: String::new(),
+            is_ascending: true,
+            collation: None,
+            additional_properties: Vec::new(),
+            _ph: PhantomData,
+        }
+    }
+
+    _impl!(property: String);
+    _impl!(is_ascending: bool);
+    _impl!(collation: Option<String>);
+    _impl!(additional_properties: Vec<String>);
+}

melib/src/backends/jmap/rfc8620/filters.rs

@@ -0,0 +1,139 @@
+/*
+ * meli - jmap module.
+ *
+ * Copyright 2019 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+
+pub trait FilterTrait<T>: Default {}
+#[derive(Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+#[serde(untagged)]
+pub enum Filter<F: FilterTrait<OBJ>, OBJ: Object> {
+    Operator {
+        operator: FilterOperator,
+        conditions: Vec<Filter<F, OBJ>>,
+    },
+    Condition(FilterCondition<F, OBJ>),
+}
+
+impl<F: FilterTrait<OBJ>, OBJ: Object> FilterTrait<OBJ> for Filter<F, OBJ> {}
+impl<F: FilterTrait<OBJ>, OBJ: Object> FilterTrait<OBJ> for FilterCondition<F, OBJ> {}
+
+#[derive(Serialize, Debug)]
+pub struct FilterCondition<F: FilterTrait<OBJ>, OBJ: Object> {
+    #[serde(flatten)]
+    pub cond: F,
+    #[serde(skip)]
+    pub _ph: PhantomData<fn() -> OBJ>,
+}
+
+#[derive(Serialize, Debug, PartialEq)]
+#[serde(rename_all = "UPPERCASE")]
+pub enum FilterOperator {
+    And,
+    Or,
+    Not,
+}
+
+impl<F: FilterTrait<OBJ>, OBJ: Object> FilterCondition<F, OBJ> {
+    pub fn new() -> Self {
+        FilterCondition {
+            cond: F::default(),
+            _ph: PhantomData,
+        }
+    }
+}
+
+impl<F: FilterTrait<OBJ>, OBJ: Object> Default for FilterCondition<F, OBJ> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<F: FilterTrait<OBJ>, OBJ: Object> Default for Filter<F, OBJ> {
+    fn default() -> Self {
+        Filter::Condition(FilterCondition::default())
+    }
+}
+
+use std::ops::{BitAndAssign, BitOrAssign, Not};
+
+impl<F: FilterTrait<OBJ>, OBJ: Object> BitAndAssign for Filter<F, OBJ> {
+    fn bitand_assign(&mut self, rhs: Self) {
+        match self {
+            Filter::Operator {
+                operator: FilterOperator::And,
+                ref mut conditions,
+            } => {
+                conditions.push(rhs);
+            }
+            Filter::Condition(_) | Filter::Operator { .. } => {
+                *self = Filter::Operator {
+                    operator: FilterOperator::And,
+                    conditions: vec![
+                        std::mem::replace(self, Filter::Condition(FilterCondition::new())),
+                        rhs,
+                    ],
+                };
+            }
+        }
+    }
+}
+
+impl<F: FilterTrait<OBJ>, OBJ: Object> BitOrAssign for Filter<F, OBJ> {
+    fn bitor_assign(&mut self, rhs: Self) {
+        match self {
+            Filter::Operator {
+                operator: FilterOperator::Or,
+                ref mut conditions,
+            } => {
+                conditions.push(rhs);
+            }
+            Filter::Condition(_) | Filter::Operator { .. } => {
+                *self = Filter::Operator {
+                    operator: FilterOperator::Or,
+                    conditions: vec![
+                        std::mem::replace(self, Filter::Condition(FilterCondition::new())),
+                        rhs,
+                    ],
+                };
+            }
+        }
+    }
+}
+
+impl<F: FilterTrait<OBJ>, OBJ: Object> Not for Filter<F, OBJ> {
+    type Output = Self;
+    fn not(self) -> Self {
+        match self {
+            Filter::Operator {
+                operator,
+                conditions,
+            } if operator == FilterOperator::Not => Filter::Operator {
+                operator: FilterOperator::Or,
+                conditions,
+            },
+            Filter::Condition(_) | Filter::Operator { .. } => Filter::Operator {
+                operator: FilterOperator::Not,
+                conditions: vec![self],
+            },
+        }
+    }
+}

melib/src/backends/maildir.rs

@@ -23,32 +23,35 @@
 mod backend;
 pub use self::backend::*;
 
+mod stream;
+pub use stream::*;
+
 use crate::backends::*;
-use crate::email::parser;
-use crate::email::{Envelope, Flag};
+use crate::email::Flag;
 use crate::error::{MeliError, Result};
 use crate::shellexpand::ShellExpandTrait;
-
-use memmap::{Mmap, Protection};
+use futures::stream::Stream;
 use std::collections::hash_map::DefaultHasher;
 use std::fs;
 use std::hash::{Hash, Hasher};
+use std::io::{BufReader, Read};
 use std::path::{Path, PathBuf};
+use std::sync::{Arc, Mutex};
 
 /// `BackendOp` implementor for Maildir
 #[derive(Debug)]
 pub struct MaildirOp {
     hash_index: HashIndexes,
-    folder_hash: FolderHash,
+    mailbox_hash: MailboxHash,
     hash: EnvelopeHash,
-    slice: Option<Mmap>,
+    slice: Option<Vec<u8>>,
 }
 
 impl Clone for MaildirOp {
     fn clone(&self) -> Self {
         MaildirOp {
             hash_index: self.hash_index.clone(),
-            folder_hash: self.folder_hash,
+            mailbox_hash: self.mailbox_hash,
             hash: self.hash,
             slice: None,
         }
@@ -56,188 +59,130 @@ impl Clone for MaildirOp {
 }
 
 impl MaildirOp {
-    pub fn new(hash: EnvelopeHash, hash_index: HashIndexes, folder_hash: FolderHash) -> Self {
+    pub fn new(hash: EnvelopeHash, hash_index: HashIndexes, mailbox_hash: MailboxHash) -> Self {
         MaildirOp {
             hash_index,
-            folder_hash,
+            mailbox_hash,
             hash,
             slice: None,
         }
     }
-    fn path(&self) -> PathBuf {
+    fn path(&self) -> Result<PathBuf> {
         let map = self.hash_index.lock().unwrap();
-        let map = &map[&self.folder_hash];
-        debug!("looking for {} in {} map", self.hash, self.folder_hash);
+        let map = &map[&self.mailbox_hash];
+        debug!("looking for {} in {} map", self.hash, self.mailbox_hash);
         if !map.contains_key(&self.hash) {
             debug!("doesn't contain it though len = {}\n{:#?}", map.len(), map);
             for e in map.iter() {
                 debug!("{:#?}", e);
             }
+            return Err(MeliError::new("File not found"));
         }
-        if let Some(modif) = &map[&self.hash].modified {
+
+        Ok(if let Some(modif) = &map[&self.hash].modified {
             match modif {
                 PathMod::Path(ref path) => path.clone(),
                 PathMod::Hash(hash) => map[&hash].to_path_buf(),
             }
         } else {
             map.get(&self.hash).unwrap().to_path_buf()
-        }
+        })
     }
 }
 
 impl<'a> BackendOp for MaildirOp {
-    fn description(&self) -> String {
-        format!("Path of file: {}", self.path().display())
-    }
-    fn as_bytes(&mut self) -> Result<&[u8]> {
+    fn as_bytes(&mut self) -> ResultFuture<Vec<u8>> {
         if self.slice.is_none() {
-            self.slice = Some(Mmap::open_path(self.path(), Protection::Read)?);
+            let file = std::fs::OpenOptions::new()
+                .read(true)
+                .write(false)
+                .open(&self.path()?)?;
+            let mut buf_reader = BufReader::new(file);
+            let mut contents = Vec::new();
+            buf_reader.read_to_end(&mut contents)?;
+            self.slice = Some(contents);
         }
-        /* Unwrap is safe since we use ? above. */
-        Ok(unsafe { self.slice.as_ref().unwrap().as_slice() })
-    }
-    fn fetch_headers(&mut self) -> Result<&[u8]> {
-        let raw = self.as_bytes()?;
-        let result = parser::headers_raw(raw).to_full_result()?;
-        Ok(result)
-    }
-    fn fetch_body(&mut self) -> Result<&[u8]> {
-        let raw = self.as_bytes()?;
-        let result = parser::body_raw(raw).to_full_result()?;
-        Ok(result)
-    }
-    fn fetch_flags(&self) -> Flag {
-        let mut flag = Flag::default();
-        let path = self.path();
-        let path = path.to_str().unwrap(); // Assume UTF-8 validity
-        if !path.contains(":2,") {
-            return flag;
-        }
-
-        for f in path.chars().rev() {
-            match f {
-                ',' => break,
-                'D' => flag |= Flag::DRAFT,
-                'F' => flag |= Flag::FLAGGED,
-                'P' => flag |= Flag::PASSED,
-                'R' => flag |= Flag::REPLIED,
-                'S' => flag |= Flag::SEEN,
-                'T' => flag |= Flag::TRASHED,
-                _ => {
-                    debug!("DEBUG: in fetch_flags, path is {}", path);
-                }
-            }
-        }
-
-        flag
+        let ret = Ok(self.slice.as_ref().unwrap().as_slice().to_vec());
+        Ok(Box::pin(async move { ret }))
     }
 
-    fn set_flag(&mut self, envelope: &mut Envelope, f: Flag) -> Result<()> {
-        let path = self.path();
-        let path = path.to_str().unwrap(); // Assume UTF-8 validity
-        let idx: usize = path
-            .rfind(":2,")
-            .ok_or_else(|| MeliError::new(format!("Invalid email filename: {:?}", self)))?
-            + 3;
-        let mut new_name: String = path[..idx].to_string();
-        let mut flags = self.fetch_flags();
-        flags.toggle(f);
-        if !(flags & Flag::DRAFT).is_empty() {
-            new_name.push('D');
-        }
-        if !(flags & Flag::FLAGGED).is_empty() {
-            new_name.push('F');
-        }
-        if !(flags & Flag::PASSED).is_empty() {
-            new_name.push('P');
-        }
-        if !(flags & Flag::REPLIED).is_empty() {
-            new_name.push('R');
-        }
-        if !(flags & Flag::SEEN).is_empty() {
-            new_name.push('S');
-        }
-        if !(flags & Flag::TRASHED).is_empty() {
-            new_name.push('T');
-        }
-        let old_hash = envelope.hash();
-        let new_name: PathBuf = new_name.into();
-        let hash_index = self.hash_index.clone();
-        let mut map = hash_index.lock().unwrap();
-        let map = map.entry(self.folder_hash).or_default();
-        map.entry(old_hash).or_default().modified = Some(PathMod::Path(new_name.clone()));
-
-        debug!("renaming {:?} to {:?}", path, new_name);
-        fs::rename(&path, &new_name)?;
-        debug!("success in rename");
-        Ok(())
+    fn fetch_flags(&self) -> ResultFuture<Flag> {
+        let path = self.path()?;
+        let ret = Ok(path.flags());
+        Ok(Box::pin(async move { ret }))
     }
 }
 
-#[derive(Debug, Default)]
-pub struct MaildirFolder {
-    hash: FolderHash,
+#[derive(Debug, Default, Clone)]
+pub struct MaildirMailbox {
+    hash: MailboxHash,
     name: String,
     fs_path: PathBuf,
     path: PathBuf,
-    parent: Option<FolderHash>,
-    children: Vec<FolderHash>,
+    parent: Option<MailboxHash>,
+    children: Vec<MailboxHash>,
+    pub usage: Arc<RwLock<SpecialUsageMailbox>>,
+    pub is_subscribed: bool,
+    permissions: MailboxPermissions,
+    pub total: Arc<Mutex<usize>>,
+    pub unseen: Arc<Mutex<usize>>,
 }
 
-impl MaildirFolder {
+impl MaildirMailbox {
     pub fn new(
         path: String,
         file_name: String,
-        parent: Option<FolderHash>,
-        children: Vec<FolderHash>,
+        parent: Option<MailboxHash>,
+        children: Vec<MailboxHash>,
+        accept_invalid: bool,
         settings: &AccountSettings,
     ) -> Result<Self> {
-        macro_rules! strip_slash {
-            ($v:expr) => {
-                if $v.ends_with("/") {
-                    &$v[..$v.len() - 1]
-                } else {
-                    $v
-                }
-            };
-        }
         let pathbuf = PathBuf::from(&path);
         let mut h = DefaultHasher::new();
         pathbuf.hash(&mut h);
 
-        /* Check if folder path (Eg `INBOX/Lists/luddites`) is included in the subscribed
+        /* Check if mailbox path (Eg `INBOX/Lists/luddites`) is included in the subscribed
          * mailboxes in user configuration */
-        let fname = if let Ok(fname) = pathbuf.strip_prefix(
-            PathBuf::from(&settings.root_folder)
-                .expand()
-                .parent()
-                .unwrap_or_else(|| &Path::new("/")),
-        ) {
-            if fname.components().count() != 0
-                && !settings
-                    .subscribed_folders
-                    .iter()
-                    .any(|x| x == strip_slash!(fname.to_str().unwrap()))
-            {
-                return Err(MeliError::new(format!(
-                    "Folder with name `{}` is not included in configured subscribed mailboxes",
-                    fname.display()
-                )));
-            }
-            Some(fname)
+        let fname = pathbuf
+            .strip_prefix(
+                PathBuf::from(&settings.root_mailbox)
+                    .expand()
+                    .parent()
+                    .unwrap_or_else(|| &Path::new("/")),
+            )
+            .ok();
+
+        let read_only = if let Ok(metadata) = std::fs::metadata(&pathbuf) {
+            metadata.permissions().readonly()
         } else {
-            None
+            true
         };
 
-        let ret = MaildirFolder {
+        let ret = MaildirMailbox {
             hash: h.finish(),
             name: file_name,
             path: fname.unwrap().to_path_buf(),
             fs_path: pathbuf,
             parent,
             children,
+            usage: Arc::new(RwLock::new(SpecialUsageMailbox::Normal)),
+            is_subscribed: false,
+            permissions: MailboxPermissions {
+                create_messages: !read_only,
+                remove_messages: !read_only,
+                set_flags: !read_only,
+                create_child: !read_only,
+                rename_messages: !read_only,
+                delete_messages: !read_only,
+                delete_mailbox: !read_only,
+                change_permissions: false,
+            },
+            unseen: Arc::new(Mutex::new(0)),
+            total: Arc::new(Mutex::new(0)),
         };
-        ret.is_valid()?;
+        if !accept_invalid {
+            ret.is_valid()?;
+        }
         Ok(ret)
     }
 
@@ -252,7 +197,7 @@ impl MaildirFolder {
             p.push(d);
             if !p.is_dir() {
                 return Err(MeliError::new(format!(
-                    "{} is not a valid maildir folder",
+                    "{} is not a valid maildir mailbox",
                     path.display()
                 )));
             }
@@ -261,8 +206,9 @@ impl MaildirFolder {
         Ok(())
     }
 }
-impl BackendFolder for MaildirFolder {
-    fn hash(&self) -> FolderHash {
+
+impl BackendMailbox for MaildirMailbox {
+    fn hash(&self) -> MailboxHash {
         self.hash
     }
 
@@ -278,22 +224,70 @@ impl BackendFolder for MaildirFolder {
         self.name = s.to_string();
     }
 
-    fn children(&self) -> &Vec<FolderHash> {
+    fn children(&self) -> &[MailboxHash] {
         &self.children
     }
 
-    fn clone(&self) -> Folder {
-        Box::new(MaildirFolder {
-            hash: self.hash,
-            name: self.name.clone(),
-            fs_path: self.fs_path.clone(),
-            path: self.path.clone(),
-            children: self.children.clone(),
-            parent: self.parent,
-        })
+    fn clone(&self) -> Mailbox {
+        Box::new(std::clone::Clone::clone(self))
     }
 
-    fn parent(&self) -> Option<FolderHash> {
+    fn special_usage(&self) -> SpecialUsageMailbox {
+        *self.usage.read().unwrap()
+    }
+
+    fn parent(&self) -> Option<MailboxHash> {
         self.parent
     }
+
+    fn permissions(&self) -> MailboxPermissions {
+        self.permissions
+    }
+    fn is_subscribed(&self) -> bool {
+        self.is_subscribed
+    }
+    fn set_is_subscribed(&mut self, new_val: bool) -> Result<()> {
+        self.is_subscribed = new_val;
+        Ok(())
+    }
+
+    fn set_special_usage(&mut self, new_val: SpecialUsageMailbox) -> Result<()> {
+        *self.usage.write()? = new_val;
+        Ok(())
+    }
+
+    fn count(&self) -> Result<(usize, usize)> {
+        Ok((*self.unseen.lock()?, *self.total.lock()?))
+    }
+}
+
+pub trait MaildirPathTrait {
+    fn flags(&self) -> Flag;
+}
+
+impl MaildirPathTrait for Path {
+    fn flags(&self) -> Flag {
+        let mut flag = Flag::default();
+        let path = self.to_string_lossy();
+        if !path.contains(":2,") {
+            return flag;
+        }
+
+        for f in path.chars().rev() {
+            match f {
+                ',' => break,
+                'D' => flag |= Flag::DRAFT,
+                'F' => flag |= Flag::FLAGGED,
+                'P' => flag |= Flag::PASSED,
+                'R' => flag |= Flag::REPLIED,
+                'S' => flag |= Flag::SEEN,
+                'T' => flag |= Flag::TRASHED,
+                _ => {
+                    debug!("DEBUG: in MaildirPathTrait::flags(), encountered unknown flag marker {:?}, path is {}", f, path);
+                }
+            }
+        }
+
+        flag
+    }
 }

melib/src/backends/maildir/stream.rs

@@ -0,0 +1,199 @@
+/*
+ * meli - maildir async
+ *
+ * Copyright 2020 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+use crate::backends::maildir::backend::move_to_cur;
+use core::future::Future;
+use core::pin::Pin;
+use futures::stream::{FuturesUnordered, StreamExt};
+use futures::task::{Context, Poll};
+use std::io::{self, Read};
+use std::os::unix::fs::PermissionsExt;
+use std::path::PathBuf;
+use std::result;
+use std::sync::{Arc, Mutex};
+
+pub struct MaildirStream {
+    payloads: Pin<
+        Box<
+            FuturesUnordered<Pin<Box<dyn Future<Output = Result<Vec<Envelope>>> + Send + 'static>>>,
+        >,
+    >,
+}
+
+impl MaildirStream {
+    pub fn new(
+        name: &str,
+        mailbox_hash: MailboxHash,
+        unseen: Arc<Mutex<usize>>,
+        total: Arc<Mutex<usize>>,
+        mut path: PathBuf,
+        root_path: PathBuf,
+        map: HashIndexes,
+        mailbox_index: Arc<Mutex<HashMap<EnvelopeHash, MailboxHash>>>,
+    ) -> Result<Pin<Box<dyn Stream<Item = Result<Vec<Envelope>>> + Send + 'static>>> {
+        let chunk_size = 2048;
+        path.push("new");
+        for d in path.read_dir()? {
+            if let Ok(p) = d {
+                move_to_cur(p.path()).ok().take();
+            }
+        }
+        path.pop();
+        path.push("cur");
+        let iter = path.read_dir()?;
+        let count = path.read_dir()?.count();
+        let mut files: Vec<PathBuf> = Vec::with_capacity(count);
+        for e in iter {
+            let e = e.and_then(|x| {
+                let path = x.path();
+                Ok(path)
+            })?;
+            files.push(e);
+        }
+        let payloads = Box::pin(if !files.is_empty() {
+            files
+                .chunks(chunk_size)
+                .map(|chunk| {
+                    let cache_dir = xdg::BaseDirectories::with_profile("meli", &name).unwrap();
+                    Box::pin(Self::chunk(
+                        SmallVec::from(chunk),
+                        cache_dir,
+                        mailbox_hash,
+                        unseen.clone(),
+                        total.clone(),
+                        root_path.clone(),
+                        map.clone(),
+                        mailbox_index.clone(),
+                    )) as Pin<Box<dyn Future<Output = _> + Send + 'static>>
+                })
+                .collect::<_>()
+        } else {
+            FuturesUnordered::new()
+        });
+        Ok(Self { payloads }.boxed())
+    }
+
+    async fn chunk(
+        chunk: SmallVec<[std::path::PathBuf; 2048]>,
+        cache_dir: xdg::BaseDirectories,
+        mailbox_hash: MailboxHash,
+        unseen: Arc<Mutex<usize>>,
+        total: Arc<Mutex<usize>>,
+        root_path: PathBuf,
+        map: HashIndexes,
+        mailbox_index: Arc<Mutex<HashMap<EnvelopeHash, MailboxHash>>>,
+    ) -> Result<Vec<Envelope>> {
+        let mut local_r: Vec<Envelope> = Vec::with_capacity(chunk.len());
+        let mut unseen_total: usize = 0;
+        let mut buf = Vec::with_capacity(4096);
+        for file in chunk {
+            /* Check if we have a cache file with this email's
+             * filename */
+            let file_name = PathBuf::from(&file)
+                .strip_prefix(&root_path)
+                .unwrap()
+                .to_path_buf();
+            if let Some(cached) = cache_dir.find_cache_file(&file_name) {
+                /* Cached struct exists, try to load it */
+                let cached_file = fs::File::open(&cached)?;
+                let filesize = cached_file.metadata()?.len();
+                let reader = io::BufReader::new(cached_file);
+                let result: result::Result<Envelope, _> = bincode::Options::deserialize_from(
+                    bincode::Options::with_limit(
+                        bincode::config::DefaultOptions::new(),
+                        2 * filesize,
+                    ),
+                    reader,
+                );
+                if let Ok(env) = result {
+                    let mut map = map.lock().unwrap();
+                    let map = map.entry(mailbox_hash).or_default();
+                    let hash = env.hash();
+                    map.insert(hash, file.clone().into());
+                    mailbox_index.lock().unwrap().insert(hash, mailbox_hash);
+                    if !env.is_seen() {
+                        unseen_total += 1;
+                    }
+                    local_r.push(env);
+                    continue;
+                }
+                /* Try delete invalid file */
+                let _ = fs::remove_file(&cached);
+            };
+            let env_hash = get_file_hash(&file);
+            {
+                let mut map = map.lock().unwrap();
+                let map = map.entry(mailbox_hash).or_default();
+                map.insert(env_hash, PathBuf::from(&file).into());
+            }
+            let mut reader = io::BufReader::new(fs::File::open(&file)?);
+            buf.clear();
+            reader.read_to_end(&mut buf)?;
+            match Envelope::from_bytes(buf.as_slice(), Some(file.flags())) {
+                Ok(mut env) => {
+                    env.set_hash(env_hash);
+                    mailbox_index.lock().unwrap().insert(env_hash, mailbox_hash);
+                    if let Ok(cached) = cache_dir.place_cache_file(file_name) {
+                        /* place result in cache directory */
+                        let f = fs::File::create(cached)?;
+                        let metadata = f.metadata()?;
+                        let mut permissions = metadata.permissions();
+
+                        permissions.set_mode(0o600); // Read/write for owner only.
+                        f.set_permissions(permissions)?;
+
+                        let writer = io::BufWriter::new(f);
+                        bincode::Options::serialize_into(
+                            bincode::config::DefaultOptions::new(),
+                            writer,
+                            &env,
+                        )?;
+                    }
+                    if !env.is_seen() {
+                        unseen_total += 1;
+                    }
+                    local_r.push(env);
+                }
+                Err(err) => {
+                    debug!(
+                        "DEBUG: hash {}, path: {} couldn't be parsed, {}",
+                        env_hash,
+                        file.as_path().display(),
+                        err,
+                    );
+                    continue;
+                }
+            }
+        }
+        *total.lock().unwrap() += local_r.len();
+        *unseen.lock().unwrap() += unseen_total;
+        Ok(local_r)
+    }
+}
+
+impl Stream for MaildirStream {
+    type Item = Result<Vec<Envelope>>;
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        let payloads = self.payloads.as_mut();
+        payloads.poll_next(cx)
+    }
+}

melib/src/backends/mbox/write.rs

@@ -0,0 +1,260 @@
+/*
+ * meli - mailbox module.
+ *
+ * Copyright 2021  Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+
+impl MboxFormat {
+    pub fn append(
+        &self,
+        writer: &mut dyn std::io::Write,
+        input: &[u8],
+        envelope_from: Option<&Address>,
+        delivery_date: Option<crate::UnixTimestamp>,
+        (flags, tags): (Flag, Vec<&str>),
+        metadata_format: MboxMetadata,
+        is_empty: bool,
+        crlf: bool,
+    ) -> Result<()> {
+        if tags.iter().any(|t| t.contains(' ')) {
+            return Err(MeliError::new("mbox tags/keywords can't contain spaces"));
+        }
+        let line_ending: &'static [u8] = if crlf { &b"\r\n"[..] } else { &b"\n"[..] };
+        if !is_empty {
+            writer.write_all(line_ending)?;
+            writer.write_all(line_ending)?;
+        }
+        writer.write_all(&b"From "[..])?;
+        if let Some(from) = envelope_from {
+            writer.write_all(from.address_spec_raw())?;
+        } else {
+            writer.write_all(&b"MAILER-DAEMON"[..])?;
+        }
+        writer.write_all(&b" "[..])?;
+        writer.write_all(
+            crate::datetime::timestamp_to_string(
+                delivery_date.unwrap_or_else(|| crate::datetime::now()),
+                Some(crate::datetime::ASCTIME_FMT),
+                true,
+            )
+            .trim()
+            .as_bytes(),
+        )?;
+        writer.write_all(line_ending)?;
+        let (mut headers, body) = parser::mail(input)?;
+        headers.retain(|(header_name, _)| {
+            !header_name.eq_ignore_ascii_case(b"Status")
+                && !header_name.eq_ignore_ascii_case(b"X-Status")
+                && !header_name.eq_ignore_ascii_case(b"X-Keywords")
+                && !header_name.eq_ignore_ascii_case(b"Content-Length")
+        });
+        let write_header_val_fn = |writer: &mut dyn std::io::Write, bytes: &[u8]| {
+            let mut i = 0;
+            if crlf {
+                while i < bytes.len() {
+                    if bytes[i..].starts_with(b"\r\n") {
+                        writer.write_all(&[b'\r', b'\n'])?;
+                        i += 2;
+                        continue;
+                    } else if bytes[i] == b'\n' {
+                        writer.write_all(&[b'\r', b'\n'])?;
+                    } else {
+                        writer.write_all(&[bytes[i]])?;
+                    }
+                    i += 1;
+                }
+            } else {
+                while i < bytes.len() {
+                    if bytes[i..].starts_with(b"\r\n") {
+                        writer.write_all(&[b'\n'])?;
+                        i += 2;
+                    } else {
+                        writer.write_all(&[bytes[i]])?;
+                        i += 1;
+                    }
+                }
+            }
+            Ok::<(), MeliError>(())
+        };
+        let write_metadata_fn = |writer: &mut dyn std::io::Write| match metadata_format {
+            MboxMetadata::CClient => {
+                for (h, v) in {
+                    if flags.is_seen() {
+                        Some((&b"Status"[..], "R".into()))
+                    } else {
+                        None
+                    }
+                    .into_iter()
+                    .chain(
+                        if !flags.is_flagged()
+                            && !flags.is_replied()
+                            && !flags.is_draft()
+                            && !flags.is_trashed()
+                        {
+                            None
+                        } else {
+                            Some((
+                                &b"X-Status"[..],
+                                format!(
+                                    "{flagged}{replied}{draft}{trashed}",
+                                    flagged = if flags.is_flagged() { "F" } else { "" },
+                                    replied = if flags.is_replied() { "A" } else { "" },
+                                    draft = if flags.is_draft() { "T" } else { "" },
+                                    trashed = if flags.is_trashed() { "D" } else { "" }
+                                ),
+                            ))
+                        },
+                    )
+                    .chain(if tags.is_empty() {
+                        None
+                    } else {
+                        Some((&b"X-Keywords"[..], tags.as_slice().join(" ")))
+                    })
+                } {
+                    writer.write_all(h)?;
+                    writer.write_all(&b": "[..])?;
+                    writer.write_all(v.as_bytes())?;
+                    writer.write_all(line_ending)?;
+                }
+                Ok::<(), MeliError>(())
+            }
+            MboxMetadata::None => Ok(()),
+        };
+
+        let body_len = {
+            let mut len = body.len();
+            if crlf {
+                let stray_lfs = body.iter().filter(|b| **b == b'\n').count()
+                    - body.windows(b"\r\n".len()).filter(|w| w == b"\r\n").count();
+                len += stray_lfs;
+            } else {
+                let crlfs = body.windows(b"\r\n".len()).filter(|w| w == b"\r\n").count();
+                len -= crlfs;
+            }
+            len
+        };
+
+        match self {
+            MboxFormat::MboxO | MboxFormat::MboxRd => Err(MeliError::new("Unimplemented.")),
+            MboxFormat::MboxCl => {
+                let len = (body_len
+                    + body
+                        .windows(b"\nFrom ".len())
+                        .filter(|w| w == b"\nFrom ")
+                        .count()
+                    + if body.starts_with(b"From ") { 1 } else { 0 })
+                .to_string();
+                for (h, v) in headers
+                    .into_iter()
+                    .chain(Some((&b"Content-Length"[..], len.as_bytes())))
+                {
+                    writer.write_all(h)?;
+                    writer.write_all(&b": "[..])?;
+                    write_header_val_fn(writer, v)?;
+                    writer.write_all(line_ending)?;
+                }
+                write_metadata_fn(writer)?;
+                writer.write_all(line_ending)?;
+
+                if body.starts_with(b"From ") {
+                    writer.write_all(&[b'>'])?;
+                }
+                let mut i = 0;
+                if crlf {
+                    while i < body.len() {
+                        if body[i..].starts_with(b"\r\n") {
+                            writer.write_all(&[b'\r', b'\n'])?;
+                            if body[i..].starts_with(b"\r\nFrom ") {
+                                writer.write_all(&[b'>'])?;
+                            }
+                            i += 2;
+                        } else if body[i] == b'\n' {
+                            writer.write_all(&[b'\r', b'\n'])?;
+                            if body[i..].starts_with(b"\nFrom ") {
+                                writer.write_all(&[b'>'])?;
+                            }
+                            i += 1;
+                        } else {
+                            writer.write_all(&[body[i]])?;
+                            i += 1;
+                        }
+                    }
+                } else {
+                    while i < body.len() {
+                        if body[i..].starts_with(b"\r\n") {
+                            writer.write_all(&[b'\n'])?;
+                            if body[i..].starts_with(b"\r\nFrom ") {
+                                writer.write_all(&[b'>'])?;
+                            }
+                            i += 2;
+                        } else {
+                            writer.write_all(&[body[i]])?;
+                            if body[i..].starts_with(b"\nFrom ") {
+                                writer.write_all(&[b'>'])?;
+                            }
+                            i += 1;
+                        }
+                    }
+                }
+                Ok(())
+            }
+            MboxFormat::MboxCl2 => {
+                let len = body_len.to_string();
+                for (h, v) in headers
+                    .into_iter()
+                    .chain(Some((&b"Content-Length"[..], len.as_bytes())))
+                {
+                    writer.write_all(h)?;
+                    writer.write_all(&b": "[..])?;
+                    write_header_val_fn(writer, v)?;
+                    writer.write_all(line_ending)?;
+                }
+                write_metadata_fn(writer)?;
+                writer.write_all(line_ending)?;
+                let mut i = 0;
+                if crlf {
+                    while i < body.len() {
+                        if body[i..].starts_with(b"\r\n") {
+                            writer.write_all(&[b'\r', b'\n'])?;
+                            i += 2;
+                            continue;
+                        } else if body[i] == b'\n' {
+                            writer.write_all(&[b'\r', b'\n'])?;
+                        } else {
+                            writer.write_all(&[body[i]])?;
+                        }
+                        i += 1;
+                    }
+                } else {
+                    while i < body.len() {
+                        if body[i..].starts_with(b"\r\n") {
+                            writer.write_all(&[b'\n'])?;
+                            i += 2;
+                        } else {
+                            writer.write_all(&[body[i]])?;
+                            i += 1;
+                        }
+                    }
+                }
+                Ok(())
+            }
+        }
+    }
+}

melib/src/backends/nntp.rs

@@ -0,0 +1,633 @@
+/*
+ * meli - nntp module.
+ *
+ * Copyright 2019 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use crate::get_conf_val;
+use crate::get_path_hash;
+use smallvec::SmallVec;
+#[macro_use]
+mod protocol_parser;
+pub use protocol_parser::*;
+mod mailbox;
+pub use mailbox::*;
+mod operations;
+pub use operations::*;
+mod connection;
+pub use connection::*;
+
+use crate::conf::AccountSettings;
+use crate::connections::timeout;
+use crate::email::*;
+use crate::error::{MeliError, Result, ResultIntoMeliError};
+use crate::{backends::*, Collection};
+use futures::lock::Mutex as FutureMutex;
+use futures::stream::Stream;
+use std::collections::{hash_map::DefaultHasher, BTreeSet, HashMap, HashSet};
+use std::hash::Hasher;
+use std::pin::Pin;
+use std::str::FromStr;
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+pub type UID = usize;
+
+pub static SUPPORTED_CAPABILITIES: &[&str] = &[
+    #[cfg(feature = "deflate_compression")]
+    "COMPRESS DEFLATE",
+    "VERSION 2",
+];
+
+#[derive(Debug, Clone)]
+pub struct NntpServerConf {
+    pub server_hostname: String,
+    pub server_username: String,
+    pub server_password: String,
+    pub server_port: u16,
+    pub use_starttls: bool,
+    pub use_tls: bool,
+    pub require_auth: bool,
+    pub danger_accept_invalid_certs: bool,
+    pub extension_use: NntpExtensionUse,
+}
+
+type Capabilities = HashSet<String>;
+
+#[derive(Debug)]
+pub struct UIDStore {
+    account_hash: AccountHash,
+    account_name: Arc<String>,
+    offline_cache: bool,
+    capabilities: Arc<Mutex<Capabilities>>,
+    hash_index: Arc<Mutex<HashMap<EnvelopeHash, (UID, MailboxHash)>>>,
+    uid_index: Arc<Mutex<HashMap<(MailboxHash, UID), EnvelopeHash>>>,
+
+    collection: Collection,
+    mailboxes: Arc<FutureMutex<HashMap<MailboxHash, NntpMailbox>>>,
+    is_online: Arc<Mutex<(Instant, Result<()>)>>,
+    event_consumer: BackendEventConsumer,
+}
+
+impl UIDStore {
+    fn new(
+        account_hash: AccountHash,
+        account_name: Arc<String>,
+        event_consumer: BackendEventConsumer,
+    ) -> Self {
+        UIDStore {
+            account_hash,
+            account_name,
+            event_consumer,
+            offline_cache: false,
+            capabilities: Default::default(),
+            hash_index: Default::default(),
+            uid_index: Default::default(),
+            mailboxes: Arc::new(FutureMutex::new(Default::default())),
+            collection: Collection::new(),
+            is_online: Arc::new(Mutex::new((
+                Instant::now(),
+                Err(MeliError::new("Account is uninitialised.")),
+            ))),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct NntpType {
+    is_subscribed: Arc<IsSubscribedFn>,
+    connection: Arc<FutureMutex<NntpConnection>>,
+    server_conf: NntpServerConf,
+    uid_store: Arc<UIDStore>,
+    can_create_flags: Arc<Mutex<bool>>,
+}
+
+impl MailBackend for NntpType {
+    fn capabilities(&self) -> MailBackendCapabilities {
+        let mut extensions = self
+            .uid_store
+            .capabilities
+            .lock()
+            .unwrap()
+            .iter()
+            .map(|c| {
+                (
+                    c.to_string(),
+                    MailBackendExtensionStatus::Unsupported { comment: None },
+                )
+            })
+            .collect::<Vec<(String, MailBackendExtensionStatus)>>();
+        let NntpExtensionUse {
+            #[cfg(feature = "deflate_compression")]
+            deflate,
+        } = self.server_conf.extension_use;
+        {
+            for (name, status) in extensions.iter_mut() {
+                match name.as_str() {
+                    "COMPRESS DEFLATE" => {
+                        #[cfg(feature = "deflate_compression")]
+                        {
+                            if deflate {
+                                *status = MailBackendExtensionStatus::Enabled { comment: None };
+                            } else {
+                                *status = MailBackendExtensionStatus::Supported {
+                                    comment: Some("Disabled by user configuration"),
+                                };
+                            }
+                        }
+                        #[cfg(not(feature = "deflate_compression"))]
+                        {
+                            *status = MailBackendExtensionStatus::Unsupported {
+                                comment: Some("melib not compiled with DEFLATE."),
+                            };
+                        }
+                    }
+                    _ => {
+                        if SUPPORTED_CAPABILITIES.contains(&name.as_str()) {
+                            *status = MailBackendExtensionStatus::Enabled { comment: None };
+                        }
+                    }
+                }
+            }
+        }
+        extensions.sort_by(|a, b| a.0.cmp(&b.0));
+        MailBackendCapabilities {
+            is_async: true,
+            is_remote: true,
+            supports_search: false,
+            extensions: Some(extensions),
+            supports_tags: false,
+            supports_submission: false,
+        }
+    }
+
+    fn fetch(
+        &mut self,
+        mailbox_hash: MailboxHash,
+    ) -> Result<Pin<Box<dyn Stream<Item = Result<Vec<Envelope>>> + Send + 'static>>> {
+        let mut state = FetchState {
+            mailbox_hash,
+            uid_store: self.uid_store.clone(),
+            connection: self.connection.clone(),
+            high_low_total: None,
+        };
+        Ok(Box::pin(async_stream::try_stream! {
+            {
+                let f = &state.uid_store.mailboxes.lock().await[&state.mailbox_hash];
+                f.exists.lock().unwrap().clear();
+                f.unseen.lock().unwrap().clear();
+            };
+            loop {
+                if let Some(ret) = state.fetch_envs().await? {
+                    yield ret;
+                    continue;
+                }
+                break;
+            }
+        }))
+    }
+
+    fn refresh(&mut self, _mailbox_hash: MailboxHash) -> ResultFuture<()> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn mailboxes(&self) -> ResultFuture<HashMap<MailboxHash, Mailbox>> {
+        let uid_store = self.uid_store.clone();
+        let connection = self.connection.clone();
+        Ok(Box::pin(async move {
+            NntpType::nntp_mailboxes(&connection).await?;
+            let mailboxes_lck = uid_store.mailboxes.lock().await;
+            let ret = mailboxes_lck
+                .iter()
+                .map(|(h, f)| (*h, Box::new(Clone::clone(f)) as Mailbox))
+                .collect();
+            Ok(ret)
+        }))
+    }
+
+    fn is_online(&self) -> ResultFuture<()> {
+        let connection = self.connection.clone();
+        Ok(Box::pin(async move {
+            match timeout(Some(Duration::from_secs(60 * 16)), connection.lock()).await {
+                Ok(mut conn) => {
+                    debug!("is_online");
+                    match debug!(timeout(Some(Duration::from_secs(60 * 16)), conn.connect()).await)
+                    {
+                        Ok(Ok(())) => Ok(()),
+                        Err(err) | Ok(Err(err)) => {
+                            conn.stream = Err(err.clone());
+                            debug!(conn.connect().await)
+                        }
+                    }
+                }
+                Err(err) => Err(err),
+            }
+        }))
+    }
+
+    fn watch(&self) -> ResultFuture<()> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn operation(&self, env_hash: EnvelopeHash) -> Result<Box<dyn BackendOp>> {
+        let (uid, mailbox_hash) = if let Some(v) =
+            self.uid_store.hash_index.lock().unwrap().get(&env_hash)
+        {
+            *v
+        } else {
+            return Err(MeliError::new(
+                    "Message not found in local cache, it might have been deleted before you requested it."
+                ));
+        };
+        Ok(Box::new(NntpOp::new(
+            uid,
+            mailbox_hash,
+            self.connection.clone(),
+            self.uid_store.clone(),
+        )))
+    }
+
+    fn save(
+        &self,
+        _bytes: Vec<u8>,
+        _mailbox_hash: MailboxHash,
+        _flags: Option<Flag>,
+    ) -> ResultFuture<()> {
+        Err(MeliError::new("NNTP doesn't support saving."))
+    }
+
+    fn copy_messages(
+        &mut self,
+        _env_hashes: EnvelopeHashBatch,
+        _source_mailbox_hash: MailboxHash,
+        _destination_mailbox_hash: MailboxHash,
+        _move_: bool,
+    ) -> ResultFuture<()> {
+        Err(MeliError::new("NNTP doesn't support copying/moving."))
+    }
+
+    fn set_flags(
+        &mut self,
+        _env_hashes: EnvelopeHashBatch,
+        _mailbox_hash: MailboxHash,
+        _flags: SmallVec<[(std::result::Result<Flag, String>, bool); 8]>,
+    ) -> ResultFuture<()> {
+        Err(MeliError::new("NNTP doesn't support flags."))
+    }
+
+    fn delete_messages(
+        &mut self,
+        _env_hashes: EnvelopeHashBatch,
+        _mailbox_hash: MailboxHash,
+    ) -> ResultFuture<()> {
+        Err(MeliError::new("NNTP doesn't support deletion."))
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn as_any_mut(&mut self) -> &mut dyn Any {
+        self
+    }
+
+    fn collection(&self) -> Collection {
+        self.uid_store.collection.clone()
+    }
+
+    fn create_mailbox(
+        &mut self,
+        _path: String,
+    ) -> ResultFuture<(MailboxHash, HashMap<MailboxHash, Mailbox>)> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn delete_mailbox(
+        &mut self,
+        _mailbox_hash: MailboxHash,
+    ) -> ResultFuture<HashMap<MailboxHash, Mailbox>> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn set_mailbox_subscription(
+        &mut self,
+        _mailbox_hash: MailboxHash,
+        _new_val: bool,
+    ) -> ResultFuture<()> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn rename_mailbox(
+        &mut self,
+        _mailbox_hash: MailboxHash,
+        _new_path: String,
+    ) -> ResultFuture<Mailbox> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn set_mailbox_permissions(
+        &mut self,
+        _mailbox_hash: MailboxHash,
+        _val: crate::backends::MailboxPermissions,
+    ) -> ResultFuture<()> {
+        Err(MeliError::new("Unimplemented."))
+    }
+
+    fn search(
+        &self,
+        _query: crate::search::Query,
+        _mailbox_hash: Option<MailboxHash>,
+    ) -> ResultFuture<SmallVec<[EnvelopeHash; 512]>> {
+        Err(MeliError::new("Unimplemented."))
+    }
+}
+
+impl NntpType {
+    pub fn new(
+        s: &AccountSettings,
+        is_subscribed: Box<dyn Fn(&str) -> bool + Send + Sync>,
+        event_consumer: BackendEventConsumer,
+    ) -> Result<Box<dyn MailBackend>> {
+        let server_hostname = get_conf_val!(s["server_hostname"])?;
+        /*let server_username = get_conf_val!(s["server_username"], "")?;
+        let server_password = if !s.extra.contains_key("server_password_command") {
+            get_conf_val!(s["server_password"], "")?.to_string()
+        } else {
+            let invocation = get_conf_val!(s["server_password_command"])?;
+            let output = std::process::Command::new("sh")
+                .args(&["-c", invocation])
+                .stdin(std::process::Stdio::piped())
+                .stdout(std::process::Stdio::piped())
+                .stderr(std::process::Stdio::piped())
+                .output()?;
+            if !output.status.success() {
+                return Err(MeliError::new(format!(
+                    "({}) server_password_command `{}` returned {}: {}",
+                    s.name,
+                    get_conf_val!(s["server_password_command"])?,
+                    output.status,
+                    String::from_utf8_lossy(&output.stderr)
+                )));
+            }
+            std::str::from_utf8(&output.stdout)?.trim_end().to_string()
+        };
+        */
+        let server_port = get_conf_val!(s["server_port"], 119)?;
+        let use_tls = get_conf_val!(s["use_tls"], server_port == 563)?;
+        let use_starttls = use_tls && get_conf_val!(s["use_starttls"], !(server_port == 563))?;
+        let danger_accept_invalid_certs: bool =
+            get_conf_val!(s["danger_accept_invalid_certs"], false)?;
+        let require_auth = get_conf_val!(s["require_auth"], true)?;
+        let server_conf = NntpServerConf {
+            server_hostname: server_hostname.to_string(),
+            server_username: if require_auth {
+                get_conf_val!(s["server_username"])?.to_string()
+            } else {
+                get_conf_val!(s["server_username"], String::new())?
+            },
+            server_password: if require_auth {
+                get_conf_val!(s["server_password"])?.to_string()
+            } else {
+                get_conf_val!(s["server_password"], String::new())?
+            },
+            require_auth,
+            server_port,
+            use_tls,
+            use_starttls,
+            danger_accept_invalid_certs,
+            extension_use: NntpExtensionUse {
+                #[cfg(feature = "deflate_compression")]
+                deflate: get_conf_val!(s["use_deflate"], true)?,
+            },
+        };
+        let account_hash = {
+            let mut hasher = DefaultHasher::new();
+            hasher.write(s.name.as_bytes());
+            hasher.finish()
+        };
+        let account_name = Arc::new(s.name().to_string());
+        let mut mailboxes = HashMap::default();
+        for (k, _f) in s.mailboxes.iter() {
+            let mailbox_hash = get_path_hash!(&k);
+            mailboxes.insert(
+                mailbox_hash,
+                NntpMailbox {
+                    hash: mailbox_hash,
+                    nntp_path: k.to_string(),
+                    high_watermark: Arc::new(Mutex::new(0)),
+                    low_watermark: Arc::new(Mutex::new(0)),
+                    exists: Default::default(),
+                    unseen: Default::default(),
+                },
+            );
+        }
+        if mailboxes.is_empty() {
+            return Err(MeliError::new(format!(
+                "{} has no newsgroups configured.",
+                account_name
+            )));
+        }
+        let uid_store: Arc<UIDStore> = Arc::new(UIDStore {
+            offline_cache: false, //get_conf_val!(s["X_header_caching"], false)?,
+            mailboxes: Arc::new(FutureMutex::new(mailboxes)),
+            ..UIDStore::new(account_hash, account_name, event_consumer)
+        });
+        let connection = NntpConnection::new_connection(&server_conf, uid_store.clone());
+
+        Ok(Box::new(NntpType {
+            server_conf,
+            is_subscribed: Arc::new(IsSubscribedFn(is_subscribed)),
+            can_create_flags: Arc::new(Mutex::new(false)),
+            connection: Arc::new(FutureMutex::new(connection)),
+            uid_store,
+        }))
+    }
+
+    pub async fn nntp_mailboxes(connection: &Arc<FutureMutex<NntpConnection>>) -> Result<()> {
+        let mut res = String::with_capacity(8 * 1024);
+        let mut conn = connection.lock().await;
+        let command = {
+            let mailboxes_lck = conn.uid_store.mailboxes.lock().await;
+            mailboxes_lck
+                .values()
+                .fold("LIST ACTIVE ".to_string(), |mut acc, x| {
+                    if acc.len() != "LIST ACTIVE ".len() {
+                        acc.push(',');
+                    }
+                    acc.push_str(x.name());
+                    acc
+                })
+        };
+        conn.send_command(command.as_bytes()).await?;
+        conn.read_response(&mut res, true, &["215 "])
+            .await
+            .chain_err_summary(|| {
+                format!(
+                    "Could not get newsgroups {}: expected LIST ACTIVE response but got: {}",
+                    &conn.uid_store.account_name, res
+                )
+            })?;
+        debug!(&res);
+        let mut mailboxes_lck = conn.uid_store.mailboxes.lock().await;
+        for l in res.split_rn().skip(1) {
+            let s = l.split_whitespace().collect::<SmallVec<[&str; 4]>>();
+            if s.len() != 3 {
+                continue;
+            }
+            let mailbox_hash = get_path_hash!(&s[0]);
+            mailboxes_lck.entry(mailbox_hash).and_modify(|m| {
+                *m.high_watermark.lock().unwrap() = usize::from_str(s[1]).unwrap_or(0);
+                *m.low_watermark.lock().unwrap() = usize::from_str(s[2]).unwrap_or(0);
+            });
+        }
+        Ok(())
+    }
+
+    pub fn validate_config(s: &AccountSettings) -> Result<()> {
+        get_conf_val!(s["server_hostname"])?;
+        get_conf_val!(s["server_username"], String::new())?;
+        if !s.extra.contains_key("server_password_command") {
+            get_conf_val!(s["server_password"], String::new())?;
+        } else if s.extra.contains_key("server_password") {
+            return Err(MeliError::new(format!(
+                "Configuration error ({}): both server_password and server_password_command are set, cannot choose",
+                s.name.as_str(),
+            )));
+        }
+        let server_port = get_conf_val!(s["server_port"], 119)?;
+        let use_tls = get_conf_val!(s["use_tls"], server_port == 563)?;
+        let use_starttls = get_conf_val!(s["use_starttls"], !(server_port == 563))?;
+        if !use_tls && use_starttls {
+            return Err(MeliError::new(format!(
+                "Configuration error ({}): incompatible use_tls and use_starttls values: use_tls = false, use_starttls = true",
+                s.name.as_str(),
+            )));
+        }
+        #[cfg(feature = "deflate_compression")]
+        get_conf_val!(s["use_deflate"], true)?;
+        #[cfg(not(feature = "deflate_compression"))]
+        if s.extra.contains_key("use_deflate") {
+            return Err(MeliError::new(format!(
+                "Configuration error ({}): setting `use_deflate` is set but this version of meli isn't compiled with DEFLATE support.",
+                s.name.as_str(),
+            )));
+        }
+        get_conf_val!(s["danger_accept_invalid_certs"], false)?;
+        Ok(())
+    }
+
+    pub fn capabilities(&self) -> Vec<String> {
+        self.uid_store
+            .capabilities
+            .lock()
+            .unwrap()
+            .iter()
+            .map(|c| c.clone())
+            .collect::<Vec<String>>()
+    }
+}
+
+struct FetchState {
+    mailbox_hash: MailboxHash,
+    connection: Arc<FutureMutex<NntpConnection>>,
+    uid_store: Arc<UIDStore>,
+    high_low_total: Option<(usize, usize, usize)>,
+}
+
+impl FetchState {
+    async fn fetch_envs(&mut self) -> Result<Option<Vec<Envelope>>> {
+        let FetchState {
+            mailbox_hash,
+            ref connection,
+            ref uid_store,
+            ref mut high_low_total,
+        } = self;
+        let mailbox_hash = *mailbox_hash;
+        let mut res = String::with_capacity(8 * 1024);
+        let mut conn = connection.lock().await;
+        if high_low_total.is_none() {
+            conn.select_group(mailbox_hash, true, &mut res).await?;
+            /*
+             *   Parameters
+                     group     Name of newsgroup
+                     number    Estimated number of articles in the group
+                     low       Reported low water mark
+                     high      Reported high water mark
+            */
+            let s = res.split_whitespace().collect::<SmallVec<[&str; 6]>>();
+            let path = conn.uid_store.mailboxes.lock().await[&mailbox_hash]
+                .name()
+                .to_string();
+            if s.len() != 5 {
+                return Err(MeliError::new(format!(
+                    "{} Could not select newsgroup {}: expected GROUP response but got: {}",
+                    &uid_store.account_name, path, res
+                )));
+            }
+            let total = usize::from_str(&s[1]).unwrap_or(0);
+            let _low = usize::from_str(&s[2]).unwrap_or(0);
+            let high = usize::from_str(&s[3]).unwrap_or(0);
+            *high_low_total = Some((high, _low, total));
+            {
+                let f = &uid_store.mailboxes.lock().await[&mailbox_hash];
+                f.exists.lock().unwrap().set_not_yet_seen(total);
+                f.unseen.lock().unwrap().set_not_yet_seen(total);
+            };
+        }
+        let (high, low, _) = high_low_total.unwrap();
+        if high <= low {
+            return Ok(None);
+        }
+        const CHUNK_SIZE: usize = 100;
+        let new_low = std::cmp::max(low, high.saturating_sub(CHUNK_SIZE));
+        high_low_total.as_mut().unwrap().0 = new_low;
+
+        conn.send_command(format!("OVER {}-{}", new_low, high).as_bytes())
+            .await?;
+        conn.read_response(&mut res, true, command_to_replycodes("OVER"))
+            .await
+            .chain_err_summary(|| {
+                format!(
+                    "{} Could not select newsgroup: expected OVER response but got: {}",
+                    &uid_store.account_name, res
+                )
+            })?;
+        let mut ret = Vec::with_capacity(high - new_low);
+        //hash_index: Arc<Mutex<HashMap<EnvelopeHash, (UID, MailboxHash)>>>,
+        //uid_index: Arc<Mutex<HashMap<(MailboxHash, UID), EnvelopeHash>>>,
+        {
+            let mut hash_index_lck = uid_store.hash_index.lock().unwrap();
+            let mut uid_index_lck = uid_store.uid_index.lock().unwrap();
+            for l in res.split_rn().skip(1) {
+                let (_, (num, env)) = protocol_parser::over_article(&l)?;
+                hash_index_lck.insert(env.hash(), (num, mailbox_hash));
+                uid_index_lck.insert((mailbox_hash, num), env.hash());
+                ret.push(env);
+            }
+        }
+        {
+            let hash_set: BTreeSet<EnvelopeHash> = ret.iter().map(|env| env.hash()).collect();
+            let f = &uid_store.mailboxes.lock().await[&mailbox_hash];
+            f.exists
+                .lock()
+                .unwrap()
+                .insert_existing_set(hash_set.clone());
+            f.unseen.lock().unwrap().insert_existing_set(hash_set);
+        };
+        Ok(Some(ret))
+    }
+}

melib/src/backends/nntp/connection.rs

@@ -0,0 +1,562 @@
+/*
+ * meli - nntp module.
+ *
+ * Copyright 2017 - 2019 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use crate::backends::{BackendMailbox, MailboxHash};
+use crate::connections::{lookup_ipv4, Connection};
+use crate::email::parser::BytesExt;
+use crate::error::*;
+extern crate native_tls;
+use futures::io::{AsyncReadExt, AsyncWriteExt};
+use native_tls::TlsConnector;
+pub use smol::Async as AsyncWrapper;
+use std::collections::HashSet;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::time::Instant;
+
+use super::{Capabilities, NntpServerConf, UIDStore};
+
+#[derive(Debug, Clone, Copy)]
+pub struct NntpExtensionUse {
+    #[cfg(feature = "deflate_compression")]
+    pub deflate: bool,
+}
+
+impl Default for NntpExtensionUse {
+    fn default() -> Self {
+        Self {
+            #[cfg(feature = "deflate_compression")]
+            deflate: true,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct NntpStream {
+    pub stream: AsyncWrapper<Connection>,
+    pub extension_use: NntpExtensionUse,
+    pub current_mailbox: MailboxSelection,
+}
+
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+pub enum MailboxSelection {
+    None,
+    Select(MailboxHash),
+}
+
+impl MailboxSelection {
+    pub fn take(&mut self) -> Self {
+        std::mem::replace(self, MailboxSelection::None)
+    }
+}
+
+async fn try_await(cl: impl Future<Output = Result<()>> + Send) -> Result<()> {
+    cl.await
+}
+
+#[derive(Debug)]
+pub struct NntpConnection {
+    pub stream: Result<NntpStream>,
+    pub server_conf: NntpServerConf,
+    pub uid_store: Arc<UIDStore>,
+}
+
+impl NntpStream {
+    pub async fn new_connection(
+        server_conf: &NntpServerConf,
+    ) -> Result<(Capabilities, NntpStream)> {
+        use std::net::TcpStream;
+        let path = &server_conf.server_hostname;
+
+        let stream = {
+            let addr = lookup_ipv4(path, server_conf.server_port)?;
+            AsyncWrapper::new(Connection::Tcp(
+                TcpStream::connect_timeout(&addr, std::time::Duration::new(16, 0))
+                    .chain_err_kind(crate::error::ErrorKind::Network)?,
+            ))
+            .chain_err_kind(crate::error::ErrorKind::Network)?
+        };
+        let mut res = String::with_capacity(8 * 1024);
+        let mut ret = NntpStream {
+            stream,
+            extension_use: server_conf.extension_use,
+            current_mailbox: MailboxSelection::None,
+        };
+
+        if server_conf.use_tls {
+            let mut connector = TlsConnector::builder();
+            if server_conf.danger_accept_invalid_certs {
+                connector.danger_accept_invalid_certs(true);
+            }
+            let connector = connector
+                .build()
+                .chain_err_kind(crate::error::ErrorKind::Network)?;
+
+            if server_conf.use_starttls {
+                ret.read_response(&mut res, false, &["200 ", "201 "])
+                    .await?;
+                ret.send_command(b"CAPABILITIES").await?;
+                ret.read_response(&mut res, true, command_to_replycodes("CAPABILITIES"))
+                    .await?;
+                if !res.starts_with("101 ") {
+                    return Err(MeliError::new(format!(
+                        "Could not connect to {}: expected CAPABILITIES response but got:{}",
+                        &server_conf.server_hostname, res
+                    )));
+                }
+                let capabilities: Vec<&str> = res.lines().skip(1).collect();
+
+                if !capabilities
+                    .iter()
+                    .any(|cap| cap.eq_ignore_ascii_case("VERSION 2"))
+                {
+                    return Err(MeliError::new(format!(
+                        "Could not connect to {}: server is not NNTP VERSION 2 compliant",
+                        &server_conf.server_hostname
+                    )));
+                }
+                if !capabilities
+                    .iter()
+                    .any(|cap| cap.eq_ignore_ascii_case("STARTTLS"))
+                {
+                    return Err(MeliError::new(format!(
+                        "Could not connect to {}: server does not support STARTTLS",
+                        &server_conf.server_hostname
+                    )));
+                }
+                ret.stream
+                    .write_all(b"STARTTLS\r\n")
+                    .await
+                    .chain_err_kind(crate::error::ErrorKind::Network)?;
+                ret.stream
+                    .flush()
+                    .await
+                    .chain_err_kind(crate::error::ErrorKind::Network)?;
+                ret.read_response(&mut res, false, command_to_replycodes("STARTTLS"))
+                    .await?;
+                if !res.starts_with("382 ") {
+                    return Err(MeliError::new(format!(
+                        "Could not connect to {}: could not begin TLS negotiation, got: {}",
+                        &server_conf.server_hostname, res
+                    )));
+                }
+            }
+
+            {
+                // FIXME: This is blocking
+                let socket = ret
+                    .stream
+                    .into_inner()
+                    .chain_err_kind(crate::error::ErrorKind::Network)?;
+                let mut conn_result = connector.connect(path, socket);
+                if let Err(native_tls::HandshakeError::WouldBlock(midhandshake_stream)) =
+                    conn_result
+                {
+                    let mut midhandshake_stream = Some(midhandshake_stream);
+                    loop {
+                        match midhandshake_stream.take().unwrap().handshake() {
+                            Ok(r) => {
+                                conn_result = Ok(r);
+                                break;
+                            }
+                            Err(native_tls::HandshakeError::WouldBlock(stream)) => {
+                                midhandshake_stream = Some(stream);
+                            }
+                            p => {
+                                p.chain_err_kind(crate::error::ErrorKind::Network)?;
+                            }
+                        }
+                    }
+                }
+                ret.stream = AsyncWrapper::new(Connection::Tls(
+                    conn_result.chain_err_kind(crate::error::ErrorKind::Network)?,
+                ))
+                .chain_err_summary(|| format!("Could not initiate TLS negotiation to {}.", path))
+                .chain_err_kind(crate::error::ErrorKind::Network)?;
+            }
+        } else {
+            ret.read_response(&mut res, false, &["200 ", "201 "])
+                .await?;
+        }
+        //ret.send_command(
+        //    format!(
+        //        "LOGIN \"{}\" \"{}\"",
+        //        &server_conf.server_username, &server_conf.server_password
+        //    )
+        //    .as_bytes(),
+        //)
+        //.await?;
+        if let Err(err) = ret
+            .stream
+            .get_ref()
+            .set_keepalive(Some(std::time::Duration::new(60 * 9, 0)))
+        {
+            crate::log(
+                format!("Could not set TCP keepalive in NNTP connection: {}", err),
+                crate::LoggingLevel::WARN,
+            );
+        }
+
+        ret.send_command(b"CAPABILITIES").await?;
+        ret.read_response(&mut res, true, command_to_replycodes("CAPABILITIES"))
+            .await?;
+        if !res.starts_with("101 ") {
+            return Err(MeliError::new(format!(
+                "Could not connect to {}: expected CAPABILITIES response but got:{}",
+                &server_conf.server_hostname, res
+            )));
+        }
+        let capabilities: HashSet<String> = res.lines().skip(1).map(|l| l.to_string()).collect();
+        if !capabilities
+            .iter()
+            .any(|cap| cap.eq_ignore_ascii_case("VERSION 2"))
+        {
+            return Err(MeliError::new(format!(
+                "Could not connect to {}: server is not NNTP compliant",
+                &server_conf.server_hostname
+            )));
+        }
+
+        if server_conf.require_auth {
+            if capabilities.iter().any(|c| c.starts_with("AUTHINFO USER")) {
+                ret.send_command(
+                    format!("AUTHINFO USER {}", server_conf.server_username).as_bytes(),
+                )
+                .await?;
+                ret.read_response(&mut res, false, command_to_replycodes("AUTHINFO USER"))
+                    .await
+                    .chain_err_summary(|| format!("Authentication state error: {}", res))
+                    .chain_err_kind(ErrorKind::Authentication)?;
+                if res.starts_with("381 ") {
+                    ret.send_command(
+                        format!("AUTHINFO PASS {}", server_conf.server_password).as_bytes(),
+                    )
+                    .await?;
+                    ret.read_response(&mut res, false, command_to_replycodes("AUTHINFO PASS"))
+                        .await
+                        .chain_err_summary(|| format!("Authentication state error: {}", res))
+                        .chain_err_kind(ErrorKind::Authentication)?;
+                }
+            } else {
+                return Err(MeliError::new(format!(
+                    "Could not connect: no supported auth mechanisms in server capabilities: {:?}",
+                    capabilities
+                ))
+                .set_err_kind(ErrorKind::Authentication));
+            }
+        }
+
+        #[cfg(feature = "deflate_compression")]
+        if capabilities.contains("COMPRESS DEFLATE") && ret.extension_use.deflate {
+            ret.send_command(b"COMPRESS DEFLATE").await?;
+            ret.read_response(&mut res, false, command_to_replycodes("COMPRESS DEFLATE"))
+                .await
+                .chain_err_summary(|| {
+                    format!(
+                        "Could not use COMPRESS DEFLATE in account `{}`: server replied with `{}`",
+                        server_conf.server_hostname, res
+                    )
+                })?;
+            let NntpStream {
+                stream,
+                extension_use,
+                current_mailbox,
+            } = ret;
+            let stream = stream.into_inner()?;
+            return Ok((
+                capabilities,
+                NntpStream {
+                    stream: AsyncWrapper::new(stream.deflate())?,
+                    extension_use,
+                    current_mailbox,
+                },
+            ));
+        }
+
+        Ok((capabilities, ret))
+    }
+
+    pub async fn read_response(
+        &mut self,
+        ret: &mut String,
+        is_multiline: bool,
+        expected_reply_code: &[&str],
+    ) -> Result<()> {
+        self.read_lines(ret, is_multiline, expected_reply_code)
+            .await?;
+        Ok(())
+    }
+
+    pub async fn read_lines(
+        &mut self,
+        ret: &mut String,
+        is_multiline: bool,
+        expected_reply_code: &[&str],
+    ) -> Result<()> {
+        let mut buf: Vec<u8> = vec![0; Connection::IO_BUF_SIZE];
+        ret.clear();
+        let mut last_line_idx: usize = 0;
+        loop {
+            match self.stream.read(&mut buf).await {
+                Ok(0) => break,
+                Ok(b) => {
+                    ret.push_str(unsafe { std::str::from_utf8_unchecked(&buf[0..b]) });
+                    if ret.len() > 4 {
+                        if ret.starts_with("205 ") {
+                            return Err(MeliError::new(format!("Disconnected: {}", ret)));
+                        } else if ret.starts_with("501 ") || ret.starts_with("500 ") {
+                            return Err(MeliError::new(format!("Syntax error: {}", ret)));
+                        } else if ret.starts_with("403 ") {
+                            return Err(MeliError::new(format!("Internal error: {}", ret)));
+                        } else if ret.starts_with("502 ")
+                            || ret.starts_with("480 ")
+                            || ret.starts_with("483 ")
+                            || ret.starts_with("401 ")
+                        {
+                            return Err(MeliError::new(format!("Connection state error: {}", ret))
+                                .set_err_kind(ErrorKind::Authentication));
+                        } else if !expected_reply_code.iter().any(|r| ret.starts_with(r)) {
+                            return Err(MeliError::new(format!("Unexpected reply code: {}", ret)));
+                        }
+                    }
+                    if let Some(mut pos) = ret[last_line_idx..].rfind("\r\n") {
+                        if !is_multiline {
+                            break;
+                        } else if let Some(pos) = ret.find("\r\n.\r\n") {
+                            ret.replace_range(pos + "\r\n".len()..pos + "\r\n.\r\n".len(), "");
+                            break;
+                        }
+                        if let Some(prev_line) =
+                            ret[last_line_idx..pos + last_line_idx].rfind("\r\n")
+                        {
+                            last_line_idx += prev_line + "\r\n".len();
+                            pos -= prev_line + "\r\n".len();
+                        }
+                        last_line_idx += pos + "\r\n".len();
+                    }
+                }
+                Err(e) => {
+                    return Err(MeliError::from(e).set_err_kind(crate::error::ErrorKind::Network));
+                }
+            }
+        }
+        //debug!("returning nntp response:\n{:?}", &ret);
+        Ok(())
+    }
+
+    pub async fn send_command(&mut self, command: &[u8]) -> Result<()> {
+        if let Err(err) = try_await(async move {
+            let command = command.trim();
+            self.stream.write_all(command).await?;
+            self.stream.write_all(b"\r\n").await?;
+            self.stream.flush().await?;
+            debug!("sent: {}", unsafe {
+                std::str::from_utf8_unchecked(command)
+            });
+            Ok(())
+        })
+        .await
+        {
+            debug!("stream send_command err {:?}", err);
+            Err(err.set_err_kind(crate::error::ErrorKind::Network))
+        } else {
+            Ok(())
+        }
+    }
+
+    pub async fn send_multiline_data_block(&mut self, data: &str) -> Result<()> {
+        if let Err(err) = try_await(async move {
+            for l in data.lines() {
+                if l.starts_with('.') {
+                    self.stream.write_all(b".").await?;
+                }
+                self.stream.write_all(l.as_bytes()).await?;
+                self.stream.write_all(b"\r\n").await?;
+            }
+            self.stream.write_all(b".\r\n").await?;
+            self.stream.flush().await?;
+            debug!("sent data block {} bytes", data.len());
+            Ok(())
+        })
+        .await
+        {
+            debug!("stream send_multiline_data_block err {:?}", err);
+            Err(err.set_err_kind(crate::error::ErrorKind::Network))
+        } else {
+            Ok(())
+        }
+    }
+}
+
+impl NntpConnection {
+    pub fn new_connection(
+        server_conf: &NntpServerConf,
+        uid_store: Arc<UIDStore>,
+    ) -> NntpConnection {
+        NntpConnection {
+            stream: Err(MeliError::new("Offline".to_string())),
+            server_conf: server_conf.clone(),
+            uid_store,
+        }
+    }
+
+    pub fn connect<'a>(&'a mut self) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'a>> {
+        Box::pin(async move {
+            if let (instant, ref mut status @ Ok(())) = *self.uid_store.is_online.lock().unwrap() {
+                if Instant::now().duration_since(instant) >= std::time::Duration::new(60 * 30, 0) {
+                    *status = Err(MeliError::new("Connection timed out"));
+                    self.stream = Err(MeliError::new("Connection timed out"));
+                }
+            }
+            if self.stream.is_ok() {
+                self.uid_store.is_online.lock().unwrap().0 = Instant::now();
+                return Ok(());
+            }
+            let new_stream = NntpStream::new_connection(&self.server_conf).await;
+            if let Err(err) = new_stream.as_ref() {
+                *self.uid_store.is_online.lock().unwrap() = (Instant::now(), Err(err.clone()));
+            } else {
+                *self.uid_store.is_online.lock().unwrap() = (Instant::now(), Ok(()));
+            }
+            let (capabilities, stream) = new_stream?;
+            self.stream = Ok(stream);
+            *self.uid_store.capabilities.lock().unwrap() = capabilities;
+            Ok(())
+        })
+    }
+
+    pub fn read_response<'a>(
+        &'a mut self,
+        ret: &'a mut String,
+        is_multiline: bool,
+        expected_reply_code: &'static [&str],
+    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'a>> {
+        Box::pin(async move {
+            ret.clear();
+            self.stream
+                .as_mut()?
+                .read_response(ret, is_multiline, expected_reply_code)
+                .await
+        })
+    }
+
+    pub async fn read_lines(
+        &mut self,
+        ret: &mut String,
+        is_multiline: bool,
+        expected_reply_code: &[&str],
+    ) -> Result<()> {
+        self.stream
+            .as_mut()?
+            .read_lines(ret, is_multiline, expected_reply_code)
+            .await?;
+        Ok(())
+    }
+
+    pub async fn send_command(&mut self, command: &[u8]) -> Result<()> {
+        if let Err(err) =
+            try_await(async { self.stream.as_mut()?.send_command(command).await }).await
+        {
+            self.stream = Err(err.clone());
+            debug!(err.kind);
+            if err.kind.is_network() {
+                debug!(self.connect().await)?;
+            }
+            Err(err)
+        } else {
+            Ok(())
+        }
+    }
+
+    pub fn add_refresh_event(&mut self, ev: crate::backends::RefreshEvent) {
+        (self.uid_store.event_consumer)(
+            self.uid_store.account_hash,
+            crate::backends::BackendEvent::Refresh(ev),
+        );
+    }
+
+    pub async fn select_group(
+        &mut self,
+        mailbox_hash: MailboxHash,
+        force: bool,
+        res: &mut String,
+    ) -> Result<()> {
+        if !force {
+            match self.stream.as_ref()?.current_mailbox {
+                MailboxSelection::Select(m) if m == mailbox_hash => return Ok(()),
+                _ => {}
+            }
+        }
+        let path = self.uid_store.mailboxes.lock().await[&mailbox_hash]
+            .name()
+            .to_string();
+        self.send_command(format!("GROUP {}", path).as_bytes())
+            .await?;
+        self.read_response(res, false, command_to_replycodes("GROUP"))
+            .await
+            .chain_err_summary(|| {
+                format!(
+                    "{} Could not select newsgroup {}: expected GROUP response but got: {}",
+                    &self.uid_store.account_name, path, res
+                )
+            })?;
+        self.stream.as_mut()?.current_mailbox = MailboxSelection::Select(mailbox_hash);
+        Ok(())
+    }
+
+    pub async fn send_multiline_data_block(&mut self, message: &str) -> Result<()> {
+        self.stream
+            .as_mut()?
+            .send_multiline_data_block(message)
+            .await
+    }
+}
+
+pub fn command_to_replycodes(c: &str) -> &'static [&'static str] {
+    if c.starts_with("OVER") {
+        &["224 "]
+    } else if c.starts_with("LIST") {
+        &["215 "]
+    } else if c.starts_with("POST") {
+        &["340 "]
+    } else if c.starts_with("STARTTLS") {
+        &["382 "]
+    } else if c.starts_with("GROUP") {
+        &["211 "]
+    } else if c.starts_with("CAPABILITIES") {
+        &["101 "]
+    } else if c.starts_with("ARTICLE") {
+        &["220 "]
+    } else if c.starts_with("DATE") {
+        &["111 "]
+    } else if c.starts_with("NEWNEWS") {
+        &["230 "]
+    } else if c.starts_with("AUTHINFO USER") {
+        &["281 ", "381 "]
+    } else if c.starts_with("AUTHINFO PASS") {
+        &["281 "]
+    } else if c.starts_with("COMPRESS DEFLATE") {
+        &["206 "]
+    } else {
+        &[]
+    }
+}

melib/src/backends/nntp/mailbox.rs

@@ -0,0 +1,97 @@
+/*
+ * meli - nntp module.
+ *
+ * Copyright 2019 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+use crate::backends::{
+    BackendMailbox, LazyCountSet, Mailbox, MailboxHash, MailboxPermissions, SpecialUsageMailbox,
+};
+use crate::error::*;
+use std::sync::{Arc, Mutex};
+
+#[derive(Debug, Default, Clone)]
+pub struct NntpMailbox {
+    pub(super) hash: MailboxHash,
+    pub(super) nntp_path: String,
+
+    pub high_watermark: Arc<Mutex<usize>>,
+    pub low_watermark: Arc<Mutex<usize>>,
+
+    pub exists: Arc<Mutex<LazyCountSet>>,
+    pub unseen: Arc<Mutex<LazyCountSet>>,
+}
+
+impl NntpMailbox {
+    pub fn nntp_path(&self) -> &str {
+        &self.nntp_path
+    }
+}
+
+impl BackendMailbox for NntpMailbox {
+    fn hash(&self) -> MailboxHash {
+        self.hash
+    }
+
+    fn name(&self) -> &str {
+        &self.nntp_path
+    }
+
+    fn path(&self) -> &str {
+        &self.nntp_path
+    }
+
+    fn change_name(&mut self, s: &str) {
+        self.nntp_path = s.to_string();
+    }
+
+    fn children(&self) -> &[MailboxHash] {
+        &[]
+    }
+
+    fn clone(&self) -> Mailbox {
+        Box::new(std::clone::Clone::clone(self))
+    }
+
+    fn special_usage(&self) -> SpecialUsageMailbox {
+        SpecialUsageMailbox::default()
+    }
+
+    fn parent(&self) -> Option<MailboxHash> {
+        None
+    }
+
+    fn permissions(&self) -> MailboxPermissions {
+        MailboxPermissions::default()
+    }
+
+    fn is_subscribed(&self) -> bool {
+        true
+    }
+
+    fn set_is_subscribed(&mut self, _new_val: bool) -> Result<()> {
+        Err(MeliError::new("Cannot set subscription in NNTP."))
+    }
+
+    fn set_special_usage(&mut self, _new_val: SpecialUsageMailbox) -> Result<()> {
+        Err(MeliError::new("Cannot set special usage in NNTP."))
+    }
+
+    fn count(&self) -> Result<(usize, usize)> {
+        Ok((self.unseen.lock()?.len(), self.exists.lock()?.len()))
+    }
+}

melib/src/backends/nntp/operations.rs

@@ -0,0 +1,93 @@
+/*
+ * meli - nntp module.
+ *
+ * Copyright 2017 - 2019 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+
+use crate::backends::*;
+use crate::email::*;
+use crate::error::MeliError;
+use std::sync::Arc;
+
+/// `BackendOp` implementor for Nntp
+#[derive(Debug, Clone)]
+pub struct NntpOp {
+    uid: usize,
+    mailbox_hash: MailboxHash,
+    connection: Arc<FutureMutex<NntpConnection>>,
+    uid_store: Arc<UIDStore>,
+}
+
+impl NntpOp {
+    pub fn new(
+        uid: usize,
+        mailbox_hash: MailboxHash,
+        connection: Arc<FutureMutex<NntpConnection>>,
+        uid_store: Arc<UIDStore>,
+    ) -> Self {
+        NntpOp {
+            uid,
+            connection,
+            mailbox_hash,
+            uid_store,
+        }
+    }
+}
+
+impl BackendOp for NntpOp {
+    fn as_bytes(&mut self) -> ResultFuture<Vec<u8>> {
+        let mailbox_hash = self.mailbox_hash;
+        let uid = self.uid;
+        let uid_store = self.uid_store.clone();
+        let connection = self.connection.clone();
+        Ok(Box::pin(async move {
+            let mut res = String::with_capacity(8 * 1024);
+            let mut conn = connection.lock().await;
+            let path = uid_store.mailboxes.lock().await[&mailbox_hash]
+                .name()
+                .to_string();
+            conn.send_command(format!("GROUP {}", path).as_bytes())
+                .await?;
+            conn.read_response(&mut res, false, &["211 "]).await?;
+            if !res.starts_with("211 ") {
+                return Err(MeliError::new(format!(
+                    "{} Could not select newsgroup {}: expected GROUP response but got: {}",
+                    &uid_store.account_name, path, res
+                )));
+            }
+            conn.send_command(format!("ARTICLE {}", uid).as_bytes())
+                .await?;
+            conn.read_response(&mut res, true, &["220 "]).await?;
+            if !res.starts_with("220 ") {
+                return Err(MeliError::new(format!(
+                    "{} Could not select article {}: expected ARTICLE response but got: {}",
+                    &uid_store.account_name, path, res
+                )));
+            }
+            let pos = res.find("\r\n").unwrap_or(0) + 2;
+
+            Ok(res.as_bytes()[pos..].to_vec())
+        }))
+    }
+
+    fn fetch_flags(&self) -> ResultFuture<Flag> {
+        Ok(Box::pin(async move { Ok(Flag::default()) }))
+    }
+}

melib/src/backends/nntp/protocol_parser.rs

@@ -0,0 +1,156 @@
+/*
+ * meli - melib crate.
+ *
+ * Copyright 2017-2020 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+use crate::email::parser::IResult;
+use nom::{
+    bytes::complete::{is_not, tag},
+    combinator::opt,
+};
+use std::str::FromStr;
+
+pub struct NntpLineIterator<'a> {
+    slice: &'a str,
+}
+
+impl<'a> std::iter::DoubleEndedIterator for NntpLineIterator<'a> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        if self.slice.is_empty() {
+            None
+        } else if let Some(pos) = self.slice.rfind("\r\n") {
+            if self.slice[..pos].is_empty() {
+                self.slice = &self.slice[..pos];
+                None
+            } else if let Some(prev_pos) = self.slice[..pos].rfind("\r\n") {
+                let ret = &self.slice[prev_pos + 2..pos + 2];
+                self.slice = &self.slice[..prev_pos + 2];
+                Some(ret)
+            } else {
+                let ret = self.slice;
+                self.slice = &self.slice[ret.len()..];
+                Some(ret)
+            }
+        } else {
+            let ret = self.slice;
+            self.slice = &self.slice[ret.len()..];
+            Some(ret)
+        }
+    }
+}
+
+impl<'a> Iterator for NntpLineIterator<'a> {
+    type Item = &'a str;
+
+    fn next(&mut self) -> Option<&'a str> {
+        if self.slice.is_empty() {
+            None
+        } else if let Some(pos) = self.slice.find("\r\n") {
+            let ret = &self.slice[..pos + 2];
+            self.slice = &self.slice[pos + 2..];
+            Some(ret)
+        } else {
+            let ret = self.slice;
+            self.slice = &self.slice[ret.len()..];
+            Some(ret)
+        }
+    }
+}
+
+pub trait NntpLineSplit {
+    fn split_rn(&self) -> NntpLineIterator;
+}
+
+impl NntpLineSplit for str {
+    fn split_rn(&self) -> NntpLineIterator {
+        NntpLineIterator { slice: self }
+    }
+}
+
+pub fn over_article(input: &str) -> IResult<&str, (UID, Envelope)> {
+    /*
+    "0" or article number (see below)
+    Subject header content
+    From header content
+    Date header content
+    Message-ID header content
+    References header content
+    :bytes metadata item
+    :lines metadata item
+    */
+    let (input, num) = is_not("\t")(input)?;
+    let (input, _) = tag("\t")(input)?;
+    let (input, subject) = opt(is_not("\t"))(input)?;
+    let (input, _) = tag("\t")(input)?;
+    let (input, from) = opt(is_not("\t"))(input)?;
+    let (input, _) = tag("\t")(input)?;
+    let (input, date) = opt(is_not("\t"))(input)?;
+    let (input, _) = tag("\t")(input)?;
+    let (input, message_id) = opt(is_not("\t"))(input)?;
+    let (input, _) = tag("\t")(input)?;
+    let (input, references) = opt(is_not("\t"))(input)?;
+    let (input, _) = tag("\t")(input)?;
+    let (input, _bytes) = opt(is_not("\t"))(input)?;
+    let (input, _) = tag("\t")(input)?;
+    let (input, _lines) = opt(is_not("\t\r\n"))(input)?;
+    let (input, _other_headers) = opt(is_not("\r\n"))(input)?;
+    let (input, _) = tag("\r\n")(input)?;
+    Ok((
+        input,
+        ({
+            let env_hash = {
+                let mut hasher = DefaultHasher::new();
+                hasher.write(num.as_bytes());
+                hasher.write(message_id.unwrap_or_default().as_bytes());
+                hasher.finish()
+            };
+            let mut env = Envelope::new(env_hash);
+            if let Some(date) = date {
+                env.set_date(date.as_bytes());
+                if let Ok(d) =
+                    crate::email::parser::dates::rfc5322_date(env.date_as_str().as_bytes())
+                {
+                    env.set_datetime(d);
+                }
+            }
+
+            if let Some(subject) = subject {
+                env.set_subject(subject.into());
+            }
+
+            if let Some(from) = from {
+                if let Ok((_, from)) =
+                    crate::email::parser::address::rfc2822address_list(from.as_bytes())
+                {
+                    env.set_from(from);
+                }
+            }
+
+            if let Some(references) = references {
+                env.set_references(references.as_bytes());
+            }
+
+            if let Some(message_id) = message_id {
+                env.set_message_id(message_id.as_bytes());
+            }
+            (usize::from_str(num).unwrap(), env)
+        }),
+    ))
+}

melib/src/backends/notmuch/message.rs

@@ -0,0 +1,288 @@
+/*
+ * melib - notmuch backend
+ *
+ * Copyright 2020 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+use crate::thread::{ThreadHash, ThreadNode, ThreadNodeHash};
+
+#[derive(Clone)]
+pub struct Message<'m> {
+    pub lib: Arc<libloading::Library>,
+    pub message: *mut notmuch_message_t,
+    pub is_from_thread: bool,
+    pub _ph: std::marker::PhantomData<&'m notmuch_message_t>,
+}
+
+impl<'m> Message<'m> {
+    pub fn find_message(db: &'m DbConnection, msg_id: &CStr) -> Result<Message<'m>> {
+        let mut message: *mut notmuch_message_t = std::ptr::null_mut();
+        let lib = db.lib.clone();
+        unsafe {
+            call!(lib, notmuch_database_find_message)(
+                *db.inner.read().unwrap(),
+                msg_id.as_ptr(),
+                &mut message as *mut _,
+            )
+        };
+        if message.is_null() {
+            return Err(MeliError::new(format!(
+                "Message with message id {:?} not found in notmuch database.",
+                msg_id
+            )));
+        }
+        Ok(Message {
+            lib,
+            message,
+            is_from_thread: false,
+            _ph: std::marker::PhantomData,
+        })
+    }
+
+    pub fn env_hash(&self) -> EnvelopeHash {
+        let msg_id = unsafe { call!(self.lib, notmuch_message_get_message_id)(self.message) };
+        let c_str = unsafe { CStr::from_ptr(msg_id) };
+        {
+            let mut hasher = DefaultHasher::default();
+            c_str.hash(&mut hasher);
+            hasher.finish()
+        }
+    }
+
+    pub fn header(&self, header: &CStr) -> Option<&[u8]> {
+        let header_val =
+            unsafe { call!(self.lib, notmuch_message_get_header)(self.message, header.as_ptr()) };
+        if header_val.is_null() {
+            None
+        } else {
+            Some(unsafe { CStr::from_ptr(header_val).to_bytes() })
+        }
+    }
+
+    pub fn msg_id(&self) -> &[u8] {
+        let c_str = self.msg_id_cstr();
+        c_str.to_bytes()
+    }
+
+    pub fn msg_id_cstr(&self) -> &CStr {
+        let msg_id = unsafe { call!(self.lib, notmuch_message_get_message_id)(self.message) };
+        unsafe { CStr::from_ptr(msg_id) }
+    }
+
+    pub fn date(&self) -> crate::datetime::UnixTimestamp {
+        (unsafe { call!(self.lib, notmuch_message_get_date)(self.message) }) as u64
+    }
+
+    pub fn into_envelope(
+        self,
+        index: &RwLock<HashMap<EnvelopeHash, CString>>,
+        tag_index: &RwLock<BTreeMap<u64, String>>,
+    ) -> Envelope {
+        let env_hash = self.env_hash();
+        let mut env = Envelope::new(env_hash);
+        index
+            .write()
+            .unwrap()
+            .insert(env_hash, self.msg_id_cstr().into());
+        let mut tag_lock = tag_index.write().unwrap();
+        let (flags, tags) = TagIterator::new(&self).collect_flags_and_tags();
+        for tag in tags {
+            let mut hasher = DefaultHasher::new();
+            hasher.write(tag.as_bytes());
+            let num = hasher.finish();
+            if !tag_lock.contains_key(&num) {
+                tag_lock.insert(num, tag);
+            }
+            env.labels_mut().push(num);
+        }
+        unsafe {
+            use crate::email::parser::address::rfc2822address_list;
+            env.set_message_id(self.msg_id())
+                .set_date(
+                    self.header(CStr::from_bytes_with_nul_unchecked(b"Date\0"))
+                        .unwrap_or_default(),
+                )
+                .set_from(
+                    rfc2822address_list(
+                        self.header(CStr::from_bytes_with_nul_unchecked(b"From\0"))
+                            .unwrap_or_default(),
+                    )
+                    .map(|(_, v)| v)
+                    .unwrap_or_default(),
+                )
+                .set_to(
+                    rfc2822address_list(
+                        self.header(CStr::from_bytes_with_nul_unchecked(b"To\0"))
+                            .unwrap_or_default(),
+                    )
+                    .map(|(_, v)| v)
+                    .unwrap_or_default(),
+                )
+                .set_cc(
+                    rfc2822address_list(
+                        self.header(CStr::from_bytes_with_nul_unchecked(b"Cc\0"))
+                            .unwrap_or_default(),
+                    )
+                    .map(|(_, v)| v)
+                    .unwrap_or_default(),
+                )
+                .set_bcc(
+                    rfc2822address_list(
+                        self.header(CStr::from_bytes_with_nul_unchecked(b"Bcc\0"))
+                            .unwrap_or_default(),
+                    )
+                    .map(|(_, v)| v)
+                    .unwrap_or_default()
+                    .to_vec(),
+                )
+                .set_subject(
+                    self.header(CStr::from_bytes_with_nul_unchecked(b"Subject\0"))
+                        .unwrap_or_default()
+                        .to_vec(),
+                )
+                .set_references(
+                    self.header(CStr::from_bytes_with_nul_unchecked(b"References\0"))
+                        .unwrap_or_default(),
+                )
+                .set_in_reply_to(
+                    self.header(CStr::from_bytes_with_nul_unchecked(b"In-Reply-To\0"))
+                        .unwrap_or_default(),
+                )
+                .set_datetime(self.date())
+                .set_flags(flags);
+        }
+        env
+    }
+
+    pub fn replies_iter(&self) -> Option<MessageIterator> {
+        if self.is_from_thread {
+            let messages = unsafe { call!(self.lib, notmuch_message_get_replies)(self.message) };
+            if messages.is_null() {
+                None
+            } else {
+                Some(MessageIterator {
+                    lib: self.lib.clone(),
+                    messages,
+                    _ph: std::marker::PhantomData,
+                    is_from_thread: true,
+                })
+            }
+        } else {
+            None
+        }
+    }
+
+    pub fn into_thread_node(&self) -> (ThreadNodeHash, ThreadNode) {
+        (
+            ThreadNodeHash::from(self.msg_id()),
+            ThreadNode {
+                message: Some(self.env_hash()),
+                parent: None,
+                other_mailbox: false,
+                children: vec![],
+                date: self.date(),
+                show_subject: true,
+                group: ThreadHash::new(),
+                unseen: false,
+            },
+        )
+    }
+
+    pub fn add_tag(&self, tag: &CStr) -> Result<()> {
+        if let Err(err) = unsafe {
+            try_call!(
+                self.lib,
+                call!(self.lib, notmuch_message_add_tag)(self.message, tag.as_ptr())
+            )
+        } {
+            return Err(MeliError::new("Could not set tag.").set_source(Some(Arc::new(err))));
+        }
+        Ok(())
+    }
+
+    pub fn remove_tag(&self, tag: &CStr) -> Result<()> {
+        if let Err(err) = unsafe {
+            try_call!(
+                self.lib,
+                call!(self.lib, notmuch_message_remove_tag)(self.message, tag.as_ptr())
+            )
+        } {
+            return Err(MeliError::new("Could not set tag.").set_source(Some(Arc::new(err))));
+        }
+        Ok(())
+    }
+
+    pub fn tags(&'m self) -> TagIterator<'m> {
+        TagIterator::new(self)
+    }
+
+    pub fn tags_to_maildir_flags(&self) -> Result<()> {
+        if let Err(err) = unsafe {
+            try_call!(
+                self.lib,
+                call!(self.lib, notmuch_message_tags_to_maildir_flags)(self.message)
+            )
+        } {
+            return Err(MeliError::new("Could not set flags.").set_source(Some(Arc::new(err))));
+        }
+        Ok(())
+    }
+
+    pub fn get_filename(&self) -> &OsStr {
+        let fs_path = unsafe { call!(self.lib, notmuch_message_get_filename)(self.message) };
+        let c_str = unsafe { CStr::from_ptr(fs_path) };
+        &OsStr::from_bytes(c_str.to_bytes())
+    }
+}
+
+impl Drop for Message<'_> {
+    fn drop(&mut self) {
+        unsafe { call!(self.lib, notmuch_message_destroy)(self.message) };
+    }
+}
+
+pub struct MessageIterator<'query> {
+    pub lib: Arc<libloading::Library>,
+    pub messages: *mut notmuch_messages_t,
+    pub is_from_thread: bool,
+    pub _ph: std::marker::PhantomData<*const Query<'query>>,
+}
+
+impl<'q> Iterator for MessageIterator<'q> {
+    type Item = Message<'q>;
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.messages.is_null() {
+            None
+        } else if unsafe { call!(self.lib, notmuch_messages_valid)(self.messages) } == 1 {
+            let message = unsafe { call!(self.lib, notmuch_messages_get)(self.messages) };
+            unsafe {
+                call!(self.lib, notmuch_messages_move_to_next)(self.messages);
+            }
+            Some(Message {
+                lib: self.lib.clone(),
+                message,
+                is_from_thread: self.is_from_thread,
+                _ph: std::marker::PhantomData,
+            })
+        } else {
+            self.messages = std::ptr::null_mut();
+            None
+        }
+    }
+}

melib/src/backends/notmuch/tags.rs

@@ -0,0 +1,129 @@
+/*
+ * melib - notmuch backend
+ *
+ * Copyright 2020 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use super::*;
+
+pub struct TagIterator<'m> {
+    pub tags: *mut notmuch_tags_t,
+    pub message: &'m Message<'m>,
+}
+
+impl Drop for TagIterator<'_> {
+    fn drop(&mut self) {
+        unsafe { call!(self.message.lib, notmuch_tags_destroy)(self.tags) };
+    }
+}
+
+impl<'m> TagIterator<'m> {
+    pub fn new(message: &'m Message<'m>) -> TagIterator<'m> {
+        TagIterator {
+            tags: unsafe { call!(message.lib, notmuch_message_get_tags)(message.message) },
+            message,
+        }
+    }
+
+    pub fn collect_flags_and_tags(self) -> (Flag, Vec<String>) {
+        fn flags(path: &CStr) -> Flag {
+            let mut flag = Flag::default();
+            let mut ptr = path.to_bytes().len().saturating_sub(1);
+            let mut is_valid = true;
+            while !path.to_bytes()[..ptr + 1].ends_with(b":2,") {
+                match path.to_bytes()[ptr] {
+                    b'D' => flag |= Flag::DRAFT,
+                    b'F' => flag |= Flag::FLAGGED,
+                    b'P' => flag |= Flag::PASSED,
+                    b'R' => flag |= Flag::REPLIED,
+                    b'S' => flag |= Flag::SEEN,
+                    b'T' => flag |= Flag::TRASHED,
+                    _ => {
+                        is_valid = false;
+                        break;
+                    }
+                }
+                if ptr == 0 {
+                    is_valid = false;
+                    break;
+                }
+                ptr -= 1;
+            }
+
+            if !is_valid {
+                return Flag::default();
+            }
+
+            flag
+        }
+        let fs_path =
+            unsafe { call!(self.message.lib, notmuch_message_get_filename)(self.message.message) };
+        let c_str = unsafe { CStr::from_ptr(fs_path) };
+
+        let tags = self.collect::<Vec<&CStr>>();
+        let mut flag = Flag::default();
+        let mut vec = vec![];
+        for t in tags {
+            match t.to_bytes() {
+                b"draft" => {
+                    flag.set(Flag::DRAFT, true);
+                }
+                b"flagged" => {
+                    flag.set(Flag::FLAGGED, true);
+                }
+                b"passed" => {
+                    flag.set(Flag::PASSED, true);
+                }
+                b"replied" => {
+                    flag.set(Flag::REPLIED, true);
+                }
+                b"unread" => {
+                    flag.set(Flag::SEEN, false);
+                }
+                b"trashed" => {
+                    flag.set(Flag::TRASHED, true);
+                }
+                _other => {
+                    vec.push(t.to_string_lossy().into_owned());
+                }
+            }
+        }
+
+        (flag | flags(c_str), vec)
+    }
+}
+
+impl<'m> Iterator for TagIterator<'m> {
+    type Item = &'m CStr;
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.tags.is_null() {
+            None
+        } else if unsafe { call!(self.message.lib, notmuch_tags_valid)(self.tags) } == 1 {
+            let ret = Some(unsafe {
+                CStr::from_ptr(call!(self.message.lib, notmuch_tags_get)(self.tags))
+            });
+            unsafe {
+                call!(self.message.lib, notmuch_tags_move_to_next)(self.tags);
+            }
+            ret
+        } else {
+            self.tags = std::ptr::null_mut();
+            None
+        }
+    }
+}

melib/src/backends/notmuch/thread.rs

@@ -0,0 +1,88 @@
+/*
+ * melib - notmuch backend
+ *
+ * Copyright 2020 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+use super::*;
+use crate::thread::ThreadHash;
+
+pub struct Thread<'query> {
+    pub lib: Arc<libloading::Library>,
+    pub ptr: *mut notmuch_thread_t,
+    pub _ph: std::marker::PhantomData<*const Query<'query>>,
+}
+
+impl<'q> Thread<'q> {
+    pub fn id(&self) -> ThreadHash {
+        let thread_id = unsafe { call!(self.lib, notmuch_thread_get_thread_id)(self.ptr) };
+        let c_str = unsafe { CStr::from_ptr(thread_id) };
+        ThreadHash::from(c_str.to_bytes())
+    }
+
+    pub fn date(&self) -> crate::datetime::UnixTimestamp {
+        (unsafe { call!(self.lib, notmuch_thread_get_newest_date)(self.ptr) }) as u64
+    }
+
+    pub fn len(&self) -> usize {
+        (unsafe { call!(self.lib, notmuch_thread_get_total_messages)(self.ptr) }) as usize
+    }
+
+    pub fn iter(&'q self) -> MessageIterator<'q> {
+        let ptr = unsafe { call!(self.lib, notmuch_thread_get_messages)(self.ptr) };
+        MessageIterator {
+            lib: self.lib.clone(),
+            messages: ptr,
+            is_from_thread: true,
+            _ph: std::marker::PhantomData,
+        }
+    }
+}
+
+impl Drop for Thread<'_> {
+    fn drop(&mut self) {
+        unsafe { call!(self.lib, notmuch_thread_destroy)(self.ptr) }
+    }
+}
+
+pub struct ThreadsIterator<'query> {
+    pub lib: Arc<libloading::Library>,
+    pub threads: *mut notmuch_threads_t,
+    pub _ph: std::marker::PhantomData<*const Query<'query>>,
+}
+
+impl<'q> Iterator for ThreadsIterator<'q> {
+    type Item = Thread<'q>;
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.threads.is_null() {
+            None
+        } else if unsafe { call!(self.lib, notmuch_threads_valid)(self.threads) } == 1 {
+            let thread = unsafe { call!(self.lib, notmuch_threads_get)(self.threads) };
+            unsafe {
+                call!(self.lib, notmuch_threads_move_to_next)(self.threads);
+            }
+            Some(Thread {
+                lib: self.lib.clone(),
+                ptr: thread,
+                _ph: std::marker::PhantomData,
+            })
+        } else {
+            self.threads = std::ptr::null_mut();
+            None
+        }
+    }
+}

melib/src/collection.rs

@@ -1,79 +1,154 @@
+/*
+ * meli - melib crate.
+ *
+ * Copyright 2017-2020 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
 use super::*;
-use crate::backends::FolderHash;
-use std::collections::BTreeMap;
-use std::fs;
-use std::io;
+use crate::backends::MailboxHash;
+use smallvec::SmallVec;
 use std::ops::{Deref, DerefMut};
+use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
 
-use fnv::FnvHashMap;
+use std::collections::{BTreeMap, HashMap, HashSet};
 
-#[derive(Debug, Clone, Deserialize, Default, Serialize)]
-pub struct Collection {
-    pub envelopes: FnvHashMap<EnvelopeHash, Envelope>,
-    message_ids: FnvHashMap<Vec<u8>, EnvelopeHash>,
-    date_index: BTreeMap<UnixTimestamp, EnvelopeHash>,
-    subject_index: Option<BTreeMap<String, EnvelopeHash>>,
-    pub threads: FnvHashMap<FolderHash, Threads>,
-    sent_folder: Option<FolderHash>,
+pub struct EnvelopeRef<'g> {
+    guard: RwLockReadGuard<'g, HashMap<EnvelopeHash, Envelope>>,
+    env_hash: EnvelopeHash,
 }
 
-impl Drop for Collection {
-    fn drop(&mut self) {
-        let cache_dir: xdg::BaseDirectories =
-            xdg::BaseDirectories::with_profile("meli", "threads".to_string()).unwrap();
-        if let Ok(cached) = cache_dir.place_cache_file("threads") {
-            /* place result in cache directory */
-            let f = match fs::File::create(cached) {
-                Ok(f) => f,
-                Err(e) => {
-                    panic!("{}", e);
-                }
-            };
-            let writer = io::BufWriter::new(f);
-            bincode::serialize_into(writer, &self.threads).unwrap();
-        }
+impl Deref for EnvelopeRef<'_> {
+    type Target = Envelope;
+
+    fn deref(&self) -> &Envelope {
+        self.guard.get(&self.env_hash).unwrap()
     }
 }
 
-impl Collection {
-    pub fn new(envelopes: FnvHashMap<EnvelopeHash, Envelope>) -> Collection {
-        let date_index = BTreeMap::new();
-        let subject_index = None;
-        let message_ids = FnvHashMap::with_capacity_and_hasher(2048, Default::default());
+pub struct EnvelopeRefMut<'g> {
+    guard: RwLockWriteGuard<'g, HashMap<EnvelopeHash, Envelope>>,
+    env_hash: EnvelopeHash,
+}
 
-        /* Scrap caching for now. When a cached threads file is loaded, we must remove/rehash the
-         * thread nodes that shouldn't exist anymore (e.g. because their file moved from /new to
-         * /cur, or it was deleted).
-         */
-        let threads = FnvHashMap::with_capacity_and_hasher(16, Default::default());
+impl Deref for EnvelopeRefMut<'_> {
+    type Target = Envelope;
+
+    fn deref(&self) -> &Envelope {
+        self.guard.get(&self.env_hash).unwrap()
+    }
+}
+
+impl DerefMut for EnvelopeRefMut<'_> {
+    fn deref_mut(&mut self) -> &mut Envelope {
+        self.guard.get_mut(&self.env_hash).unwrap()
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Collection {
+    pub envelopes: Arc<RwLock<HashMap<EnvelopeHash, Envelope>>>,
+    pub message_id_index: Arc<RwLock<HashMap<Vec<u8>, EnvelopeHash>>>,
+    pub threads: Arc<RwLock<HashMap<MailboxHash, Threads>>>,
+    pub sent_mailbox: Arc<RwLock<Option<MailboxHash>>>,
+    pub mailboxes: Arc<RwLock<HashMap<MailboxHash, HashSet<EnvelopeHash>>>>,
+    pub tag_index: Arc<RwLock<BTreeMap<u64, String>>>,
+}
+
+impl Default for Collection {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/*
+impl Drop for Collection {
+    fn drop(&mut self) {
+            let cache_dir: xdg::BaseDirectories =
+                xdg::BaseDirectories::with_profile("meli", "threads".to_string()).unwrap();
+            if let Ok(cached) = cache_dir.place_cache_file("threads") {
+                /* place result in cache directory */
+                let f = match fs::File::create(cached) {
+                    Ok(f) => f,
+                    Err(e) => {
+                        panic!("{}", e);
+                    }
+                };
+                let writer = io::BufWriter::new(f);
+                let _ = bincode::Options::serialize_into(
+                    bincode::config::DefaultOptions::new(),
+                    writer,
+                    &self.thread,
+                );
+            }
+    }
+}
+*/
+
+impl Collection {
+    pub fn new() -> Collection {
+        let message_id_index = Arc::new(RwLock::new(HashMap::with_capacity_and_hasher(
+            16,
+            Default::default(),
+        )));
+        let threads = Arc::new(RwLock::new(HashMap::with_capacity_and_hasher(
+            16,
+            Default::default(),
+        )));
+        let mailboxes = Arc::new(RwLock::new(HashMap::with_capacity_and_hasher(
+            16,
+            Default::default(),
+        )));
 
         Collection {
-            envelopes,
-            date_index,
-            message_ids,
-            subject_index,
+            envelopes: Arc::new(RwLock::new(Default::default())),
+            tag_index: Arc::new(RwLock::new(BTreeMap::default())),
+            message_id_index,
             threads,
-            sent_folder: None,
+            mailboxes,
+            sent_mailbox: Arc::new(RwLock::new(None)),
         }
     }
 
     pub fn len(&self) -> usize {
-        self.envelopes.len()
+        self.envelopes.read().unwrap().len()
     }
 
     pub fn is_empty(&self) -> bool {
-        self.envelopes.is_empty()
+        self.envelopes.read().unwrap().is_empty()
     }
 
-    pub fn remove(&mut self, envelope_hash: EnvelopeHash, folder_hash: FolderHash) {
+    pub fn remove(&self, envelope_hash: EnvelopeHash, mailbox_hash: MailboxHash) {
         debug!("DEBUG: Removing {}", envelope_hash);
-        self.envelopes.remove(&envelope_hash);
-        self.threads
-            .entry(folder_hash)
+        self.envelopes.write().unwrap().remove(&envelope_hash);
+        self.mailboxes
+            .write()
+            .unwrap()
+            .entry(mailbox_hash)
+            .and_modify(|m| {
+                m.remove(&envelope_hash);
+            });
+        let mut threads_lck = self.threads.write().unwrap();
+        threads_lck
+            .entry(mailbox_hash)
             .or_default()
             .remove(envelope_hash);
-        for (h, t) in self.threads.iter_mut() {
-            if *h == folder_hash {
+        for (h, t) in threads_lck.iter_mut() {
+            if *h == mailbox_hash {
                 continue;
             }
             t.remove(envelope_hash);
@@ -81,81 +156,83 @@ impl Collection {
     }
 
     pub fn rename(
-        &mut self,
+        &self,
         old_hash: EnvelopeHash,
         new_hash: EnvelopeHash,
-        folder_hash: FolderHash,
-    ) {
-        if !self.envelopes.contains_key(&old_hash) {
-            return;
+        mailbox_hash: MailboxHash,
+    ) -> bool {
+        if !self.envelopes.read().unwrap().contains_key(&old_hash) {
+            return false;
         }
-        let mut env = self.envelopes.remove(&old_hash).unwrap();
-        env.set_hash(new_hash);
-        self.message_ids
-            .insert(env.message_id().raw().to_vec(), new_hash);
-        self.envelopes.insert(new_hash, env);
+        let mut envelope = self.envelopes.write().unwrap().remove(&old_hash).unwrap();
+        self.mailboxes
+            .write()
+            .unwrap()
+            .entry(mailbox_hash)
+            .and_modify(|m| {
+                m.remove(&old_hash);
+                m.insert(new_hash);
+            });
+        envelope.set_hash(new_hash);
+        self.envelopes.write().unwrap().insert(new_hash, envelope);
+        let mut threads_lck = self.threads.write().unwrap();
         {
-            if self
-                .threads
-                .entry(folder_hash)
+            if threads_lck
+                .entry(mailbox_hash)
                 .or_default()
                 .update_envelope(&self.envelopes, old_hash, new_hash)
                 .is_ok()
             {
-                return;
+                return true;
             }
         }
         /* envelope is not in threads, so insert it */
-        self.threads
-            .entry(folder_hash)
+        threads_lck
+            .entry(mailbox_hash)
             .or_default()
-            .insert(&mut self.envelopes, new_hash);
-        for (h, t) in self.threads.iter_mut() {
-            if *h == folder_hash {
+            .insert(&self.envelopes, new_hash);
+        for (h, t) in threads_lck.iter_mut() {
+            if *h == mailbox_hash {
                 continue;
             }
             t.update_envelope(&self.envelopes, old_hash, new_hash)
                 .ok()
                 .take();
         }
+        true
     }
 
-    /// Merge new Mailbox to collection and update threads.
-    /// Returns a list of already existing folders whose threads were updated
+    /// Merge new mailbox to collection and update threads.
+    /// Returns a list of already existing mailboxs whose threads were updated
     pub fn merge(
-        &mut self,
-        mut new_envelopes: FnvHashMap<EnvelopeHash, Envelope>,
-        folder_hash: FolderHash,
-        mailbox: &mut Mailbox,
-        sent_folder: Option<FolderHash>,
-    ) -> Option<StackVec<FolderHash>> {
-        self.sent_folder = sent_folder;
-        new_envelopes.retain(|&h, e| {
-            if self.message_ids.contains_key(e.message_id().raw()) {
-                /* skip duplicates until a better way to handle them is found. */
-                //FIXME
-                mailbox.remove(h);
-                false
-            } else {
-                self.message_ids.insert(e.message_id().raw().to_vec(), h);
-                true
-            }
-        });
+        &self,
+        mut new_envelopes: HashMap<EnvelopeHash, Envelope>,
+        mailbox_hash: MailboxHash,
+        sent_mailbox: Option<MailboxHash>,
+    ) -> Option<SmallVec<[MailboxHash; 8]>> {
+        *self.sent_mailbox.write().unwrap() = sent_mailbox;
 
-        let &mut Collection {
-            ref mut threads,
-            ref mut envelopes,
-            ref sent_folder,
+        let Collection {
+            ref threads,
+            ref envelopes,
+            ref mailboxes,
+            ref sent_mailbox,
             ..
         } = self;
 
-        if !threads.contains_key(&folder_hash) {
-            threads.insert(folder_hash, Threads::new(&mut new_envelopes));
+        let mut threads_lck = threads.write().unwrap();
+        let mut mailboxes_lck = mailboxes.write().unwrap();
+        if !threads_lck.contains_key(&mailbox_hash) {
+            threads_lck.insert(mailbox_hash, Threads::new(new_envelopes.len()));
+            mailboxes_lck.insert(mailbox_hash, new_envelopes.keys().cloned().collect());
             for (h, e) in new_envelopes {
-                envelopes.insert(h, e);
+                envelopes.write().unwrap().insert(h, e);
             }
         } else {
-            threads.entry(folder_hash).and_modify(|t| {
+            mailboxes_lck.entry(mailbox_hash).and_modify(|m| {
+                m.extend(new_envelopes.keys().cloned());
+            });
+            threads_lck.entry(mailbox_hash).and_modify(|t| {
                 let mut ordered_hash_set =
                     new_envelopes.keys().cloned().collect::<Vec<EnvelopeHash>>();
                 ordered_hash_set.sort_by(|a, b| {
@@ -165,60 +242,80 @@ impl Collection {
                         .unwrap()
                 });
                 for h in ordered_hash_set {
-                    envelopes.insert(h, new_envelopes.remove(&h).unwrap());
+                    envelopes
+                        .write()
+                        .unwrap()
+                        .insert(h, new_envelopes.remove(&h).unwrap());
                     t.insert(envelopes, h);
                 }
             });
         }
 
-        let mut ret = StackVec::new();
-        let keys = threads.keys().cloned().collect::<Vec<FolderHash>>();
+        let mut ret = SmallVec::new();
+        let keys = threads_lck.keys().cloned().collect::<Vec<MailboxHash>>();
         for t_fh in keys {
-            if t_fh == folder_hash {
+            if t_fh == mailbox_hash {
                 continue;
             }
-            if sent_folder.map(|f| f == folder_hash).unwrap_or(false) {
-                let mut ordered_hash_set = threads[&folder_hash]
+            if sent_mailbox
+                .read()
+                .unwrap()
+                .map(|f| f == mailbox_hash)
+                .unwrap_or(false)
+            {
+                let envelopes_lck = envelopes.read().unwrap();
+                let mut ordered_hash_set = threads_lck[&mailbox_hash]
                     .hash_set
                     .iter()
                     .cloned()
                     .collect::<Vec<EnvelopeHash>>();
                 ordered_hash_set.sort_by(|a, b| {
-                    envelopes[a]
+                    envelopes_lck[a]
                         .date()
-                        .partial_cmp(&envelopes[b].date())
+                        .partial_cmp(&envelopes_lck[b].date())
                         .unwrap()
                 });
+                drop(envelopes_lck);
                 let mut updated = false;
                 for h in ordered_hash_set {
-                    updated |= threads.entry(t_fh).or_default().insert_reply(envelopes, h);
+                    updated |= threads_lck
+                        .entry(t_fh)
+                        .or_default()
+                        .insert_reply(envelopes, h);
                 }
                 if updated {
                     ret.push(t_fh);
                 }
                 continue;
             }
-            if sent_folder.map(|f| f == t_fh).unwrap_or(false) {
-                let mut ordered_hash_set = threads[&t_fh]
+            if sent_mailbox
+                .read()
+                .unwrap()
+                .map(|f| f == t_fh)
+                .unwrap_or(false)
+            {
+                let envelopes_lck = envelopes.read().unwrap();
+                let mut ordered_hash_set = threads_lck[&t_fh]
                     .hash_set
                     .iter()
                     .cloned()
                     .collect::<Vec<EnvelopeHash>>();
                 ordered_hash_set.sort_by(|a, b| {
-                    envelopes[a]
+                    envelopes_lck[a]
                         .date()
-                        .partial_cmp(&envelopes[b].date())
+                        .partial_cmp(&envelopes_lck[b].date())
                         .unwrap()
                 });
+                drop(envelopes_lck);
                 let mut updated = false;
                 for h in ordered_hash_set {
-                    updated |= threads
-                        .entry(folder_hash)
+                    updated |= threads_lck
+                        .entry(mailbox_hash)
                         .or_default()
                         .insert_reply(envelopes, h);
                 }
                 if updated {
-                    ret.push(folder_hash);
+                    ret.push(mailbox_hash);
                 }
             }
         }
@@ -230,27 +327,39 @@ impl Collection {
     }
 
     pub fn update(
-        &mut self,
+        &self,
         old_hash: EnvelopeHash,
         mut envelope: Envelope,
-        folder_hash: FolderHash,
+        mailbox_hash: MailboxHash,
     ) {
-        let old_env = self.envelopes.remove(&old_hash).unwrap();
+        let old_env = self.envelopes.write().unwrap().remove(&old_hash).unwrap();
         envelope.set_thread(old_env.thread());
         let new_hash = envelope.hash();
-        self.message_ids
-            .insert(envelope.message_id().raw().to_vec(), new_hash);
-        self.envelopes.insert(new_hash, envelope);
-        if self.sent_folder.map(|f| f == folder_hash).unwrap_or(false) {
-            for (_, t) in self.threads.iter_mut() {
+        self.mailboxes
+            .write()
+            .unwrap()
+            .entry(mailbox_hash)
+            .and_modify(|m| {
+                m.remove(&old_hash);
+                m.insert(new_hash);
+            });
+        self.envelopes.write().unwrap().insert(new_hash, envelope);
+        let mut threads_lck = self.threads.write().unwrap();
+        if self
+            .sent_mailbox
+            .read()
+            .unwrap()
+            .map(|f| f == mailbox_hash)
+            .unwrap_or(false)
+        {
+            for (_, t) in threads_lck.iter_mut() {
                 t.update_envelope(&self.envelopes, old_hash, new_hash)
                     .unwrap_or(());
             }
         }
         {
-            if self
-                .threads
-                .entry(folder_hash)
+            if threads_lck
+                .entry(mailbox_hash)
                 .or_default()
                 .update_envelope(&self.envelopes, old_hash, new_hash)
                 .is_ok()
@@ -259,12 +368,12 @@ impl Collection {
             }
         }
         /* envelope is not in threads, so insert it */
-        self.threads
-            .entry(folder_hash)
+        threads_lck
+            .entry(mailbox_hash)
             .or_default()
-            .insert(&mut self.envelopes, new_hash);
-        for (h, t) in self.threads.iter_mut() {
-            if *h == folder_hash {
+            .insert(&self.envelopes, new_hash);
+        for (h, t) in threads_lck.iter_mut() {
+            if *h == mailbox_hash {
                 continue;
             }
             t.update_envelope(&self.envelopes, old_hash, new_hash)
@@ -273,42 +382,128 @@ impl Collection {
         }
     }
 
-    pub fn insert(&mut self, envelope: Envelope, folder_hash: FolderHash) -> &Envelope {
-        let hash = envelope.hash();
-        self.message_ids
-            .insert(envelope.message_id().raw().to_vec(), hash);
-        self.envelopes.insert(hash, envelope);
-        if !self
-            .threads
-            .entry(folder_hash)
-            .or_default()
-            .insert_reply(&mut self.envelopes, hash)
+    pub fn update_flags(&self, env_hash: EnvelopeHash, mailbox_hash: MailboxHash) {
+        let mut threads_lck = self.threads.write().unwrap();
+        if self
+            .sent_mailbox
+            .read()
+            .unwrap()
+            .map(|f| f == mailbox_hash)
+            .unwrap_or(false)
         {
-            self.threads
-                .entry(folder_hash)
+            for (_, t) in threads_lck.iter_mut() {
+                t.update_envelope(&self.envelopes, env_hash, env_hash)
+                    .unwrap_or(());
+            }
+        }
+        {
+            if threads_lck
+                .entry(mailbox_hash)
                 .or_default()
-                .insert(&mut self.envelopes, hash);
+                .update_envelope(&self.envelopes, env_hash, env_hash)
+                .is_ok()
+            {
+                return;
+            }
+        }
+        /* envelope is not in threads, so insert it */
+        threads_lck
+            .entry(mailbox_hash)
+            .or_default()
+            .insert(&self.envelopes, env_hash);
+        for (h, t) in threads_lck.iter_mut() {
+            if *h == mailbox_hash {
+                continue;
+            }
+            t.update_envelope(&self.envelopes, env_hash, env_hash)
+                .ok()
+                .take();
         }
-        &self.envelopes[&hash]
     }
-    pub fn insert_reply(&mut self, env_hash: EnvelopeHash) {
-        debug_assert!(self.envelopes.contains_key(&env_hash));
-        for (_, t) in self.threads.iter_mut() {
-            t.insert_reply(&mut self.envelopes, env_hash);
+
+    pub fn insert(&self, envelope: Envelope, mailbox_hash: MailboxHash) -> bool {
+        let hash = envelope.hash();
+        self.mailboxes
+            .write()
+            .unwrap()
+            .entry(mailbox_hash)
+            .and_modify(|m| {
+                m.insert(hash);
+            });
+        self.envelopes.write().unwrap().insert(hash, envelope);
+        self.threads
+            .write()
+            .unwrap()
+            .entry(mailbox_hash)
+            .or_default()
+            .insert(&self.envelopes, hash);
+        if self
+            .sent_mailbox
+            .read()
+            .unwrap()
+            .map(|f| f == mailbox_hash)
+            .unwrap_or(false)
+        {
+            self.insert_reply(hash);
+        }
+        false
+    }
+
+    pub fn insert_reply(&self, env_hash: EnvelopeHash) {
+        debug_assert!(self.envelopes.read().unwrap().contains_key(&env_hash));
+        for (_, t) in self.threads.write().unwrap().iter_mut() {
+            t.insert_reply(&self.envelopes, env_hash);
+        }
+    }
+
+    pub fn get_env(&'_ self, env_hash: EnvelopeHash) -> EnvelopeRef<'_> {
+        let guard: RwLockReadGuard<'_, _> = self.envelopes.read().unwrap();
+        EnvelopeRef { guard, env_hash }
+    }
+
+    pub fn get_env_mut(&'_ self, env_hash: EnvelopeHash) -> EnvelopeRefMut<'_> {
+        let guard = self.envelopes.write().unwrap();
+        EnvelopeRefMut { guard, env_hash }
+    }
+
+    pub fn get_threads(&'_ self, hash: MailboxHash) -> RwRef<'_, MailboxHash, Threads> {
+        let guard = self.threads.read().unwrap();
+        RwRef { guard, hash }
+    }
+
+    pub fn get_mailbox(
+        &'_ self,
+        hash: MailboxHash,
+    ) -> RwRef<'_, MailboxHash, HashSet<EnvelopeHash>> {
+        let guard = self.mailboxes.read().unwrap();
+        RwRef { guard, hash }
+    }
+
+    pub fn contains_key(&self, env_hash: &EnvelopeHash) -> bool {
+        self.envelopes.read().unwrap().contains_key(env_hash)
+    }
+
+    pub fn new_mailbox(&self, mailbox_hash: MailboxHash) {
+        let mut mailboxes_lck = self.mailboxes.write().unwrap();
+        if !mailboxes_lck.contains_key(&mailbox_hash) {
+            mailboxes_lck.insert(mailbox_hash, Default::default());
+            self.threads
+                .write()
+                .unwrap()
+                .insert(mailbox_hash, Threads::default());
         }
     }
 }
 
-impl Deref for Collection {
-    type Target = FnvHashMap<EnvelopeHash, Envelope>;
-
-    fn deref(&self) -> &FnvHashMap<EnvelopeHash, Envelope> {
-        &self.envelopes
-    }
+pub struct RwRef<'g, K: std::cmp::Eq + std::hash::Hash, V> {
+    guard: RwLockReadGuard<'g, HashMap<K, V>>,
+    hash: K,
 }
 
-impl DerefMut for Collection {
-    fn deref_mut(&mut self) -> &mut FnvHashMap<EnvelopeHash, Envelope> {
-        &mut self.envelopes
+impl<K: std::cmp::Eq + std::hash::Hash, V> Deref for RwRef<'_, K, V> {
+    type Target = V;
+
+    fn deref(&self) -> &V {
+        self.guard.get(&self.hash).unwrap()
     }
 }

melib/src/conf.rs

@@ -18,17 +18,25 @@
  * You should have received a copy of the GNU General Public License
  * along with meli. If not, see <http://www.gnu.org/licenses/>.
  */
-use std::collections::hash_map::HashMap;
+
+//! Basic mail account configuration to use with [`backends`](./backends/index.html)
+use crate::backends::SpecialUsageMailbox;
+use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use std::collections::HashMap;
 
 #[derive(Debug, Serialize, Default, Clone)]
 pub struct AccountSettings {
     pub name: String,
-    pub root_folder: String,
+    pub root_mailbox: String,
     pub format: String,
     pub identity: String,
     pub read_only: bool,
     pub display_name: Option<String>,
-    pub subscribed_folders: Vec<String>,
+    pub subscribed_mailboxes: Vec<String>,
+    #[serde(default)]
+    pub mailboxes: HashMap<String, MailboxConf>,
+    #[serde(default)]
+    pub manual_refresh: bool,
     #[serde(flatten)]
     pub extra: HashMap<String, String>,
 }
@@ -43,8 +51,8 @@ impl AccountSettings {
     pub fn set_name(&mut self, s: String) {
         self.name = s;
     }
-    pub fn root_folder(&self) -> &str {
-        &self.root_folder
+    pub fn root_mailbox(&self) -> &str {
+        &self.root_mailbox
     }
     pub fn identity(&self) -> &str {
         &self.identity
@@ -56,7 +64,144 @@ impl AccountSettings {
         self.display_name.as_ref()
     }
 
-    pub fn subscribed_folders(&self) -> &Vec<String> {
-        &self.subscribed_folders
+    pub fn subscribed_mailboxes(&self) -> &Vec<String> {
+        &self.subscribed_mailboxes
+    }
+
+    #[cfg(feature = "vcard")]
+    pub fn vcard_folder(&self) -> Option<&str> {
+        self.extra.get("vcard_folder").map(String::as_str)
+    }
+}
+
+#[serde(default)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct MailboxConf {
+    #[serde(alias = "rename")]
+    pub alias: Option<String>,
+    #[serde(default = "false_val")]
+    pub autoload: bool,
+    #[serde(default)]
+    pub subscribe: ToggleFlag,
+    #[serde(default)]
+    pub ignore: ToggleFlag,
+    #[serde(default = "none")]
+    pub usage: Option<SpecialUsageMailbox>,
+    #[serde(flatten)]
+    pub extra: HashMap<String, String>,
+}
+
+impl Default for MailboxConf {
+    fn default() -> Self {
+        MailboxConf {
+            alias: None,
+            autoload: false,
+            subscribe: ToggleFlag::Unset,
+            ignore: ToggleFlag::Unset,
+            usage: None,
+            extra: HashMap::default(),
+        }
+    }
+}
+
+impl MailboxConf {
+    pub fn alias(&self) -> Option<&str> {
+        self.alias.as_deref()
+    }
+}
+
+pub fn true_val() -> bool {
+    true
+}
+
+pub fn false_val() -> bool {
+    false
+}
+
+pub fn none<T>() -> Option<T> {
+    None
+}
+
+#[derive(Copy, Debug, Clone, PartialEq)]
+pub enum ToggleFlag {
+    Unset,
+    InternalVal(bool),
+    False,
+    True,
+    Ask,
+}
+
+impl From<bool> for ToggleFlag {
+    fn from(val: bool) -> Self {
+        if val {
+            ToggleFlag::True
+        } else {
+            ToggleFlag::False
+        }
+    }
+}
+
+impl Default for ToggleFlag {
+    fn default() -> Self {
+        ToggleFlag::Unset
+    }
+}
+
+impl ToggleFlag {
+    pub fn is_unset(&self) -> bool {
+        ToggleFlag::Unset == *self
+    }
+    pub fn is_internal(&self) -> bool {
+        if let ToggleFlag::InternalVal(_) = *self {
+            true
+        } else {
+            false
+        }
+    }
+
+    pub fn is_ask(&self) -> bool {
+        *self == ToggleFlag::Ask
+    }
+
+    pub fn is_false(&self) -> bool {
+        ToggleFlag::False == *self || ToggleFlag::InternalVal(false) == *self
+    }
+
+    pub fn is_true(&self) -> bool {
+        ToggleFlag::True == *self || ToggleFlag::InternalVal(true) == *self
+    }
+}
+
+impl Serialize for ToggleFlag {
+    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        match self {
+            ToggleFlag::Unset | ToggleFlag::InternalVal(_) => serializer.serialize_none(),
+            ToggleFlag::False => serializer.serialize_bool(false),
+            ToggleFlag::True => serializer.serialize_bool(true),
+            ToggleFlag::Ask => serializer.serialize_str("ask"),
+        }
+    }
+}
+
+impl<'de> Deserialize<'de> for ToggleFlag {
+    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = <String>::deserialize(deserializer);
+        Ok(match s? {
+            s if s.eq_ignore_ascii_case("true") => ToggleFlag::True,
+            s if s.eq_ignore_ascii_case("false") => ToggleFlag::False,
+            s if s.eq_ignore_ascii_case("ask") => ToggleFlag::Ask,
+            s => {
+                return Err(serde::de::Error::custom(format!(
+                    r#"expected one of "true", "false", "ask", found `{}`"#,
+                    s
+                )))
+            }
+        })
     }
 }

melib/src/connections.rs

@@ -0,0 +1,297 @@
+/*
+ * meli - melib library
+ *
+ * Copyright 2020  Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//! Connections layers (TCP/fd/TLS/Deflate) to use with remote backends.
+#[cfg(feature = "deflate_compression")]
+use flate2::{read::DeflateDecoder, write::DeflateEncoder, Compression};
+#[cfg(any(target_os = "openbsd", target_os = "netbsd", target_os = "haiku"))]
+use libc::SO_KEEPALIVE as KEEPALIVE_OPTION;
+#[cfg(any(target_os = "macos", target_os = "ios"))]
+use libc::TCP_KEEPALIVE as KEEPALIVE_OPTION;
+#[cfg(not(any(
+    target_os = "openbsd",
+    target_os = "netbsd",
+    target_os = "haiku",
+    target_os = "macos",
+    target_os = "ios"
+)))]
+use libc::TCP_KEEPIDLE as KEEPALIVE_OPTION;
+use libc::{self, c_int, c_void};
+use std::os::unix::io::AsRawFd;
+use std::time::Duration;
+
+#[derive(Debug)]
+pub enum Connection {
+    Tcp(std::net::TcpStream),
+    Fd(std::os::unix::io::RawFd),
+    #[cfg(feature = "tls")]
+    Tls(native_tls::TlsStream<Self>),
+    #[cfg(feature = "deflate_compression")]
+    Deflate {
+        inner: DeflateEncoder<DeflateDecoder<Box<Self>>>,
+    },
+}
+
+use Connection::*;
+
+macro_rules! syscall {
+    ($fn: ident ( $($arg: expr),* $(,)* ) ) => {{
+        #[allow(unused_unsafe)]
+        let res = unsafe { libc::$fn($($arg, )*) };
+        if res == -1 {
+            Err(std::io::Error::last_os_error())
+        } else {
+            Ok(res)
+        }
+    }};
+}
+
+impl Connection {
+    pub const IO_BUF_SIZE: usize = 64 * 1024;
+    #[cfg(feature = "deflate_compression")]
+    pub fn deflate(self) -> Self {
+        Connection::Deflate {
+            inner: DeflateEncoder::new(
+                DeflateDecoder::new_with_buf(Box::new(self), vec![0; Self::IO_BUF_SIZE]),
+                Compression::default(),
+            ),
+        }
+    }
+
+    pub fn set_nonblocking(&self, nonblocking: bool) -> std::io::Result<()> {
+        match self {
+            Tcp(ref t) => t.set_nonblocking(nonblocking),
+            #[cfg(feature = "tls")]
+            Tls(ref t) => t.get_ref().set_nonblocking(nonblocking),
+            Fd(fd) => {
+                //FIXME TODO Review
+                nix::fcntl::fcntl(
+                    *fd,
+                    nix::fcntl::FcntlArg::F_SETFL(if nonblocking {
+                        nix::fcntl::OFlag::O_NONBLOCK
+                    } else {
+                        !nix::fcntl::OFlag::O_NONBLOCK
+                    }),
+                )
+                .map_err(|err| {
+                    std::io::Error::from_raw_os_error(err.as_errno().map(|n| n as i32).unwrap_or(0))
+                })?;
+                Ok(())
+            }
+            #[cfg(feature = "deflate_compression")]
+            Deflate { ref inner, .. } => inner.get_ref().get_ref().set_nonblocking(nonblocking),
+        }
+    }
+
+    pub fn set_read_timeout(&self, dur: Option<Duration>) -> std::io::Result<()> {
+        match self {
+            Tcp(ref t) => t.set_read_timeout(dur),
+            #[cfg(feature = "tls")]
+            Tls(ref t) => t.get_ref().set_read_timeout(dur),
+            Fd(_) => Ok(()),
+            #[cfg(feature = "deflate_compression")]
+            Deflate { ref inner, .. } => inner.get_ref().get_ref().set_read_timeout(dur),
+        }
+    }
+
+    pub fn set_write_timeout(&self, dur: Option<Duration>) -> std::io::Result<()> {
+        match self {
+            Tcp(ref t) => t.set_write_timeout(dur),
+            #[cfg(feature = "tls")]
+            Tls(ref t) => t.get_ref().set_write_timeout(dur),
+            Fd(_) => Ok(()),
+            #[cfg(feature = "deflate_compression")]
+            Deflate { ref inner, .. } => inner.get_ref().get_ref().set_write_timeout(dur),
+        }
+    }
+
+    pub fn keepalive(&self) -> std::io::Result<Option<Duration>> {
+        if let Fd(_) = self {
+            return Ok(None);
+        }
+        unsafe {
+            let raw: c_int = self.getsockopt(libc::SOL_SOCKET, libc::SO_KEEPALIVE)?;
+            if raw == 0 {
+                return Ok(None);
+            }
+            let secs: c_int = self.getsockopt(libc::IPPROTO_TCP, KEEPALIVE_OPTION)?;
+            Ok(Some(Duration::new(secs as u64, 0)))
+        }
+    }
+
+    pub fn set_keepalive(&self, keepalive: Option<Duration>) -> std::io::Result<()> {
+        if let Fd(_) = self {
+            return Ok(());
+        }
+        unsafe {
+            self.setsockopt(
+                libc::SOL_SOCKET,
+                libc::SO_KEEPALIVE,
+                keepalive.is_some() as c_int,
+            )?;
+            if let Some(dur) = keepalive {
+                // TODO: checked cast here
+                self.setsockopt(libc::IPPROTO_TCP, KEEPALIVE_OPTION, dur.as_secs() as c_int)?;
+            }
+            Ok(())
+        }
+    }
+
+    unsafe fn setsockopt<T>(&self, opt: c_int, val: c_int, payload: T) -> std::io::Result<()>
+    where
+        T: Copy,
+    {
+        let payload = &payload as *const T as *const c_void;
+        syscall!(setsockopt(
+            self.as_raw_fd(),
+            opt,
+            val,
+            payload,
+            std::mem::size_of::<T>() as libc::socklen_t,
+        ))?;
+        Ok(())
+    }
+
+    unsafe fn getsockopt<T: Copy>(&self, opt: c_int, val: c_int) -> std::io::Result<T> {
+        let mut slot: T = std::mem::zeroed();
+        let mut len = std::mem::size_of::<T>() as libc::socklen_t;
+        syscall!(getsockopt(
+            self.as_raw_fd(),
+            opt,
+            val,
+            &mut slot as *mut _ as *mut _,
+            &mut len,
+        ))?;
+        assert_eq!(len as usize, std::mem::size_of::<T>());
+        Ok(slot)
+    }
+}
+
+impl Drop for Connection {
+    fn drop(&mut self) {
+        if let Fd(fd) = self {
+            let _ = nix::unistd::close(*fd);
+        }
+    }
+}
+
+impl std::io::Read for Connection {
+    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+        match self {
+            Tcp(ref mut t) => t.read(buf),
+            #[cfg(feature = "tls")]
+            Tls(ref mut t) => t.read(buf),
+            Fd(f) => {
+                use std::os::unix::io::{FromRawFd, IntoRawFd};
+                let mut f = unsafe { std::fs::File::from_raw_fd(*f) };
+                let ret = f.read(buf);
+                let _ = f.into_raw_fd();
+                ret
+            }
+            #[cfg(feature = "deflate_compression")]
+            Deflate { ref mut inner, .. } => inner.read(buf),
+        }
+    }
+}
+
+impl std::io::Write for Connection {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        match self {
+            Tcp(ref mut t) => t.write(buf),
+            #[cfg(feature = "tls")]
+            Tls(ref mut t) => t.write(buf),
+            Fd(f) => {
+                use std::os::unix::io::{FromRawFd, IntoRawFd};
+                let mut f = unsafe { std::fs::File::from_raw_fd(*f) };
+                let ret = f.write(buf);
+                let _ = f.into_raw_fd();
+                ret
+            }
+            #[cfg(feature = "deflate_compression")]
+            Deflate { ref mut inner, .. } => inner.write(buf),
+        }
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        match self {
+            Tcp(ref mut t) => t.flush(),
+            #[cfg(feature = "tls")]
+            Tls(ref mut t) => t.flush(),
+            Fd(f) => {
+                use std::os::unix::io::{FromRawFd, IntoRawFd};
+                let mut f = unsafe { std::fs::File::from_raw_fd(*f) };
+                let ret = f.flush();
+                let _ = f.into_raw_fd();
+                ret
+            }
+            #[cfg(feature = "deflate_compression")]
+            Deflate { ref mut inner, .. } => inner.flush(),
+        }
+    }
+}
+
+impl std::os::unix::io::AsRawFd for Connection {
+    fn as_raw_fd(&self) -> std::os::unix::io::RawFd {
+        match self {
+            Tcp(ref t) => t.as_raw_fd(),
+            #[cfg(feature = "tls")]
+            Tls(ref t) => t.get_ref().as_raw_fd(),
+            Fd(f) => *f,
+            #[cfg(feature = "deflate_compression")]
+            Deflate { ref inner, .. } => inner.get_ref().get_ref().as_raw_fd(),
+        }
+    }
+}
+
+pub fn lookup_ipv4(host: &str, port: u16) -> crate::Result<std::net::SocketAddr> {
+    use std::net::ToSocketAddrs;
+
+    let addrs = (host, port).to_socket_addrs()?;
+    for addr in addrs {
+        if let std::net::SocketAddr::V4(_) = addr {
+            return Ok(addr);
+        }
+    }
+
+    Err(
+        crate::error::MeliError::new(format!("Could not lookup address {}:{}", host, port))
+            .set_kind(crate::error::ErrorKind::Network),
+    )
+}
+
+use futures::future::{self, Either, Future};
+
+pub async fn timeout<O>(dur: Option<Duration>, f: impl Future<Output = O>) -> crate::Result<O> {
+    futures::pin_mut!(f);
+    if let Some(dur) = dur {
+        match future::select(f, smol::Timer::after(dur)).await {
+            Either::Left((out, _)) => Ok(out),
+            Either::Right(_) => Err(crate::error::MeliError::new("Timed out.")
+                .set_kind(crate::error::ErrorKind::Timeout)),
+        }
+    } else {
+        Ok(f.await)
+    }
+}
+
+pub async fn sleep(dur: Duration) {
+    smol::Timer::after(dur).await;
+}

melib/src/datetime.rs

@@ -0,0 +1,739 @@
+/*
+ * meli - melib POSIX libc time interface
+ *
+ * Copyright 2020 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//! Functions for dealing with date strings and UNIX Epoch timestamps.
+//!
+//! # Examples
+//!
+//! ```rust
+//! # use melib::datetime::*;
+//! // Get current UNIX Epoch timestamp.
+//! let now: UnixTimestamp = now();
+//!
+//! // Parse date from string
+//! let date_val = "Wed, 8 Jan 2020 10:44:03 -0800";
+//! let timestamp = rfc822_to_timestamp(date_val).unwrap();
+//! assert_eq!(timestamp, 1578509043);
+//!
+//! // Convert timestamp back to string
+//! let s = timestamp_to_string(timestamp, Some("%Y-%m-%d"), true);
+//! assert_eq!(s, "2020-01-08");
+//! ```
+use crate::error::{Result, ResultIntoMeliError};
+use std::borrow::Cow;
+use std::convert::TryInto;
+use std::ffi::{CStr, CString};
+
+pub type UnixTimestamp = u64;
+pub const RFC3339_FMT_WITH_TIME: &str = "%Y-%m-%dT%H:%M:%S\0";
+pub const RFC3339_FMT: &str = "%Y-%m-%d\0";
+pub const RFC822_FMT_WITH_TIME: &str = "%a, %e %h %Y %H:%M:%S \0";
+pub const RFC822_FMT: &str = "%e %h %Y %H:%M:%S \0";
+pub const DEFAULT_FMT: &str = "%a, %d %b %Y %R\0";
+//"Tue May 21 13:46:22 1991\n"
+pub const ASCTIME_FMT: &str = "%a %b %d %H:%M:%S %Y\n\0";
+
+extern "C" {
+    fn strptime(
+        s: *const std::os::raw::c_char,
+        format: *const std::os::raw::c_char,
+        tm: *mut libc::tm,
+    ) -> *const std::os::raw::c_char;
+
+    fn strftime(
+        s: *mut std::os::raw::c_char,
+        max: libc::size_t,
+        format: *const std::os::raw::c_char,
+        tm: *const libc::tm,
+    ) -> libc::size_t;
+
+    fn mktime(tm: *const libc::tm) -> libc::time_t;
+
+    fn localtime_r(timep: *const libc::time_t, tm: *mut libc::tm) -> *mut libc::tm;
+
+    fn gettimeofday(tv: *mut libc::timeval, tz: *mut libc::timezone) -> i32;
+}
+
+struct Locale {
+    new_locale: libc::locale_t,
+    old_locale: libc::locale_t,
+}
+
+impl Drop for Locale {
+    fn drop(&mut self) {
+        unsafe {
+            let _ = libc::uselocale(self.old_locale);
+            libc::freelocale(self.new_locale);
+        }
+    }
+}
+
+// How to unit test this? Test machine is not guaranteed to have non-english locales.
+impl Locale {
+    fn new(
+        mask: std::os::raw::c_int,
+        locale: *const std::os::raw::c_char,
+        base: libc::locale_t,
+    ) -> Result<Self> {
+        let new_locale = unsafe { libc::newlocale(mask, locale, base) };
+        if new_locale.is_null() {
+            return Err(nix::Error::last().into());
+        }
+        let old_locale = unsafe { libc::uselocale(new_locale) };
+        if old_locale.is_null() {
+            unsafe { libc::freelocale(new_locale) };
+            return Err(nix::Error::last().into());
+        }
+        Ok(Locale {
+            new_locale,
+            old_locale,
+        })
+    }
+}
+
+pub fn timestamp_to_string(timestamp: UnixTimestamp, fmt: Option<&str>, posix: bool) -> String {
+    let mut new_tm: libc::tm = unsafe { std::mem::zeroed() };
+    unsafe {
+        let i: i64 = timestamp.try_into().unwrap_or(0);
+        localtime_r(&i as *const i64, &mut new_tm as *mut libc::tm);
+    }
+    let format: Cow<'_, CStr> = if let Some(cs) = fmt
+        .map(str::as_bytes)
+        .map(CStr::from_bytes_with_nul)
+        .and_then(|res| res.ok())
+    {
+        Cow::from(cs)
+    } else if let Some(cstring) = fmt
+        .map(str::as_bytes)
+        .map(CString::new)
+        .and_then(|res| res.ok())
+    {
+        Cow::from(cstring)
+    } else {
+        unsafe { CStr::from_bytes_with_nul_unchecked(DEFAULT_FMT.as_bytes()).into() }
+    };
+
+    let mut vec: [u8; 256] = [0; 256];
+    let ret = {
+        let _with_locale: Option<Result<Locale>> = if posix {
+            Some(
+                Locale::new(
+                    libc::LC_TIME,
+                    b"C\0".as_ptr() as *const i8,
+                    std::ptr::null_mut(),
+                )
+                .chain_err_summary(|| "Could not set locale for datetime conversion")
+                .chain_err_kind(crate::error::ErrorKind::External),
+            )
+        } else {
+            None
+        };
+
+        unsafe {
+            strftime(
+                vec.as_mut_ptr() as *mut _,
+                256,
+                format.as_ptr(),
+                &new_tm as *const _,
+            )
+        }
+    };
+
+    String::from_utf8_lossy(&vec[0..ret]).into_owned()
+}
+
+fn tm_to_secs(tm: libc::tm) -> std::result::Result<i64, ()> {
+    let mut is_leap = false;
+    let mut year = tm.tm_year;
+    let mut month = tm.tm_mon;
+    if month >= 12 || month < 0 {
+        let mut adj = month / 12;
+        month %= 12;
+        if month < 0 {
+            adj -= 1;
+            month += 12;
+        }
+        year += adj;
+    }
+    let mut t = year_to_secs(year.into(), &mut is_leap)?;
+    t += month_to_secs(month.try_into().unwrap_or(0), is_leap);
+    t += 86400 * (tm.tm_mday - 1) as i64;
+    t += 3600 * (tm.tm_hour) as i64;
+    t += 60 * (tm.tm_min) as i64;
+    t += tm.tm_sec as i64;
+    Ok(t)
+}
+
+fn year_to_secs(year: i64, is_leap: &mut bool) -> std::result::Result<i64, ()> {
+    if year < -100 {
+        /* Sorry time travelers. */
+        return Err(());
+    }
+
+    if year - 2 <= 136 {
+        let y = year;
+        let mut leaps = (y - 68) >> 2;
+        if (y - 68) & 3 == 0 {
+            leaps -= 1;
+            *is_leap = true;
+        } else {
+            *is_leap = false;
+        }
+        return Ok((31536000 * (y - 70) + 86400 * leaps)
+            .try_into()
+            .unwrap_or(0));
+    }
+
+    let cycles = (year - 100) / 400;
+    let centuries;
+    let mut leaps;
+    let mut rem;
+
+    rem = (year - 100) % 400;
+
+    if rem == 0 {
+        *is_leap = true;
+        centuries = 0;
+        leaps = 0;
+    } else {
+        if rem >= 200 {
+            if rem >= 300 {
+                centuries = 3;
+                rem -= 300;
+            } else {
+                centuries = 2;
+                rem -= 200;
+            }
+        } else if rem >= 100 {
+            centuries = 1;
+            rem -= 100;
+        } else {
+            centuries = 0;
+        }
+        if rem == 0 {
+            *is_leap = false;
+            leaps = 0;
+        } else {
+            leaps = rem / 4;
+            rem %= 4;
+            *is_leap = rem == 0;
+        }
+    }
+
+    leaps += 97 * cycles + 24 * centuries - if *is_leap { 1 } else { 0 };
+
+    match (year - 100).overflowing_mul(31536000) {
+        (_, true) => Err(()),
+        (res, false) => Ok(res + leaps * 86400 + 946684800 + 86400),
+    }
+}
+
+fn month_to_secs(month: usize, is_leap: bool) -> i64 {
+    const SECS_THROUGH_MONTH: [i64; 12] = [
+        0,
+        31 * 86400,
+        59 * 86400,
+        90 * 86400,
+        120 * 86400,
+        151 * 86400,
+        181 * 86400,
+        212 * 86400,
+        243 * 86400,
+        273 * 86400,
+        304 * 86400,
+        334 * 86400,
+    ];
+    let mut t = SECS_THROUGH_MONTH[month];
+    if is_leap && month >= 2 {
+        t += 86400;
+    }
+    t
+}
+
+pub fn rfc822_to_timestamp<T>(s: T) -> Result<UnixTimestamp>
+where
+    T: Into<Vec<u8>>,
+{
+    let s = CString::new(s)?;
+    let mut new_tm: libc::tm = unsafe { std::mem::zeroed() };
+    for fmt in &[RFC822_FMT_WITH_TIME, RFC822_FMT] {
+        let fmt = unsafe { CStr::from_bytes_with_nul_unchecked(fmt.as_bytes()) };
+        let ret = {
+            let _with_locale = Locale::new(
+                libc::LC_TIME,
+                b"C\0".as_ptr() as *const i8,
+                std::ptr::null_mut(),
+            )
+            .chain_err_summary(|| "Could not set locale for datetime conversion")
+            .chain_err_kind(crate::error::ErrorKind::External)?;
+            unsafe { strptime(s.as_ptr(), fmt.as_ptr(), &mut new_tm as *mut _) }
+        };
+
+        if ret.is_null() {
+            continue;
+        }
+        let rest = unsafe { CStr::from_ptr(ret) };
+        let tm_gmtoff = if rest.to_bytes().len() > 4
+            && rest.to_bytes().is_ascii()
+            && rest.to_bytes()[1..5].iter().all(u8::is_ascii_digit)
+        {
+            // safe since rest.to_bytes().is_ascii()
+            let offset = unsafe { std::str::from_utf8_unchecked(&rest.to_bytes()[0..5]) };
+            if let (Ok(mut hr_offset), Ok(mut min_offset)) =
+                (offset[1..3].parse::<i64>(), offset[3..5].parse::<i64>())
+            {
+                if rest.to_bytes()[0] == b'-' {
+                    hr_offset = -hr_offset;
+                    min_offset = -min_offset;
+                }
+                hr_offset * 60 * 60 + min_offset * 60
+            } else {
+                0
+            }
+        } else {
+            let rest = if rest.to_bytes().starts_with(b"(") && rest.to_bytes().ends_with(b")") {
+                &rest.to_bytes()[1..rest.to_bytes().len() - 1]
+            } else {
+                rest.to_bytes()
+            };
+
+            if let Ok(idx) = TIMEZONE_ABBR.binary_search_by(|probe| probe.0.cmp(rest)) {
+                let (hr_offset, min_offset) = TIMEZONE_ABBR[idx].1;
+                (hr_offset as i64) * 60 * 60 + (min_offset as i64) * 60
+            } else {
+                0
+            }
+        };
+        return Ok(tm_to_secs(new_tm)
+            .map(|res| (res - tm_gmtoff) as u64)
+            .unwrap_or(0));
+    }
+    Ok(0)
+}
+
+pub fn rfc3339_to_timestamp<T>(s: T) -> Result<UnixTimestamp>
+where
+    T: Into<Vec<u8>>,
+{
+    let s = CString::new(s)?;
+    let mut new_tm: libc::tm = unsafe { std::mem::zeroed() };
+    for fmt in &[RFC3339_FMT_WITH_TIME, RFC3339_FMT] {
+        let fmt = unsafe { CStr::from_bytes_with_nul_unchecked(fmt.as_bytes()) };
+        let ret = {
+            let _with_locale = Locale::new(
+                libc::LC_TIME,
+                b"C\0".as_ptr() as *const i8,
+                std::ptr::null_mut(),
+            )
+            .chain_err_summary(|| "Could not set locale for datetime conversion")
+            .chain_err_kind(crate::error::ErrorKind::External)?;
+            unsafe { strptime(s.as_ptr(), fmt.as_ptr(), &mut new_tm as *mut _) }
+        };
+        if ret.is_null() {
+            continue;
+        }
+        let rest = unsafe { CStr::from_ptr(ret) };
+        let tm_gmtoff = if rest.to_bytes().len() > 4
+            && rest.to_bytes().is_ascii()
+            && rest.to_bytes()[1..3].iter().all(u8::is_ascii_digit)
+            && rest.to_bytes()[4..6].iter().all(u8::is_ascii_digit)
+        {
+            // safe since rest.to_bytes().is_ascii()
+            let offset = unsafe { std::str::from_utf8_unchecked(&rest.to_bytes()[0..6]) };
+            if let (Ok(mut hr_offset), Ok(mut min_offset)) =
+                (offset[1..3].parse::<i64>(), offset[4..6].parse::<i64>())
+            {
+                if rest.to_bytes()[0] == b'-' {
+                    hr_offset = -hr_offset;
+                    min_offset = -min_offset;
+                }
+                hr_offset * 60 * 60 + min_offset * 60
+            } else {
+                0
+            }
+        } else {
+            let rest = if rest.to_bytes().starts_with(b"(") && rest.to_bytes().ends_with(b")") {
+                &rest.to_bytes()[1..rest.to_bytes().len() - 1]
+            } else {
+                rest.to_bytes()
+            };
+
+            if let Ok(idx) = TIMEZONE_ABBR.binary_search_by(|probe| probe.0.cmp(rest)) {
+                let (hr_offset, min_offset) = TIMEZONE_ABBR[idx].1;
+                (hr_offset as i64) * 60 * 60 + (min_offset as i64) * 60
+            } else {
+                0
+            }
+        };
+        return Ok(tm_to_secs(new_tm)
+            .map(|res| (res - tm_gmtoff) as u64)
+            .unwrap_or(0));
+    }
+    Ok(0)
+}
+
+// FIXME: Handle non-local timezone?
+pub fn timestamp_from_string<T>(s: T, fmt: &str) -> Result<Option<UnixTimestamp>>
+where
+    T: Into<Vec<u8>>,
+{
+    let mut new_tm: libc::tm = unsafe { std::mem::zeroed() };
+    let fmt: Cow<'_, CStr> = if let Ok(cs) = CStr::from_bytes_with_nul(fmt.as_bytes()) {
+        Cow::from(cs)
+    } else {
+        Cow::from(CString::new(fmt.as_bytes())?)
+    };
+    unsafe {
+        let ret = strptime(
+            CString::new(s)?.as_ptr(),
+            fmt.as_ptr(),
+            &mut new_tm as *mut _,
+        );
+        if ret.is_null() {
+            return Ok(None);
+        }
+        Ok(Some(mktime(&new_tm as *const _) as u64))
+    }
+}
+
+pub fn now() -> UnixTimestamp {
+    use std::mem::MaybeUninit;
+    let mut tv = MaybeUninit::<libc::timeval>::uninit();
+    let mut tz = MaybeUninit::<libc::timezone>::uninit();
+    unsafe {
+        let ret = gettimeofday(tv.as_mut_ptr(), tz.as_mut_ptr());
+        if ret == -1 {
+            unreachable!("gettimeofday returned -1");
+        }
+        (tv.assume_init()).tv_sec as UnixTimestamp
+    }
+}
+
+#[test]
+fn test_datetime_timestamp() {
+    timestamp_to_string(0, None, false);
+}
+
+#[test]
+fn test_datetime_rfcs() {
+    if unsafe { libc::setlocale(libc::LC_ALL, b"\0".as_ptr() as _) }.is_null() {
+        println!("Unable to set locale.");
+    }
+    /* Some tests were lazily stolen from https://rachelbythebay.com/w/2013/06/11/time/ */
+
+    assert_eq!(
+        rfc822_to_timestamp("Wed, 8 Jan 2020 10:44:03 -0800").unwrap(),
+        1578509043
+    );
+
+    /*
+    macro_rules! mkt {
+        ($year:literal, $month:literal, $day:literal, $hour:literal, $minute:literal, $second:literal) => {
+            libc::tm {
+                tm_sec: $second,
+                tm_min: $minute,
+                tm_hour: $hour,
+                tm_mday: $day,
+                tm_mon: $month - 1,
+                tm_year: $year - 1900,
+                tm_wday: 0,
+                tm_yday: 0,
+                tm_isdst: 0,
+                tm_gmtoff: 0,
+                tm_zone: std::ptr::null(),
+            }
+        };
+    }
+    */
+    //unsafe { __tm_to_secs(&mkt!(2009, 02, 13, 23, 31, 30) as *const _) },
+    assert_eq!(
+        rfc822_to_timestamp("Fri, 13 Feb 2009 15:31:30 -0800").unwrap(),
+        1234567890
+    );
+
+    //unsafe { __tm_to_secs(&mkt!(2931, 05, 05, 00, 33, 09) as *const _) },
+    assert_eq!(
+        rfc822_to_timestamp("Sat, 05 May 2931 00:33:09 +0000").unwrap(),
+        30336942789
+    );
+    //2214-11-06 20:05:12 = 7726651512 [OK]
+    assert_eq!(
+        rfc822_to_timestamp("Sun, 06 Nov 2214 17:05:12 -0300").unwrap(), //2214-11-06 20:05:12
+        7726651512
+    );
+    assert_eq!(
+        rfc822_to_timestamp("Sun, 06 Nov 2214 17:05:12 -0300").unwrap(), //2214-11-06 20:05:12
+        rfc822_to_timestamp("Sun, 06 Nov 2214 17:05:12 (ADT)").unwrap(), //2214-11-06 20:05:12
+    );
+    //2661-11-06 06:38:02 = 21832612682 [OK]
+    assert_eq!(
+        rfc822_to_timestamp("Wed, 06 Nov 2661 06:38:02 +0000").unwrap(), //2661-11-06 06:38:02
+        21832612682
+    );
+    //2508-12-09 04:27:08 = 17007251228 [OK]
+    assert_eq!(
+        rfc822_to_timestamp("Sun, 09 Dec 2508 04:27:08 +0000").unwrap(), //2508-12-09 04:27:08
+        17007251228
+    );
+    //2375-11-07 05:08:24 = 12807349704 [OK]
+    assert_eq!(
+        rfc822_to_timestamp("Fri, 07 Nov 2375 05:08:24 +0000").unwrap(), //2375-11-07 05:08:24
+        12807349704
+    );
+    //2832-09-03 02:46:10 = 27223353970 [OK]
+    assert_eq!(
+        rfc822_to_timestamp("Fri, 03 Sep 2832 02:46:10 +0000").unwrap(), //2832-09-03 02:46:10
+        27223353970
+    );
+    //2983-02-25 12:47:17 = 31972020437 [OK]
+    assert_eq!(
+        rfc822_to_timestamp("Tue, 25 Feb 2983 15:47:17 +0300").unwrap(), //2983-02-25 12:47:17
+        31972020437
+    );
+
+    assert_eq!(
+        rfc822_to_timestamp("Thu, 30 Mar 2017 17:32:06 +0300 (EEST)").unwrap(),
+        1490884326
+    );
+
+    assert_eq!(
+        rfc822_to_timestamp("Mon, 24 Apr 2017 17:36:34 +0530").unwrap(),
+        1493035594
+    );
+
+    assert_eq!(
+        rfc822_to_timestamp("Mon, 24 Apr 2017 17:36:34 +0530").unwrap(),
+        rfc822_to_timestamp("Mon, 24 Apr 2017 12:06:34 +0000").unwrap(),
+    );
+
+    assert_eq!(
+        rfc822_to_timestamp("Mon, 24 Apr 2017 17:36:34 +0530").unwrap(),
+        rfc822_to_timestamp("Mon, 24 Apr 2017 17:36:34 (SLST)").unwrap(),
+    );
+
+    assert_eq!(
+        rfc822_to_timestamp("Mon, 24 Apr 2017 17:36:34 +0530").unwrap(),
+        rfc822_to_timestamp("Mon, 24 Apr 2017 17:36:34 SLST").unwrap(),
+    );
+
+    assert_eq!(
+        rfc822_to_timestamp("27 Dec 2019 14:42:46 +0100").unwrap(),
+        1577454166
+    );
+
+    assert_eq!(
+        rfc822_to_timestamp("Mon, 16 Mar 2020 10:23:01 +0200").unwrap(),
+        1584346981
+    );
+}
+
+#[allow(clippy::zero_prefixed_literal)]
+const TIMEZONE_ABBR: &[(&[u8], (i8, i8))] = &[
+    (b"ACDT", (10, 30)),
+    (b"ACST", (09, 30)),
+    (b"ACT", (-05, 0)),
+    (b"ACWST", (08, 45)),
+    (b"ADT", (-03, 0)),
+    (b"AEDT", (11, 0)),
+    (b"AEST", (10, 0)),
+    (b"AFT", (04, 30)),
+    (b"AKDT", (-08, 0)),
+    (b"AKST", (-09, 0)),
+    (b"ALMT", (06, 0)),
+    (b"AMST", (-03, 0)),
+    (b"AMT", (-04, 0)), /* Amazon Time */
+    (b"ANAT", (12, 0)),
+    (b"AQTT", (05, 0)),
+    (b"ART", (-03, 0)),
+    (b"AST", (-04, 0)),
+    (b"AST", (03, 0)),
+    (b"AWST", (08, 0)),
+    (b"AZOST", (0, 0)),
+    (b"AZOT", (-01, 0)),
+    (b"AZT", (04, 0)),
+    (b"BDT", (08, 0)),
+    (b"BIOT", (06, 0)),
+    (b"BIT", (-12, 0)),
+    (b"BOT", (-04, 0)),
+    (b"BRST", (-02, 0)),
+    (b"BRT", (-03, 0)),
+    (b"BST", (06, 0)),
+    (b"BTT", (06, 0)),
+    (b"CAT", (02, 0)),
+    (b"CCT", (06, 30)),
+    (b"CDT", (-05, 0)),
+    (b"CEST", (02, 0)),
+    (b"CET", (01, 0)),
+    (b"CHADT", (13, 45)),
+    (b"CHAST", (12, 45)),
+    (b"CHOST", (09, 0)),
+    (b"CHOT", (08, 0)),
+    (b"CHST", (10, 0)),
+    (b"CHUT", (10, 0)),
+    (b"CIST", (-08, 0)),
+    (b"CIT", (08, 0)),
+    (b"CKT", (-10, 0)),
+    (b"CLST", (-03, 0)),
+    (b"CLT", (-04, 0)),
+    (b"COST", (-04, 0)),
+    (b"COT", (-05, 0)),
+    (b"CST", (-06, 0)),
+    (b"CT", (08, 0)),
+    (b"CVT", (-01, 0)),
+    (b"CWST", (08, 45)),
+    (b"CXT", (07, 0)),
+    (b"DAVT", (07, 0)),
+    (b"DDUT", (10, 0)),
+    (b"DFT", (01, 0)),
+    (b"EASST", (-05, 0)),
+    (b"EAST", (-06, 0)),
+    (b"EAT", (03, 0)),
+    (b"ECT", (-05, 0)),
+    (b"EDT", (-04, 0)),
+    (b"EEST", (03, 0)),
+    (b"EET", (02, 0)),
+    (b"EGST", (0, 0)),
+    (b"EGT", (-01, 0)),
+    (b"EIT", (09, 0)),
+    (b"EST", (-05, 0)),
+    (b"FET", (03, 0)),
+    (b"FJT", (12, 0)),
+    (b"FKST", (-03, 0)),
+    (b"FKT", (-04, 0)),
+    (b"FNT", (-02, 0)),
+    (b"GALT", (-06, 0)),
+    (b"GAMT", (-09, 0)),
+    (b"GET", (04, 0)),
+    (b"GFT", (-03, 0)),
+    (b"GILT", (12, 0)),
+    (b"GIT", (-09, 0)),
+    (b"GMT", (0, 0)),
+    (b"GST", (04, 0)),
+    (b"GYT", (-04, 0)),
+    (b"HAEC", (02, 0)),
+    (b"HDT", (-09, 0)),
+    (b"HKT", (08, 0)),
+    (b"HMT", (05, 0)),
+    (b"HOVST", (08, 0)),
+    (b"HOVT", (07, 0)),
+    (b"HST", (-10, 0)),
+    (b"ICT", (07, 0)),
+    (b"IDLW", (-12, 0)),
+    (b"IDT", (03, 0)),
+    (b"IOT", (03, 0)),
+    (b"IRDT", (04, 30)),
+    (b"IRKT", (08, 0)),
+    (b"IRST", (03, 30)),
+    (b"IST", (05, 30)),
+    (b"JST", (09, 0)),
+    (b"KALT", (02, 0)),
+    (b"KGT", (06, 0)),
+    (b"KOST", (11, 0)),
+    (b"KRAT", (07, 0)),
+    (b"KST", (09, 0)),
+    (b"LHST", (10, 30)),
+    (b"LINT", (14, 0)),
+    (b"MAGT", (12, 0)),
+    (b"MART", (-09, -30)),
+    (b"MAWT", (05, 0)),
+    (b"MDT", (-06, 0)),
+    (b"MEST", (02, 0)),
+    (b"MET", (01, 0)),
+    (b"MHT", (12, 0)),
+    (b"MIST", (11, 0)),
+    (b"MIT", (-09, -30)),
+    (b"MMT", (06, 30)),
+    (b"MSK", (03, 0)),
+    (b"MST", (08, 0)),
+    (b"MUT", (04, 0)),
+    (b"MVT", (05, 0)),
+    (b"MYT", (08, 0)),
+    (b"NCT", (11, 0)),
+    (b"NDT", (-02, -30)),
+    (b"NFT", (11, 0)),
+    (b"NOVT", (07, 0)),
+    (b"NPT", (05, 45)),
+    (b"NST", (-03, -30)),
+    (b"NT", (-03, -30)),
+    (b"NUT", (-11, 0)),
+    (b"NZDT", (13, 0)),
+    (b"NZST", (12, 0)),
+    (b"OMST", (06, 0)),
+    (b"ORAT", (05, 0)),
+    (b"PDT", (-07, 0)),
+    (b"PET", (-05, 0)),
+    (b"PETT", (12, 0)),
+    (b"PGT", (10, 0)),
+    (b"PHOT", (13, 0)),
+    (b"PHT", (08, 0)),
+    (b"PKT", (05, 0)),
+    (b"PMDT", (-02, 0)),
+    (b"PMST", (-03, 0)),
+    (b"PONT", (11, 0)),
+    (b"PST", (-08, 0)),
+    (b"PST", (08, 0)),
+    (b"PYST", (-03, 0)),
+    (b"PYT", (-04, 0)),
+    (b"RET", (04, 0)),
+    (b"ROTT", (-03, 0)),
+    (b"SAKT", (11, 0)),
+    (b"SAMT", (04, 0)),
+    (b"SAST", (02, 0)),
+    (b"SBT", (11, 0)),
+    (b"SCT", (04, 0)),
+    (b"SDT", (-10, 0)),
+    (b"SGT", (08, 0)),
+    (b"SLST", (05, 30)),
+    (b"SRET", (11, 0)),
+    (b"SRT", (-03, 0)),
+    (b"SST", (08, 0)),
+    (b"SYOT", (03, 0)),
+    (b"TAHT", (-10, 0)),
+    (b"TFT", (05, 0)),
+    (b"THA", (07, 0)),
+    (b"TJT", (05, 0)),
+    (b"TKT", (13, 0)),
+    (b"TLT", (09, 0)),
+    (b"TMT", (05, 0)),
+    (b"TOT", (13, 0)),
+    (b"TRT", (03, 0)),
+    (b"TVT", (12, 0)),
+    (b"ULAST", (09, 0)),
+    (b"ULAT", (08, 0)),
+    (b"UTC", (0, 0)),
+    (b"UYST", (-02, 0)),
+    (b"UYT", (-03, 0)),
+    (b"UZT", (05, 0)),
+    (b"VET", (-04, 0)),
+    (b"VLAT", (10, 0)),
+    (b"VOLT", (04, 0)),
+    (b"VOST", (06, 0)),
+    (b"VUT", (11, 0)),
+    (b"WAKT", (12, 0)),
+    (b"WAST", (02, 0)),
+    (b"WAT", (01, 0)),
+    (b"WEST", (01, 0)),
+    (b"WET", (0, 0)),
+    (b"WIT", (07, 0)),
+    (b"WST", (08, 0)),
+    (b"YAKT", (09, 0)),
+    (b"YEKT", (05, 0)),
+];

melib/src/email/address.rs

@@ -19,7 +19,11 @@
  * along with meli. If not, see <http://www.gnu.org/licenses/>.
  */
 
+//! Email addresses. Parsing functions are in [melib::email::parser::address](../parser/address/index.html).
 use super::*;
+use std::collections::HashSet;
+use std::convert::TryFrom;
+use std::hash::{Hash, Hasher};
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct GroupAddress {
@@ -29,12 +33,63 @@ pub struct GroupAddress {
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
+/**
+ * Container for an address.
+ *
+ * ```text
+ * >           raw: Vec<u8>
+ * > ┌──────────┴────────────┐
+ * > Name <address@domain.tld>
+ * > └─┬┘  └──────────┬─────┘
+ * > display_name     │
+ * >                  │
+ * >            address_spec
+ *
+ *
+ * >           raw: Vec<u8>
+ * > ┌──────────┴────────────────────┐
+ * > "Name Name2" <address@domain.tld>
+ * >  └─────┬──┘   └──────────┬─────┘
+ * > display_name             │
+ * >                          │
+ * >                    address_spec
+ *```
+ */
 pub struct MailboxAddress {
     pub raw: Vec<u8>,
     pub display_name: StrBuilder,
     pub address_spec: StrBuilder,
 }
 
+impl Eq for MailboxAddress {}
+
+impl PartialEq for MailboxAddress {
+    fn eq(&self, other: &MailboxAddress) -> bool {
+        self.address_spec.display_bytes(&self.raw) == other.address_spec.display_bytes(&other.raw)
+    }
+}
+
+/// An email address.
+///
+/// Conforms to [RFC5322 - Internet Message Format](https://tools.ietf.org/html/rfc5322).
+///
+/// # Creating an `Address`
+/// You can directly create an address with `Address::new`,
+///
+/// ```rust
+/// # use melib::email::Address;
+/// let addr = Address::new(Some("Jörg Doe".to_string()), "joerg@example.com".to_string());
+/// assert_eq!(addr.to_string().as_str(), "Jörg Doe <joerg@example.com>");
+/// ```
+///
+/// or parse it from a raw value:
+///
+/// ```rust
+/// let (rest_bytes, addr) = melib::email::parser::address::address("=?utf-8?q?J=C3=B6rg_Doe?= <joerg@example.com>".as_bytes()).unwrap();
+/// assert!(rest_bytes.is_empty());
+/// assert_eq!(addr.get_display_name(), Some("Jörg Doe".to_string()));
+/// assert_eq!(addr.get_email(), "joerg@example.com".to_string());
+/// ```
 #[derive(Clone, Serialize, Deserialize)]
 pub enum Address {
     Mailbox(MailboxAddress),
@@ -42,19 +97,103 @@ pub enum Address {
 }
 
 impl Address {
-    pub fn get_display_name(&self) -> String {
+    pub fn new(display_name: Option<String>, address: String) -> Self {
+        Address::Mailbox(if let Some(d) = display_name {
+            MailboxAddress {
+                raw: format!("{} <{}>", d, address).into_bytes(),
+                display_name: StrBuilder {
+                    offset: 0,
+                    length: d.len(),
+                },
+                address_spec: StrBuilder {
+                    offset: d.len() + 2,
+                    length: address.len(),
+                },
+            }
+        } else {
+            MailboxAddress {
+                raw: format!("{}", address).into_bytes(),
+                display_name: StrBuilder {
+                    offset: 0,
+                    length: 0,
+                },
+                address_spec: StrBuilder {
+                    offset: 0,
+                    length: address.len(),
+                },
+            }
+        })
+    }
+
+    pub fn new_group(display_name: String, mailbox_list: Vec<Address>) -> Self {
+        Address::Group(GroupAddress {
+            raw: format!(
+                "{}:{};",
+                display_name,
+                mailbox_list
+                    .iter()
+                    .map(|a| a.to_string())
+                    .collect::<Vec<String>>()
+                    .join(",")
+            )
+            .into_bytes(),
+            display_name: StrBuilder {
+                offset: 0,
+                length: display_name.len(),
+            },
+            mailbox_list,
+        })
+    }
+
+    pub fn raw(&self) -> &[u8] {
         match self {
-            Address::Mailbox(m) => m.display_name.display(&m.raw),
-            Address::Group(g) => g.display_name.display(&g.raw),
+            Address::Mailbox(m) => m.raw.as_slice(),
+            Address::Group(g) => g.raw.as_slice(),
         }
     }
 
+    /// Get the display name of this address.
+    ///
+    /// If it's a group, it's the name of the group. Otherwise it's the `display_name` part of
+    /// the mailbox:
+    ///
+    ///
+    /// ```text
+    ///           raw                         raw
+    /// ┌──────────┴────────────┐   ┌──────────┴────────────────────┐
+    /// Name <address@domain.tld>   "Name Name2" <address@domain.tld>
+    /// └─┬┘  └──────────┬─────┘     └─────┬──┘   └──────────┬─────┘
+    /// display_name     │          display_name             │
+    ///                  │                                   │
+    ///            address_spec                        address_spec
+    ///```
+    pub fn get_display_name(&self) -> Option<String> {
+        let ret = match self {
+            Address::Mailbox(m) => m.display_name.display(&m.raw),
+            Address::Group(g) => g.display_name.display(&g.raw),
+        };
+        if ret.is_empty() {
+            None
+        } else {
+            Some(ret)
+        }
+    }
+
+    /// Get the address spec part of this address. A group returns an empty `String`.
     pub fn get_email(&self) -> String {
         match self {
             Address::Mailbox(m) => m.address_spec.display(&m.raw),
             Address::Group(_) => String::new(),
         }
     }
+
+    pub fn address_spec_raw(&self) -> &[u8] {
+        match self {
+            Address::Mailbox(m) => m.address_spec.display_bytes(&m.raw),
+            Address::Group(g) => &g.raw,
+        }
+    }
+
     pub fn get_fqdn(&self) -> Option<String> {
         match self {
             Address::Mailbox(m) => {
@@ -67,38 +206,122 @@ impl Address {
     }
 
     pub fn get_tags(&self, separator: char) -> Vec<String> {
-        let mut ret = Vec::new();
         let email = self.get_email();
-        let at_pos = email.as_bytes().iter().position(|&b| b == b'@').unwrap();
+        let at_pos = email
+            .as_bytes()
+            .iter()
+            .position(|&b| b == b'@')
+            .unwrap_or(0);
         let email: &str = email[..at_pos].into();
-        ret.extend(email.split(separator).skip(1).map(str::to_string));
-        ret
+        email
+            .split(separator)
+            .skip(1)
+            .map(str::to_string)
+            .collect::<_>()
+    }
+
+    pub fn list_try_from(val: &str) -> Result<Vec<Address>> {
+        Ok(parser::address::rfc2822address_list(val.as_bytes())?
+            .1
+            .to_vec())
+    }
+
+    pub fn contains_address(&self, other: &Address) -> bool {
+        match self {
+            Address::Mailbox(_) => self == other,
+            Address::Group(g) => g
+                .mailbox_list
+                .iter()
+                .any(|addr| addr.contains_address(other)),
+        }
+    }
+
+    /// Get subaddress out of an address (e.g. `ken+subaddress@example.org`).
+    ///
+    /// Subaddresses are commonly text following a "+" character in an email address's local part
+    /// . They are defined in [RFC5233 `Sieve Email Filtering: Subaddress Extension`](https://tools.ietf.org/html/rfc5233.html)
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use melib::email::Address;
+    /// let addr = "ken+sieve@example.org";
+    /// let (rest, val) = melib::email::parser::address::address(addr.as_bytes()).unwrap();
+    /// assert!(rest.is_empty());
+    /// assert_eq!(
+    ///     val.subaddress("+"),
+    ///     Some((
+    ///         Address::new(None, "ken@example.org".to_string()),
+    ///         "sieve".to_string()
+    ///     ))
+    /// );
+    /// ```
+    pub fn subaddress(&self, separator: &str) -> Option<(Self, String)> {
+        match self {
+            Address::Mailbox(_) => {
+                let email = self.get_email();
+                let (local_part, domain) =
+                    match super::parser::address::addr_spec_raw(email.as_bytes())
+                        .map_err(|err| Into::<MeliError>::into(err))
+                        .and_then(|(_, (l, d))| {
+                            Ok((String::from_utf8(l.into())?, String::from_utf8(d.into())?))
+                        }) {
+                        Ok(v) => v,
+                        Err(_) => return None,
+                    };
+                let s = local_part.split(separator).collect::<Vec<_>>();
+                if s.len() < 2 {
+                    return None;
+                }
+                let subaddress = &local_part[s[0].len() + separator.len()..];
+
+                let display_name = self.get_display_name();
+                Some((
+                    Self::new(display_name, format!("{}@{}", s[0], domain)),
+                    subaddress.to_string(),
+                ))
+            }
+            Address::Group(_) => None,
+        }
     }
 }
 
 impl Eq for Address {}
+
 impl PartialEq for Address {
     fn eq(&self, other: &Address) -> bool {
         match (self, other) {
             (Address::Mailbox(_), Address::Group(_)) | (Address::Group(_), Address::Mailbox(_)) => {
                 false
             }
-            (Address::Mailbox(s), Address::Mailbox(o)) => {
-                s.address_spec.display_bytes(&s.raw) == o.address_spec.display_bytes(&o.raw)
-            }
+            (Address::Mailbox(s), Address::Mailbox(o)) => s == o,
             (Address::Group(s), Address::Group(o)) => {
                 s.display_name.display_bytes(&s.raw) == o.display_name.display_bytes(&o.raw)
-                    && s.mailbox_list
-                        .iter()
-                        .zip(o.mailbox_list.iter())
-                        .fold(true, |b, (s, o)| b && (s == o))
+                    && s.mailbox_list.iter().collect::<HashSet<_>>()
+                        == o.mailbox_list.iter().collect::<HashSet<_>>()
             }
         }
     }
 }
 
-impl fmt::Display for Address {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+impl Hash for Address {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        match self {
+            Address::Mailbox(s) => {
+                s.address_spec.display_bytes(&s.raw).hash(state);
+            }
+            Address::Group(s) => {
+                s.display_name.display_bytes(&s.raw).hash(state);
+                for sub in &s.mailbox_list {
+                    sub.hash(state);
+                }
+            }
+        }
+    }
+}
+
+impl core::fmt::Display for Address {
+    fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
         match self {
             Address::Mailbox(m) if m.display_name.length > 0 => write!(
                 f,
@@ -121,9 +344,31 @@ impl fmt::Display for Address {
     }
 }
 
-impl fmt::Debug for Address {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        fmt::Display::fmt(self, f)
+impl core::fmt::Debug for Address {
+    fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
+        match self {
+            Address::Mailbox(m) => f
+                .debug_struct("Address::Mailbox")
+                .field("display_name", &m.display_name.display(&m.raw))
+                .field("address_spec", &m.address_spec.display(&m.raw))
+                .finish(),
+            Address::Group(g) => {
+                let attachment_strings: Vec<String> =
+                    g.mailbox_list.iter().map(|a| format!("{}", a)).collect();
+
+                f.debug_struct("Address::Group")
+                    .field("display_name", &g.display_name.display(&g.raw))
+                    .field("addresses", &attachment_strings.join(", "))
+                    .finish()
+            }
+        }
+    }
+}
+
+impl TryFrom<&str> for Address {
+    type Error = MeliError;
+    fn try_from(val: &str) -> Result<Address> {
+        Ok(parser::address::address(val.as_bytes())?.1)
     }
 }
 
@@ -148,7 +393,7 @@ impl StrBuilder {
     pub fn display<'a>(&self, s: &'a [u8]) -> String {
         let offset = self.offset;
         let length = self.length;
-        String::from_utf8(s[offset..offset + length].to_vec()).unwrap()
+        String::from_utf8_lossy(&s[offset..offset + length]).to_string()
     }
 
     pub fn display_bytes<'a>(&self, b: &'a [u8]) -> &'a [u8] {
@@ -162,7 +407,7 @@ pub struct MessageID(pub Vec<u8>, pub StrBuilder);
 
 impl StrBuild for MessageID {
     fn new(string: &[u8], slice: &[u8]) -> Self {
-        let offset = string.find(slice).unwrap();
+        let offset = string.find(slice).unwrap_or(0);
         MessageID(
             string.to_owned(),
             StrBuilder {
@@ -183,24 +428,26 @@ impl StrBuild for MessageID {
 
 #[test]
 fn test_strbuilder() {
-    let m_id = b"<20170825132332.6734-1@el13635@mail.ntua.gr>";
-    let (_, slice) = parser::message_id(m_id).unwrap();
+    let m_id = b"<20170825132332.6734-1@mail.ntua.gr>";
+    let (_, val) = parser::address::msg_id(m_id).unwrap();
     assert_eq!(
-        MessageID::new(m_id, slice),
+        val,
         MessageID(
             m_id.to_vec(),
             StrBuilder {
                 offset: 1,
-                length: 43,
+                length: 35,
             }
         )
     );
 }
 
-impl fmt::Display for MessageID {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+impl core::fmt::Display for MessageID {
+    fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
         if self.val().is_ascii() {
-            write!(f, "{}", unsafe { str::from_utf8_unchecked(self.val()) })
+            write!(f, "{}", unsafe {
+                std::str::from_utf8_unchecked(self.val())
+            })
         } else {
             write!(f, "{}", String::from_utf8_lossy(self.val()))
         }
@@ -212,8 +459,8 @@ impl PartialEq for MessageID {
         self.raw() == other.raw()
     }
 }
-impl fmt::Debug for MessageID {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+impl core::fmt::Debug for MessageID {
+    fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
         write!(f, "{}", String::from_utf8(self.raw().to_vec()).unwrap())
     }
 }
@@ -224,8 +471,39 @@ pub struct References {
     pub refs: Vec<MessageID>,
 }
 
-impl fmt::Debug for References {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+impl core::fmt::Debug for References {
+    fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
         write!(f, "{:#?}", self.refs)
     }
 }
+
+#[macro_export]
+macro_rules! make_address {
+    ($d:expr, $a:expr) => {
+        Address::Mailbox(if $d.is_empty() {
+            MailboxAddress {
+                raw: format!("{}", $a).into_bytes(),
+                display_name: StrBuilder {
+                    offset: 0,
+                    length: 0,
+                },
+                address_spec: StrBuilder {
+                    offset: 0,
+                    length: $a.len(),
+                },
+            }
+        } else {
+            MailboxAddress {
+                raw: format!("{} <{}>", $d, $a).into_bytes(),
+                display_name: StrBuilder {
+                    offset: 0,
+                    length: $d.len(),
+                },
+                address_spec: StrBuilder {
+                    offset: $d.len() + 2,
+                    length: $a.len(),
+                },
+            }
+        })
+    };
+}

melib/src/email/attachment_types.rs

@@ -31,8 +31,18 @@ pub enum Charset {
     UTF16,
     ISO8859_1,
     ISO8859_2,
+    ISO8859_3,
+    ISO8859_4,
+    ISO8859_5,
+    ISO8859_6,
     ISO8859_7,
+    ISO8859_8,
+    ISO8859_10,
+    ISO8859_13,
+    ISO8859_14,
     ISO8859_15,
+    ISO8859_16,
+    Windows1250,
     Windows1251,
     Windows1252,
     Windows1253,
@@ -40,6 +50,9 @@ pub enum Charset {
     GB2312,
     BIG5,
     ISO2022JP,
+    EUCJP,
+    KOI8R,
+    KOI8U,
 }
 
 impl Default for Charset {
@@ -50,22 +63,94 @@ impl Default for Charset {
 
 impl<'a> From<&'a [u8]> for Charset {
     fn from(b: &'a [u8]) -> Self {
-        // TODO: Case insensitivity
         match b.trim() {
-            b"us-ascii" | b"ascii" | b"US-ASCII" => Charset::Ascii,
-            b"utf-8" | b"UTF-8" => Charset::UTF8,
-            b"utf-16" | b"UTF-16" => Charset::UTF16,
-            b"iso-8859-1" | b"ISO-8859-1" => Charset::ISO8859_1,
-            b"iso-8859-2" | b"ISO-8859-2" => Charset::ISO8859_2,
-            b"iso-8859-7" | b"ISO-8859-7" | b"iso8859-7" => Charset::ISO8859_7,
-            b"iso-8859-15" | b"ISO-8859-15" => Charset::ISO8859_15,
-            b"windows-1251" | b"Windows-1251" => Charset::Windows1251,
-            b"windows-1252" | b"Windows-1252" => Charset::Windows1252,
-            b"windows-1253" | b"Windows-1253" => Charset::Windows1253,
-            b"GBK" | b"gbk" => Charset::GBK,
-            b"gb2312" | b"GB2312" => Charset::GB2312,
-            b"BIG5" | b"big5" => Charset::BIG5,
-            b"ISO-2022-JP" | b"iso-2022-JP" => Charset::ISO2022JP,
+            b if b.eq_ignore_ascii_case(b"us-ascii") || b.eq_ignore_ascii_case(b"ascii") => {
+                Charset::Ascii
+            }
+            b if b.eq_ignore_ascii_case(b"utf-8") || b.eq_ignore_ascii_case(b"utf8") => {
+                Charset::UTF8
+            }
+            b if b.eq_ignore_ascii_case(b"utf-16") || b.eq_ignore_ascii_case(b"utf16") => {
+                Charset::UTF16
+            }
+            b if b.eq_ignore_ascii_case(b"iso-8859-1") || b.eq_ignore_ascii_case(b"iso8859-1") => {
+                Charset::ISO8859_1
+            }
+            b if b.eq_ignore_ascii_case(b"iso-8859-2") || b.eq_ignore_ascii_case(b"iso8859-2") => {
+                Charset::ISO8859_2
+            }
+            b if b.eq_ignore_ascii_case(b"iso-8859-3") || b.eq_ignore_ascii_case(b"iso8859-3") => {
+                Charset::ISO8859_3
+            }
+            b if b.eq_ignore_ascii_case(b"iso-8859-4") || b.eq_ignore_ascii_case(b"iso8859-4") => {
+                Charset::ISO8859_4
+            }
+            b if b.eq_ignore_ascii_case(b"iso-8859-5") || b.eq_ignore_ascii_case(b"iso8859-5") => {
+                Charset::ISO8859_5
+            }
+            b if b.eq_ignore_ascii_case(b"iso-8859-6") || b.eq_ignore_ascii_case(b"iso8859-6") => {
+                Charset::ISO8859_6
+            }
+            b if b.eq_ignore_ascii_case(b"iso-8859-7") || b.eq_ignore_ascii_case(b"iso8859-7") => {
+                Charset::ISO8859_7
+            }
+            b if b.eq_ignore_ascii_case(b"iso-8859-8") || b.eq_ignore_ascii_case(b"iso8859-8") => {
+                Charset::ISO8859_8
+            }
+            b if b.eq_ignore_ascii_case(b"iso-8859-10")
+                || b.eq_ignore_ascii_case(b"iso8859-10") =>
+            {
+                Charset::ISO8859_10
+            }
+            b if b.eq_ignore_ascii_case(b"iso-8859-13")
+                || b.eq_ignore_ascii_case(b"iso8859-13") =>
+            {
+                Charset::ISO8859_13
+            }
+            b if b.eq_ignore_ascii_case(b"iso-8859-14")
+                || b.eq_ignore_ascii_case(b"iso8859-14") =>
+            {
+                Charset::ISO8859_14
+            }
+            b if b.eq_ignore_ascii_case(b"iso-8859-15")
+                || b.eq_ignore_ascii_case(b"iso8859-15") =>
+            {
+                Charset::ISO8859_15
+            }
+            b if b.eq_ignore_ascii_case(b"iso-8859-16")
+                || b.eq_ignore_ascii_case(b"iso8859-16") =>
+            {
+                Charset::ISO8859_16
+            }
+            b if b.eq_ignore_ascii_case(b"windows-1250")
+                || b.eq_ignore_ascii_case(b"windows1250") =>
+            {
+                Charset::Windows1250
+            }
+            b if b.eq_ignore_ascii_case(b"windows-1251")
+                || b.eq_ignore_ascii_case(b"windows1251") =>
+            {
+                Charset::Windows1251
+            }
+            b if b.eq_ignore_ascii_case(b"windows-1252")
+                || b.eq_ignore_ascii_case(b"windows1252") =>
+            {
+                Charset::Windows1252
+            }
+            b if b.eq_ignore_ascii_case(b"windows-1253")
+                || b.eq_ignore_ascii_case(b"windows1253") =>
+            {
+                Charset::Windows1253
+            }
+            b if b.eq_ignore_ascii_case(b"gbk") => Charset::GBK,
+            b if b.eq_ignore_ascii_case(b"gb2312") || b.eq_ignore_ascii_case(b"gb-2312") => {
+                Charset::GB2312
+            }
+            b if b.eq_ignore_ascii_case(b"big5") => Charset::BIG5,
+            b if b.eq_ignore_ascii_case(b"iso-2022-jp") => Charset::ISO2022JP,
+            b if b.eq_ignore_ascii_case(b"euc-jp") => Charset::EUCJP,
+            b if b.eq_ignore_ascii_case(b"koi8-r") => Charset::KOI8R,
+            b if b.eq_ignore_ascii_case(b"koi8-u") => Charset::KOI8U,
             _ => {
                 debug!("unknown tag is {:?}", str::from_utf8(b));
                 Charset::Ascii
@@ -82,24 +167,39 @@ impl Display for Charset {
             Charset::UTF16 => write!(f, "utf-16"),
             Charset::ISO8859_1 => write!(f, "iso-8859-1"),
             Charset::ISO8859_2 => write!(f, "iso-8859-2"),
+            Charset::ISO8859_3 => write!(f, "iso-8859-3"),
+            Charset::ISO8859_4 => write!(f, "iso-8859-4"),
+            Charset::ISO8859_5 => write!(f, "iso-8859-5"),
+            Charset::ISO8859_6 => write!(f, "iso-8859-6"),
             Charset::ISO8859_7 => write!(f, "iso-8859-7"),
+            Charset::ISO8859_8 => write!(f, "iso-8859-8"),
+            Charset::ISO8859_10 => write!(f, "iso-8859-10"),
+            Charset::ISO8859_13 => write!(f, "iso-8859-13"),
+            Charset::ISO8859_14 => write!(f, "iso-8859-14"),
             Charset::ISO8859_15 => write!(f, "iso-8859-15"),
+            Charset::ISO8859_16 => write!(f, "iso-8859-16"),
+            Charset::Windows1250 => write!(f, "windows-1250"),
             Charset::Windows1251 => write!(f, "windows-1251"),
             Charset::Windows1252 => write!(f, "windows-1252"),
             Charset::Windows1253 => write!(f, "windows-1253"),
-            Charset::GBK => write!(f, "GBK"),
+            Charset::GBK => write!(f, "gbk"),
             Charset::GB2312 => write!(f, "gb2312"),
-            Charset::BIG5 => write!(f, "BIG5"),
-            Charset::ISO2022JP => write!(f, "ISO-2022-JP"),
+            Charset::BIG5 => write!(f, "big5"),
+            Charset::ISO2022JP => write!(f, "iso-2022-jp"),
+            Charset::EUCJP => write!(f, "euc-jp"),
+            Charset::KOI8R => write!(f, "koi8-r"),
+            Charset::KOI8U => write!(f, "koi8-u"),
         }
     }
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub enum MultipartType {
-    Mixed,
     Alternative,
     Digest,
+    Encrypted,
+    Mixed,
+    Related,
     Signed,
 }
 
@@ -111,12 +211,18 @@ impl Default for MultipartType {
 
 impl Display for MultipartType {
     fn fmt(&self, f: &mut Formatter) -> FmtResult {
-        match self {
-            MultipartType::Mixed => write!(f, "multipart/mixed"),
-            MultipartType::Alternative => write!(f, "multipart/alternative"),
-            MultipartType::Digest => write!(f, "multipart/digest"),
-            MultipartType::Signed => write!(f, "multipart/signed"),
-        }
+        write!(
+            f,
+            "{}",
+            match self {
+                MultipartType::Alternative => "multipart/alternative",
+                MultipartType::Digest => "multipart/digest",
+                MultipartType::Encrypted => "multipart/encrypted",
+                MultipartType::Mixed => "multipart/mixed",
+                MultipartType::Related => "multipart/related",
+                MultipartType::Signed => "multipart/signed",
+            }
+        )
     }
 }
 
@@ -128,8 +234,12 @@ impl From<&[u8]> for MultipartType {
             MultipartType::Alternative
         } else if val.eq_ignore_ascii_case(b"digest") {
             MultipartType::Digest
+        } else if val.eq_ignore_ascii_case(b"encrypted") {
+            MultipartType::Encrypted
         } else if val.eq_ignore_ascii_case(b"signed") {
             MultipartType::Signed
+        } else if val.eq_ignore_ascii_case(b"related") {
+            MultipartType::Related
         } else {
             Default::default()
         }
@@ -140,6 +250,7 @@ impl From<&[u8]> for MultipartType {
 pub enum ContentType {
     Text {
         kind: Text,
+        parameters: Vec<(Vec<u8>, Vec<u8>)>,
         charset: Charset,
     },
     Multipart {
@@ -149,6 +260,7 @@ pub enum ContentType {
     },
     MessageRfc822,
     PGPSignature,
+    CMSSignature,
     Other {
         tag: Vec<u8>,
         name: Option<String>,
@@ -162,11 +274,81 @@ impl Default for ContentType {
     fn default() -> Self {
         ContentType::Text {
             kind: Text::Plain,
+            parameters: Vec::new(),
             charset: Charset::UTF8,
         }
     }
 }
 
+impl PartialEq<&str> for ContentType {
+    fn eq(&self, other: &&str) -> bool {
+        match (self, *other) {
+            (
+                ContentType::Text {
+                    kind: Text::Plain, ..
+                },
+                "text/plain",
+            ) => true,
+            (
+                ContentType::Text {
+                    kind: Text::Html, ..
+                },
+                "text/html",
+            ) => true,
+            (
+                ContentType::Multipart {
+                    kind: MultipartType::Alternative,
+                    ..
+                },
+                "multipart/alternative",
+            ) => true,
+            (
+                ContentType::Multipart {
+                    kind: MultipartType::Digest,
+                    ..
+                },
+                "multipart/digest",
+            ) => true,
+            (
+                ContentType::Multipart {
+                    kind: MultipartType::Encrypted,
+                    ..
+                },
+                "multipart/encrypted",
+            ) => true,
+            (
+                ContentType::Multipart {
+                    kind: MultipartType::Mixed,
+                    ..
+                },
+                "multipart/mixed",
+            ) => true,
+            (
+                ContentType::Multipart {
+                    kind: MultipartType::Related,
+                    ..
+                },
+                "multipart/related",
+            ) => true,
+            (
+                ContentType::Multipart {
+                    kind: MultipartType::Signed,
+                    ..
+                },
+                "multipart/signed",
+            ) => true,
+            (ContentType::PGPSignature, "application/pgp-signature") => true,
+            (ContentType::CMSSignature, "application/pkcs7-signature") => true,
+            (ContentType::MessageRfc822, "message/rfc822") => true,
+            (ContentType::Other { tag, .. }, _) => {
+                other.eq_ignore_ascii_case(&String::from_utf8_lossy(&tag))
+            }
+            (ContentType::OctetStream { .. }, "application/octet-stream") => true,
+            _ => false,
+        }
+    }
+}
+
 impl Display for ContentType {
     fn fmt(&self, f: &mut Formatter) -> FmtResult {
         match self {
@@ -174,6 +356,7 @@ impl Display for ContentType {
             ContentType::Multipart { kind: k, .. } => k.fmt(f),
             ContentType::Other { ref tag, .. } => write!(f, "{}", String::from_utf8_lossy(tag)),
             ContentType::PGPSignature => write!(f, "application/pgp-signature"),
+            ContentType::CMSSignature => write!(f, "application/pkcs7-signature"),
             ContentType::MessageRfc822 => write!(f, "message/rfc822"),
             ContentType::OctetStream { .. } => write!(f, "application/octet-stream"),
         }
@@ -200,7 +383,7 @@ impl ContentType {
         }
     }
 
-    pub fn make_boundary(parts: &Vec<AttachmentBuilder>) -> String {
+    pub fn make_boundary(parts: &[AttachmentBuilder]) -> String {
         use crate::email::compose::random::gen_boundary;
         let mut boundary = "bzz_bzz__bzz__".to_string();
         let mut random_boundary = gen_boundary();
@@ -227,7 +410,7 @@ impl ContentType {
             }
         }
 
-        boundary.extend(random_boundary.chars());
+        boundary.push_str(&random_boundary);
         /* rfc134
          * "The only mandatory parameter for the multipart Content-Type is the boundary parameter,
          * which consists of 1 to 70 characters from a set of characters known to be very robust
@@ -243,6 +426,14 @@ impl ContentType {
             _ => None,
         }
     }
+
+    pub fn parts(&self) -> Option<&[Attachment]> {
+        if let ContentType::Multipart { ref parts, .. } = self {
+            Some(parts)
+        } else {
+            None
+        }
+    }
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -319,3 +510,52 @@ impl From<&[u8]> for ContentTransferEncoding {
         }
     }
 }
+
+#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
+pub struct ContentDisposition {
+    pub kind: ContentDispositionKind,
+    pub filename: Option<String>,
+    pub creation_date: Option<String>,
+    pub modification_date: Option<String>,
+    pub read_date: Option<String>,
+    pub size: Option<String>,
+    pub parameter: Vec<String>,
+}
+
+#[derive(Clone, Debug, Copy, Serialize, Deserialize, PartialEq, Eq)]
+pub enum ContentDispositionKind {
+    Inline,
+    Attachment,
+}
+
+impl ContentDispositionKind {
+    pub fn is_inline(&self) -> bool {
+        *self == ContentDispositionKind::Inline
+    }
+
+    pub fn is_attachment(&self) -> bool {
+        *self == ContentDispositionKind::Attachment
+    }
+}
+
+impl Default for ContentDispositionKind {
+    fn default() -> Self {
+        ContentDispositionKind::Inline
+    }
+}
+
+impl Display for ContentDispositionKind {
+    fn fmt(&self, f: &mut Formatter) -> FmtResult {
+        match *self {
+            ContentDispositionKind::Inline => write!(f, "inline"),
+            ContentDispositionKind::Attachment => write!(f, "attachment"),
+        }
+    }
+}
+impl From<&[u8]> for ContentDisposition {
+    fn from(val: &[u8]) -> ContentDisposition {
+        crate::email::parser::attachments::content_disposition(val)
+            .map(|(_, v)| v)
+            .unwrap_or_default()
+    }
+}