meli
/
meli
8
Fork 13
Code Issues 10 Pull Requests 10 Actions Packages Projects 1 Releases 18 Wiki Activity

melib/nntp: add support for storing read status locally

pull/260/head
Manos Pitsidianakis 2023-07-16 11:37:09 +03:00
parent 519257b08f
commit e9cd800f49
Signed by: Manos Pitsidianakis
GPG Key ID: 7729C7707F7E09D0
14 changed files with 283 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
meli/docs/meli.conf.5
View File

@ -401,6 +401,10 @@ The port to connect to
.Pq Em optional
Do not validate TLS certificates.
.Pq Em false \" default value
.It Ic store_flags_locally Ar boolean
.Pq Em optional
Store seen status locally in an sqlite3 database.
.Pq Em true \" default value
.El
.Pp
You have to explicitly state the groups you want to see in the

14
meli/src/components/mail/listing/compact.rs
View File

@ -479,9 +479,10 @@ impl MailListingTrait for CompactListing {
+ 1
+ entry_strings.subject.grapheme_width()
+ 1
+ entry_strings.tags.grapheme_width())
.try_into()
.unwrap_or(255),
+ entry_strings.tags.grapheme_width()
+ 16)
.try_into()
.unwrap_or(255),
);
min_width.1 = cmp::max(min_width.1, entry_strings.date.grapheme_width()); /* date */
min_width.2 = cmp::max(min_width.2, entry_strings.from.grapheme_width()); /* from */
@ -491,7 +492,8 @@ impl MailListingTrait for CompactListing {
+ 1
+ entry_strings.subject.grapheme_width()
+ 1
+ entry_strings.tags.grapheme_width(),
+ entry_strings.tags.grapheme_width()
+ 16,
); /* subject */
self.rows.insert_thread(
thread,
@ -1269,7 +1271,9 @@ impl CompactListing {
((x, idx), (min_width.3, idx)),
None,
);
columns[3][(x, idx)].set_bg(row_attr.bg).set_ch(' ');
if let Some(c) = columns[3].get_mut(x, idx) {
c.set_bg(row_attr.bg).set_ch(' ');
}
let x = {
let mut x = x + 1;
for (t, &color) in strings.tags.split_whitespace().zip(strings.tags.1.iter()) {

5
meli/src/conf/accounts.rs
View File

@ -53,8 +53,9 @@ use melib::{
use smallvec::SmallVec;
use super::{AccountConf, FileMailboxConf};
#[cfg(feature = "sqlite3")]
use crate::command::actions::AccountAction;
use crate::{
command::actions::AccountAction,
jobs::{JobId, JoinHandle},
types::UIEvent::{self, EnvelopeRemove, EnvelopeRename, EnvelopeUpdate, Notification},
MainLoopHandler, StatusEvent, ThreadEvent,
@ -1328,7 +1329,7 @@ impl Account {
&mut self,
message: String,
send_mail: crate::conf::composing::SendMail,
complete_in_background: bool,
#[allow(unused_variables)] complete_in_background: bool,
) -> Result<Option<JoinHandle<Result<()>>>> {
use std::{
io::Write,

1
meli/src/conf/themes.rs
View File

@ -1003,6 +1003,7 @@ impl<'de> Deserialize<'de> for Themes {
impl Themes {
fn validate_keys(name: &str, theme: &Theme, hash_set: &HashSet<&'static str>) -> Result<()> {
#[allow(unused_mut)]
let mut keys = theme
.keys()
.filter_map(|k| {

2
meli/src/state.rs
View File

@ -949,7 +949,7 @@ impl State {
}
}
#[cfg(not(feature = "sqlite3"))]
AccountAction(ref account_name, ReIndex) => {
AccountAction(_, ReIndex) => {
self.context.replies.push_back(UIEvent::Notification(
None,
"Message index rebuild failed: meli is not built with sqlite3 support."

3
meli/src/subcommands.rs
View File

@ -19,8 +19,9 @@
* along with meli. If not, see <http://www.gnu.org/licenses/>.
*/
#[cfg(feature = "cli-docs")]
use std::io::prelude::*;
use std::{
io::prelude::*,
path::PathBuf,
process::{Command, Stdio},
};

2
melib/Cargo.toml
View File

@ -69,7 +69,7 @@ mailin-embedded = { version = "0.7", features = ["rtls"] }
stderrlog = "^0.5"
[features]
default = ["unicode_algorithms", "imap_backend", "maildir_backend", "mbox_backend", "vcard", "sqlite3", "smtp", "deflate_compression"]
default = ["unicode_algorithms", "imap_backend", "maildir_backend", "mbox_backend", "vcard", "smtp", "deflate_compression"]
debug-tracing = []
deflate_compression = ["flate2", "imap-codec/ext_compress"]

4
melib/src/backends.rs
View File

@ -697,6 +697,10 @@ impl fmt::Debug for LazyCountSet {
}
impl LazyCountSet {
pub fn new() -> Self {
Self::default()
}
pub fn set_not_yet_seen(&mut self, new_val: usize) {
self.not_yet_seen = new_val;
}

4
melib/src/backends/imap.rs
View File

@ -35,7 +35,6 @@ pub use watch::*;
mod search;
pub use search::*;
mod cache;
use cache::{ImapCacheReset, ModSequence};
pub mod error;
pub mod managesieve;
mod untagged;
@ -50,6 +49,9 @@ use std::{
time::{Duration, SystemTime},
};
#[cfg(feature = "sqlite3")]
use cache::ImapCacheReset;
use cache::ModSequence;
use futures::{lock::Mutex as FutureMutex, stream::Stream};
use imap_codec::{
command::CommandBody,

4
melib/src/backends/imap/cache.rs
View File

@ -751,14 +751,14 @@ mod default_m {
}
impl ImapCacheReset for DefaultCache {
fn reset_db(uid_store: &UIDStore) -> Result<()> {
fn reset_db(_: &UIDStore) -> Result<()> {
Err(Error::new("melib is not built with any imap cache").set_kind(ErrorKind::Bug))
}
}
impl ImapCache for DefaultCache {
fn reset(&mut self) -> Result<()> {
DefaultCache::reset_db(&self.uid_store)
Ok(())
}
fn mailbox_state(&mut self, _mailbox_hash: MailboxHash) -> Result<Option<()>> {

130
melib/src/backends/nntp.rs
View File

@ -29,6 +29,7 @@
use smallvec::SmallVec;
use crate::{get_conf_val, get_path_hash};
mod store;
#[macro_use]
mod protocol_parser;
pub use protocol_parser::*;
@ -56,6 +57,7 @@ use crate::{
error::{Error, Result, ResultIntoError},
utils::futures::timeout,
Collection,
RefreshEventKind::NewFlags,
};
pub type UID = usize;
@ -121,13 +123,14 @@ pub struct UIDStore {
account_hash: AccountHash,
account_name: Arc<str>,
capabilities: Arc<Mutex<Capabilities>>,
message_id_index: Arc<Mutex<HashMap<String, EnvelopeHash>>>,
message_id_index: Arc<FutureMutex<HashMap<String, EnvelopeHash>>>,
hash_index: Arc<Mutex<HashMap<EnvelopeHash, (UID, MailboxHash)>>>,
uid_index: Arc<Mutex<HashMap<(MailboxHash, UID), EnvelopeHash>>>,
uid_index: Arc<FutureMutex<HashMap<(MailboxHash, UID), EnvelopeHash>>>,
collection: Collection,
store: Arc<FutureMutex<Option<store::Store>>>,
mailboxes: Arc<FutureMutex<HashMap<MailboxHash, NntpMailbox>>>,
is_online: Arc<Mutex<(Instant, Result<()>)>>,
is_online: Arc<FutureMutex<(Instant, Result<()>)>>,
event_consumer: BackendEventConsumer,
}
@ -141,13 +144,14 @@ impl UIDStore {
account_hash,
account_name,
event_consumer,
store: Default::default(),
capabilities: Default::default(),
message_id_index: Default::default(),
hash_index: Default::default(),
uid_index: Default::default(),
mailboxes: Arc::new(FutureMutex::new(Default::default())),
collection: Collection::new(),
is_online: Arc::new(Mutex::new((
is_online: Arc::new(FutureMutex::new((
Instant::now(),
Err(Error::new("Account is uninitialised.")),
))),
@ -285,6 +289,7 @@ impl MailBackend for NntpType {
let mut res = String::with_capacity(8 * 1024);
let mut conn = timeout(Some(Duration::from_secs(60 * 16)), connection.lock()).await?;
if let Some(mut latest_article) = latest_article {
let mut unseen = LazyCountSet::new();
let timestamp = latest_article - 10 * 60;
let datetime_str = crate::utils::datetime::timestamp_to_string_utc(
timestamp,
@ -299,7 +304,7 @@ impl MailBackend for NntpType {
.await?;
conn.read_response(&mut res, true, &["230 "]).await?;
let message_ids = {
let message_id_lck = uid_store.message_id_index.lock().unwrap();
let message_id_lck = uid_store.message_id_index.lock().await;
res.split_rn()
.skip(1)
.map(|s| s.trim())
@ -315,11 +320,21 @@ impl MailBackend for NntpType {
conn.send_command(format!("OVER {}", msg_id).as_bytes())
.await?;
conn.read_response(&mut res, true, &["224 "]).await?;
let mut message_id_lck = uid_store.message_id_index.lock().unwrap();
let mut message_id_lck = uid_store.message_id_index.lock().await;
let mut uid_index_lck = uid_store.uid_index.lock().await;
let store_lck = uid_store.store.lock().await;
let mut hash_index_lck = uid_store.hash_index.lock().unwrap();
let mut uid_index_lck = uid_store.uid_index.lock().unwrap();
for l in res.split_rn().skip(1) {
let (_, (num, env)) = protocol_parser::over_article(l)?;
let (_, (num, mut env)) = protocol_parser::over_article(l)?;
if let Some(s) = store_lck.as_ref() {
env.set_flags(s.flags(env.hash(), mailbox_hash, num)?);
if !env.is_seen() {
unseen.insert_new(env.hash());
}
} else {
unseen.insert_new(env.hash());
}
env_hash_set.insert(env.hash());
message_id_lck.insert(env.message_id_display().to_string(), env.hash());
hash_index_lck.insert(env.hash(), (num, mailbox_hash));
@ -342,7 +357,7 @@ impl MailBackend for NntpType {
.lock()
.unwrap()
.insert_existing_set(env_hash_set.clone());
f.unseen.lock().unwrap().insert_existing_set(env_hash_set);
f.unseen.lock().unwrap().insert_set(unseen.set);
}
return Ok(());
}
@ -432,11 +447,56 @@ impl MailBackend for NntpType {
fn set_flags(
&mut self,
_env_hashes: EnvelopeHashBatch,
_mailbox_hash: MailboxHash,
_flags: SmallVec<[(std::result::Result<Flag, String>, bool); 8]>,
env_hashes: EnvelopeHashBatch,
mailbox_hash: MailboxHash,
flags: SmallVec<[(std::result::Result<Flag, String>, bool); 8]>,
) -> ResultFuture<()> {
Err(Error::new("NNTP doesn't support flags."))
let uid_store = self.uid_store.clone();
Ok(Box::pin(async move {
let uids: SmallVec<[(EnvelopeHash, UID); 64]> = {
let hash_index_lck = uid_store.hash_index.lock().unwrap();
env_hashes
.iter()
.filter_map(|env_hash| {
hash_index_lck
.get(&env_hash)
.cloned()
.map(|(uid, _)| (env_hash, uid))
})
.collect()
};
if uids.is_empty() {
return Ok(());
}
let fsets = &uid_store.mailboxes.lock().await[&mailbox_hash];
let store_lck = uid_store.store.lock().await;
if let Some(s) = store_lck.as_ref() {
for (flag, on) in flags {
if let Ok(f) = flag {
for (env_hash, uid) in &uids {
let mut current_val = s.flags(*env_hash, mailbox_hash, *uid)?;
current_val.set(f, on);
if !current_val.intersects(Flag::SEEN) {
fsets.unseen.lock().unwrap().insert_new(*env_hash);
} else {
fsets.unseen.lock().unwrap().remove(*env_hash);
}
s.set_flags(*env_hash, mailbox_hash, *uid, current_val)?;
(uid_store.event_consumer)(
uid_store.account_hash,
BackendEvent::Refresh(RefreshEvent {
account_hash: uid_store.account_hash,
mailbox_hash,
kind: NewFlags(*env_hash, (current_val, vec![])),
}),
);
}
}
}
}
Ok(())
}))
}
fn delete_messages(
@ -588,6 +648,16 @@ impl NntpType {
let danger_accept_invalid_certs: bool =
get_conf_val!(s["danger_accept_invalid_certs"], false)?;
let require_auth = get_conf_val!(s["require_auth"], false)?;
let store_flags_locally = get_conf_val!(s["store_flags_locally"], true)?;
#[cfg(not(feature = "sqlite3"))]
if store_flags_locally {
return Err(Error::new(format!(
"{}: store_flags_locally is on but this copy of melib isn't built with sqlite3 \
support.",
&s.name
)));
}
let server_conf = NntpServerConf {
server_hostname: server_hostname.to_string(),
server_username: if require_auth {
@ -639,6 +709,11 @@ impl NntpType {
}
let uid_store: Arc<UIDStore> = Arc::new(UIDStore {
mailboxes: Arc::new(FutureMutex::new(mailboxes)),
store: if store_flags_locally {
Arc::new(FutureMutex::new(Some(store::Store::new(&s.name)?)))
} else {
Default::default()
},
..UIDStore::new(account_hash, account_name, event_consumer)
});
let connection = NntpConnection::new_connection(&server_conf, uid_store.clone());
@ -724,6 +799,16 @@ impl NntpType {
.unwrap_or_else(|| Ok($default))
}};
}
#[cfg(feature = "sqlite3")]
get_conf_val!(s["store_flags_locally"], true)?;
#[cfg(not(feature = "sqlite3"))]
if get_conf_val!(s["store_flags_locally"], false)? {
return Err(Error::new(format!(
"{}: store_flags_locally is on but this copy of melib isn't built with sqlite3 \
support.",
&s.name
)));
}
get_conf_val!(s["require_auth"], false)?;
get_conf_val!(s["server_hostname"])?;
get_conf_val!(s["server_username"], String::new())?;
@ -804,6 +889,7 @@ impl FetchState {
let mailbox_hash = *mailbox_hash;
let mut res = String::with_capacity(8 * 1024);
let mut conn = connection.lock().await;
let mut unseen = LazyCountSet::new();
if high_low_total.is_none() {
conn.select_group(mailbox_hash, true, &mut res).await?;
/*
@ -830,7 +916,6 @@ impl FetchState {
{
let f = &uid_store.mailboxes.lock().await[&mailbox_hash];
f.exists.lock().unwrap().set_not_yet_seen(total);
f.unseen.lock().unwrap().set_not_yet_seen(total);
};
}
let (high, low, _) = high_low_total.unwrap();
@ -857,11 +942,20 @@ impl FetchState {
//uid_index: Arc<Mutex<HashMap<(MailboxHash, UID), EnvelopeHash>>>,
let mut latest_article: Option<crate::UnixTimestamp> = None;
{
let mut message_id_lck = uid_store.message_id_index.lock().unwrap();
let mut message_id_lck = uid_store.message_id_index.lock().await;
let mut uid_index_lck = uid_store.uid_index.lock().await;
let store_lck = uid_store.store.lock().await;
let mut hash_index_lck = uid_store.hash_index.lock().unwrap();
let mut uid_index_lck = uid_store.uid_index.lock().unwrap();
for l in res.split_rn().skip(1) {
let (_, (num, env)) = protocol_parser::over_article(l)?;
let (_, (num, mut env)) = protocol_parser::over_article(l)?;
if let Some(s) = store_lck.as_ref() {
env.set_flags(s.flags(env.hash(), mailbox_hash, num)?);
if !env.is_seen() {
unseen.insert_new(env.hash());
}
} else {
unseen.insert_new(env.hash());
}
message_id_lck.insert(env.message_id_display().to_string(), env.hash());
hash_index_lck.insert(env.hash(), (num, mailbox_hash));
uid_index_lck.insert((mailbox_hash, num), env.hash());
@ -881,7 +975,7 @@ impl FetchState {
.lock()
.unwrap()
.insert_existing_set(hash_set.clone());
f.unseen.lock().unwrap().insert_existing_set(hash_set);
*f.unseen.lock().unwrap() = unseen;
};
Ok(Some(ret))
}

8
melib/src/backends/nntp/connection.rs
View File

@ -420,21 +420,21 @@ impl NntpConnection {
pub fn connect<'a>(&'a mut self) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'a>> {
Box::pin(async move {
if let (instant, ref mut status @ Ok(())) = *self.uid_store.is_online.lock().unwrap() {
if let (instant, ref mut status @ Ok(())) = *self.uid_store.is_online.lock().await {
if Instant::now().duration_since(instant) >= std::time::Duration::new(60 * 30, 0) {
*status = Err(Error::new("Connection timed out"));
self.stream = Err(Error::new("Connection timed out"));
}
}
if self.stream.is_ok() {
self.uid_store.is_online.lock().unwrap().0 = Instant::now();
self.uid_store.is_online.lock().await.0 = Instant::now();
return Ok(());
}
let new_stream = NntpStream::new_connection(&self.server_conf).await;
if let Err(err) = new_stream.as_ref() {
*self.uid_store.is_online.lock().unwrap() = (Instant::now(), Err(err.clone()));
*self.uid_store.is_online.lock().await = (Instant::now(), Err(err.clone()));
} else {
*self.uid_store.is_online.lock().unwrap() = (Instant::now(), Ok(()));
*self.uid_store.is_online.lock().await = (Instant::now(), Ok(()));
}
let (capabilities, stream) = new_stream?;
self.stream = Ok(stream);

1
melib/src/backends/nntp/protocol_parser.rs
View File

@ -124,7 +124,6 @@ pub fn over_article(input: &str) -> IResult<&str, (UID, Envelope)> {
EnvelopeHash(hasher.finish())
};
let mut env = Envelope::new(env_hash);
env.set_seen();
if let Some(date) = date {
env.set_date(date.as_bytes());
if let Ok(d) =

137
melib/src/backends/nntp/store.rs 100644
View File

@ -0,0 +1,137 @@
/*
* meli - nntp module.
*
* Copyright 2023 Manos Pitsidianakis
*
* This file is part of meli.
*
* meli is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* meli is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with meli. If not, see <http://www.gnu.org/licenses/>.
*/
//! Store article seen/read flags in an sqlite3 database, since NNTP has no
//! concept of server-side flag bookkeeping.
pub use inner::*;
#[cfg(feature = "sqlite3")]
mod inner {
use crate::{
backends::nntp::UID,
email::Flag,
utils::sqlite3::{self, Connection, DatabaseDescription},
EnvelopeHash, MailboxHash, Result,
};
pub const DB_DESCRIPTION: DatabaseDescription = DatabaseDescription {
name: "nntp_store.db",
init_script: Some(
"PRAGMA foreign_keys = true;
PRAGMA encoding = 'UTF-8';
CREATE TABLE IF NOT EXISTS article (
hash INTEGER NOT NULL,
mailbox_hash INTEGER NOT NULL,
uid INTEGER NOT NULL,
flags INTEGER NOT NULL DEFAULT 0,
tags TEXT,
PRIMARY KEY (mailbox_hash, uid)
);
CREATE INDEX IF NOT EXISTS article_uid_idx ON article(mailbox_hash, uid);
CREATE INDEX IF NOT EXISTS article_idx ON article(hash);",
),
version: 1,
};
#[derive(Debug)]
pub struct Store {
connection: Connection,
}
impl Store {
pub fn new(id: &str) -> Result<Self> {
Ok(Self {
connection: sqlite3::open_or_create_db(&DB_DESCRIPTION, Some(id))?,
})
}
pub fn set_flags(
&self,
envelope_hash: EnvelopeHash,
mailbox_hash: MailboxHash,
uid: UID,
new_value: Flag,
) -> Result<()> {
self.connection.execute(
"INSERT OR REPLACE INTO article(hash, mailbox_hash, uid, flags) VALUES (?, ?, ?, \
?)",
sqlite3::params![&envelope_hash, &mailbox_hash, &uid, &new_value.bits()],
)?;
Ok(())
}
pub fn flags(
&self,
envelope_hash: EnvelopeHash,
mailbox_hash: MailboxHash,
uid: UID,
) -> Result<Flag> {
self.connection.execute(
"INSERT OR IGNORE INTO article(hash,mailbox_hash,uid,flags) VALUES(?1,?2,?3,0);",
sqlite3::params![&envelope_hash, &mailbox_hash, &uid],
)?;
let mut stmt = self.connection.prepare(
"SELECT flags FROM article WHERE hash = ?1 AND mailbox_hash = ?2 AND uid = ?3;",
)?;
Ok(Flag::from_bits({
stmt.query_row(
sqlite3::params![&envelope_hash, &mailbox_hash, &uid],
|row| {
let flag: u8 = row.get(0)?;
Ok(flag)
},
)?
})
.unwrap_or_default())
}
}
}
#[cfg(not(feature = "sqlite3"))]
mod inner {
use crate::{
backends::nntp::UID, email::Flag, EnvelopeHash, Error, ErrorKind, MailboxHash, Result,
};
#[derive(Debug)]
pub struct Store;
impl Store {
pub fn new(_: &str) -> Result<Self> {
Ok(Self)
}
pub fn set_flags(&self, _: EnvelopeHash, _: MailboxHash, _: UID, _: Flag) -> Result<()> {
Ok(())
}
pub fn flags(&self, _: EnvelopeHash, _: MailboxHash, _: UID) -> Result<Flag> {
Err(Error::new(
"NNTP store flag cache accessed but this copy of melib isn't built with sqlite3 \
support. This is a bug and should be reported.",
)
.set_kind(ErrorKind::Bug))
}
}
}