🐝 I really like where this mua is(was?) headed, but it seems as though there has not been much activity recently.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

548 lines
22 KiB

/*
* meli - sqlite3.rs
*
* Copyright 2019 Manos Pitsidianakis
*
* This file is part of meli.
*
* meli is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* meli is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with meli. If not, see <http://www.gnu.org/licenses/>.
*/
/*! Use an sqlite3 database for fast searching.
*/
use smallvec::SmallVec;
use crate::cache::{escape_double_quote, query, Query::{self, *}};
use crate::melib::parsec::Parser;
use melib::{
backends::MailBackend,
email::{Envelope, EnvelopeHash},
log,
thread::{SortField, SortOrder},
MeliError, Result, ERROR,
};
use rusqlite::{params, Connection};
use std::path::PathBuf;
use std::convert::TryInto;
use std::sync::{Arc, RwLock};
pub fn db_path() -> Result<PathBuf> {
let data_dir =
xdg::BaseDirectories::with_prefix("meli").map_err(|e| MeliError::new(e.to_string()))?;
Ok(data_dir
.place_data_file("index.db")
.map_err(|e| MeliError::new(e.to_string()))?)
}
//#[inline(always)]
//fn fts5_bareword(w: &str) -> Cow<str> {
// if w == "AND" || w == "OR" || w == "NOT" {
// Cow::from(w)
// } else {
// if !w.is_ascii() {
// Cow::from(format!("\"{}\"", escape_double_quote(w)))
// } else {
// for &b in w.as_bytes() {
// if !(b > 0x2f && b < 0x3a)
// || !(b > 0x40 && b < 0x5b)
// || !(b > 0x60 && b < 0x7b)
// || b != 0x60
// || b != 26
// {
// return Cow::from(format!("\"{}\"", escape_double_quote(w)));
// }
// }
// Cow::from(w)
// }
// }
//}
//
pub fn open_db() -> Result<Connection> {
let db_path = db_path()?;
if !db_path.exists() {
return Err(MeliError::new("Database hasn't been initialised. Run `reindex` command"));
}
Connection::open(&db_path).map_err(|e| MeliError::new(e.to_string()))
}
pub fn open_or_create_db() -> Result<Connection> {
let db_path = db_path()?;
let mut set_mode = false;
if !db_path.exists() {
log(
format!("Creating index database in {}", db_path.display()),
melib::INFO,
);
set_mode = true;
}
let conn = Connection::open(&db_path).map_err(|e| MeliError::new(e.to_string()))?;
if set_mode {
use std::os::unix::fs::PermissionsExt;
let file = std::fs::File::open(&db_path)?;
let metadata = file.metadata()?;
let mut permissions = metadata.permissions();
permissions.set_mode(0o600); // Read/write for owner only.
file.set_permissions(permissions)?;
}
conn.execute_batch(
"CREATE TABLE IF NOT EXISTS envelopes (
id INTEGER PRIMARY KEY,
account_id INTEGER REFERENCES accounts ON UPDATE CASCADE,
hash BLOB NOT NULL UNIQUE,
date TEXT NOT NULL,
_from TEXT NOT NULL,
_to TEXT NOT NULL,
cc TEXT NOT NULL,
bcc TEXT NOT NULL,
subject TEXT NOT NULL,
message_id TEXT NOT NULL,
in_reply_to TEXT NOT NULL,
_references TEXT NOT NULL,
flags INTEGER NOT NULL,
has_attachments BOOLEAN NOT NULL,
body_text TEXT NOT NULL,
timestamp BLOB NOT NULL
);
CREATE TABLE IF NOT EXISTS folders (
id INTEGER PRIMARY KEY,
account_id INTEGER NOT NULL REFERENCES accounts ON UPDATE CASCADE,
hash BLOB NOT NULL,
date TEXT NOT NULL,
name TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS accounts (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL UNIQUE
);
CREATE TABLE IF NOT EXISTS folder_and_envelope (
folder_id INTEGER NOT NULL,
envelope_id INTEGER NOT NULL,
PRIMARY KEY (folder_id, envelope_id),
FOREIGN KEY(folder_id) REFERENCES folders(id) ON UPDATE CASCADE,
FOREIGN KEY(envelope_id) REFERENCES envelopes(id) ON UPDATE CASCADE
);
CREATE INDEX IF NOT EXISTS folder_env_idx ON folder_and_envelope(folder_id);
CREATE INDEX IF NOT EXISTS env_folder_idx ON folder_and_envelope(envelope_id);
CREATE UNIQUE INDEX IF NOT EXISTS acc_idx ON accounts(name);
CREATE INDEX IF NOT EXISTS envelope_timestamp_index ON envelopes (timestamp);
CREATE INDEX IF NOT EXISTS envelope__from_index ON envelopes (_from);
CREATE INDEX IF NOT EXISTS envelope__to_index ON envelopes (_to);
CREATE INDEX IF NOT EXISTS envelope_cc_index ON envelopes (cc);
CREATE INDEX IF NOT EXISTS envelope_bcc_index ON envelopes (bcc);
CREATE INDEX IF NOT EXISTS envelope_message_id_index ON envelopes (message_id);
CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(subject, body_text, content=envelopes, content_rowid=id);
-- Triggers to keep the FTS index up to date.
CREATE TRIGGER IF NOT EXISTS envelopes_ai AFTER INSERT ON envelopes BEGIN
INSERT INTO fts(rowid, subject, body_text) VALUES (new.id, new.subject, new.body_text);
END;
CREATE TRIGGER IF NOT EXISTS envelopes_ad AFTER DELETE ON envelopes BEGIN
INSERT INTO fts(fts, rowid, subject, body_text) VALUES('delete', old.id, old.subject, old.body_text);
END;
CREATE TRIGGER IF NOT EXISTS envelopes_au AFTER UPDATE ON envelopes BEGIN
INSERT INTO fts(fts, rowid, subject, body_text) VALUES('delete', old.id, old.subject, old.body_text);
INSERT INTO fts(rowid, subject, body_text) VALUES (new.id, new.subject, new.body_text);
END; ",
)
.map_err(|e| MeliError::new(e.to_string()))?;
Ok(conn)
}
pub fn insert(envelope: &Envelope, backend: &Arc<RwLock<Box<dyn MailBackend>>>, acc_name: &str) -> Result<()> {
let conn = open_db()?;
let backend_lck = backend.read().unwrap();
let op = backend_lck.operation(envelope.hash());
let body = match envelope.body(op) {
Ok(body) => body.text(),
Err(err) => {
debug!(
"{}",
format!(
"Failed to open envelope {}: {}",
envelope.message_id_display(),
err.to_string()
)
);
log(
format!(
"Failed to open envelope {}: {}",
envelope.message_id_display(),
err.to_string()
),
ERROR,
);
return Err(err);
}
};
if let Err(err) = conn.execute("INSERT OR IGNORE INTO accounts (name) VALUES (?1)", params![acc_name, ]) {
debug!(
"Failed to insert envelope {}: {}",
envelope.message_id_display(),
err.to_string()
);
log(
format!(
"Failed to insert envelope {}: {}",
envelope.message_id_display(),
err.to_string()
),
ERROR,
);
return Err(MeliError::new(err.to_string()));
}
let account_id: i32 = {
let mut stmt = conn.prepare("SELECT id FROM accounts WHERE name = ?").unwrap();
let x = stmt.query_map(params![acc_name], |row| row.get(0)).unwrap().next().unwrap().unwrap();
x
};
if let Err(err) = conn.execute(
"INSERT OR REPLACE INTO envelopes (account_id, hash, date, _from, _to, cc, bcc, subject, message_id, in_reply_to, _references, flags, has_attachments, body_text, timestamp)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)",
params![account_id, envelope.hash().to_be_bytes().to_vec(), envelope.date_as_str(), envelope.field_from_to_string(), envelope.field_to_to_string(), envelope.field_cc_to_string(), envelope.field_bcc_to_string(), envelope.subject().into_owned().trim_end_matches('\u{0}'), envelope.message_id_display().to_string(), envelope.in_reply_to_display().map(|f| f.to_string()).unwrap_or(String::new()), envelope.field_references_to_string(), i64::from(envelope.flags().bits()), if envelope.has_attachments() { 1 } else { 0 }, body, envelope.date().to_be_bytes().to_vec()],
)
.map_err(|e| MeliError::new(e.to_string())) {
debug!(
"Failed to insert envelope {}: {}",
envelope.message_id_display(),
err.to_string()
);
log(
format!(
"Failed to insert envelope {}: {}",
envelope.message_id_display(),
err.to_string()
),
ERROR,
);
}
Ok(())
}
pub fn remove(env_hash: EnvelopeHash) -> Result<()> {
let conn = open_db()?;
if let Err(err) = conn.execute(
"DELETE FROM envelopes WHERE hash = ?",
params![env_hash.to_be_bytes().to_vec(), ])
.map_err(|e| MeliError::new(e.to_string())) {
debug!(
"Failed to remove envelope {}: {}",
env_hash,
err.to_string()
);
log(
format!(
"Failed to remove envelope {}: {}",
env_hash,
err.to_string()
),
ERROR,
);
return Err(err);
}
Ok(())
}
pub fn index(context: &mut crate::state::Context, account_name: &str) -> Result<()> {
let account = if let Some(a) = context.accounts.iter().find(|acc| acc.name() == account_name) {
a} else {
return Err(MeliError::new(format!("Account {} was not found.", account_name)));
};
let (acc_name, acc_mutex, backend_mutex):( String, Arc<RwLock<_>>, Arc<_>) = if *account.settings.conf.cache_type() != crate::conf::CacheType::Sqlite3 {
return Err(MeliError::new(format!("Account {} doesn't have an sqlite3 search backend.", account_name)));
} else {
(
account.name().to_string(),
account.collection.envelopes.clone(),
account.backend.clone(),
)};
let conn = open_or_create_db()?;
let work_context = context.work_controller().get_context();
let env_hashes =
acc_mutex.read().unwrap().keys().cloned().collect::<Vec<_>>();
/* Sleep, index and repeat in order not to block the main process */
let handle = std::thread::Builder::new().name(String::from("rebuilding index")).spawn(move || {
let thread_id = std::thread::current().id();
let sleep_dur = std::time::Duration::from_millis(20);
if let Err(err) = conn.execute(
"INSERT OR REPLACE INTO accounts (name) VALUES (?1)", params![acc_name.as_str(),],).map_err(|e| MeliError::new(e.to_string())) {
debug!("{}",
format!(
"Failed to update index: {}",
err.to_string()
));
log(
format!(
"Failed to update index: {}",
err.to_string()
),
ERROR,
);
}
let account_id: i32 = {
let mut stmt = conn.prepare("SELECT id FROM accounts WHERE name = ?").unwrap();
let x = stmt.query_map(params![acc_name.as_str()], |row| row.get(0)).unwrap().next().unwrap().unwrap();
x
};
let mut ctr = 0;
debug!("{}", format!("Rebuilding {} index. {}/{}", acc_name, ctr, env_hashes.len()));
work_context
.set_status
.send((thread_id, format!("Rebuilding {} index. {}/{}", acc_name, ctr, env_hashes.len())))
.unwrap();
for chunk in env_hashes.chunks(200) {
ctr += chunk.len();
let envelopes_lck = acc_mutex.read().unwrap();
let backend_lck = backend_mutex.read().unwrap();
for env_hash in chunk {
if let Some(e) = envelopes_lck.get(&env_hash) {
let op = backend_lck.operation(e.hash());
let body = match e.body(op) {
Ok(body) => body.text(),
Err(err) => {
debug!("{}",
format!(
"Failed to open envelope {}: {}",
e.message_id_display(),
err.to_string()
));
log(
format!(
"Failed to open envelope {}: {}",
e.message_id_display(),
err.to_string()
),
ERROR,
);
return;
}
};
if let Err(err) = conn.execute(
"INSERT OR REPLACE INTO envelopes (account_id, hash, date, _from, _to, cc, bcc, subject, message_id, in_reply_to, _references, flags, has_attachments, body_text, timestamp)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)",
params![account_id, e.hash().to_be_bytes().to_vec(), e.date_as_str(), e.field_from_to_string(), e.field_to_to_string(), e.field_cc_to_string(), e.field_bcc_to_string(), e.subject().into_owned().trim_end_matches('\u{0}'), e.message_id_display().to_string(), e.in_reply_to_display().map(|f| f.to_string()).unwrap_or(String::new()), e.field_references_to_string(), i64::from(e.flags().bits()), if e.has_attachments() { 1 } else { 0 }, body, e.date().to_be_bytes().to_vec()],
)
.map_err(|e| MeliError::new(e.to_string())) {
debug!("{}",
format!(
"Failed to insert envelope {}: {}",
e.message_id_display(),
err.to_string()
));
log(
format!(
"Failed to insert envelope {}: {}",
e.message_id_display(),
err.to_string()
),
ERROR,
);
}
}
}
drop(envelopes_lck);
work_context
.set_status
.send((thread_id, format!("Rebuilding {} index. {}/{}", acc_name, ctr, env_hashes.len())))
.unwrap();
std::thread::sleep(sleep_dur);
}
work_context.finished.send(thread_id).unwrap();
})?;
context.work_controller().static_threads.lock()?.insert(
handle.thread().id(),
String::from("Rebuilding sqlite3 index").into(),
);
Ok(())
}
pub fn search(
term: &str,
(sort_field, sort_order): (SortField, SortOrder),
) -> Result<SmallVec<[EnvelopeHash; 512]>> {
let conn = open_db()?;
let sort_field = match debug!(sort_field) {
SortField::Subject => "subject",
SortField::Date => "timestamp",
};
let sort_order = match debug!(sort_order) {
SortOrder::Asc => "ASC",
SortOrder::Desc => "DESC",
};
let mut stmt = conn
.prepare(
debug!(format!(
"SELECT hash FROM envelopes WHERE {} ORDER BY {} {};",
query_to_sql(&query().parse(term)?.1),
sort_field,
sort_order
))
.as_str(),
)
.map_err(|e| MeliError::new(e.to_string()))?;
let results = stmt
.query_map(rusqlite::NO_PARAMS, |row| Ok(row.get(0)?))
.map_err(|e| MeliError::new(e.to_string()))?
.map(|r: std::result::Result<Vec<u8>, rusqlite::Error>| {
Ok(u64::from_be_bytes(
r.map_err(|e| MeliError::new(e.to_string()))?
.as_slice()
.try_into()
.map_err(|e: std::array::TryFromSliceError| MeliError::new(e.to_string()))?,
))
})
.collect::<Result<SmallVec<[EnvelopeHash; 512]>>>();
results
}
/// Translates a `Query` to an Sqlite3 expression in a `String`.
pub fn query_to_sql(q: &Query) -> String {
fn rec(q: &Query, s: &mut String) {
match q {
Subject(t) => {
s.push_str("subject LIKE \"%");
s.extend(escape_double_quote(t).chars());
s.push_str("%\" ");
}
From(t) => {
s.push_str("_from LIKE \"%");
s.extend(escape_double_quote(t).chars());
s.push_str("%\" ");
}
To(t) => {
s.push_str("_to LIKE \"%");
s.extend(escape_double_quote(t).chars());
s.push_str("%\" ");
}
Cc(t) => {
s.push_str("cc LIKE \"%");
s.extend(escape_double_quote(t).chars());
s.push_str("%\" ");
}
Bcc(t) => {
s.push_str("bcc LIKE \"%");
s.extend(escape_double_quote(t).chars());
s.push_str("%\" ");
}
AllText(t) => {
s.push_str("body_text LIKE \"%");
s.extend(escape_double_quote(t).chars());
s.push_str("%\" ");
}
And(q1, q2) => {
s.push_str("(");
rec(q1, s);
s.push_str(") AND (");
rec(q2, s);
s.push_str(") ");
}
Or(q1, q2) => {
s.push_str("(");
rec(q1, s);
s.push_str(") OR (");
rec(q2, s);
s.push_str(") ");
}
Not(q) => {
s.push_str("NOT (");
rec(q, s);
s.push_str(") ");
}
Flags(v) => {
let total = v.len();
if total > 1 {
s.push_str("(");
}
for (i, f) in v.iter().enumerate() {
match f.as_str() {
"draft" => {
s.push_str(" (flags & 8 > 0) ");
}
"deleted" | "trashed" => {
s.push_str(" (flags & 6 > 0) ");
}
"flagged" => {
s.push_str(" (flags & 16 > 0) ");
}
"recent" => {
s.push_str(" (flags & 4 == 0) ");
}
"seen" | "read" => {
s.push_str(" (flags & 4 > 0) ");
}
"unseen" | "unread" => {
s.push_str(" (flags & 4 == 0) ");
}
"answered" | "replied" => {
s.push_str(" (flags & 2 > 0) ");
}
"unanswered" => {
s.push_str(" (flags & 2 == 0) ");
}
_ => {
continue;
}
}
if total > 1 && i != total - 1 {
s.push_str(" AND ");
}
}
if total > 1 {
s.push_str(") ");
}
}
HasAttachment => {
s.push_str("has_attachments == 1 ");
}
_ => {}
}
}
let mut ret = String::new();
rec(q, &mut ret);
ret
}
#[test]
fn test_query_to_sql() {
assert_eq!(
"(subject LIKE \"%test%\" ) AND (body_text LIKE \"%i%\" ) ",
&query_to_sql(&query().parse_complete("subject: test and i").unwrap().1)
);
assert_eq!(
"(subject LIKE \"%github%\" ) OR ((_from LIKE \"%epilys%\" ) AND ((subject LIKE \"%lib%\" ) OR (subject LIKE \"%meli%\" ) ) ) ",
&query_to_sql(
&query()
.parse_complete(
"subject: github or (from: epilys and (subject:lib or subject: meli))"
)
.unwrap()
.1
)
);
}