519 lines
18 KiB
Rust
519 lines
18 KiB
Rust
/*
|
|
* meli - sqlite3.rs
|
|
*
|
|
* Copyright 2019 Manos Pitsidianakis
|
|
*
|
|
* This file is part of meli.
|
|
*
|
|
* meli is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* meli is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with meli. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
/*! Use an sqlite3 database for fast searching.
|
|
*/
|
|
use crate::melib::ResultIntoMeliError;
|
|
use melib::search::{
|
|
escape_double_quote,
|
|
Query::{self, *},
|
|
};
|
|
use melib::{
|
|
backends::{MailBackend, ResultFuture},
|
|
email::{Envelope, EnvelopeHash},
|
|
log,
|
|
sqlite3::{
|
|
self as melib_sqlite3,
|
|
rusqlite::{self, params},
|
|
DatabaseDescription,
|
|
},
|
|
thread::{SortField, SortOrder},
|
|
MeliError, Result, ERROR,
|
|
};
|
|
|
|
use smallvec::SmallVec;
|
|
use std::convert::TryInto;
|
|
use std::path::PathBuf;
|
|
use std::sync::{Arc, RwLock};
|
|
|
|
const DB: DatabaseDescription = DatabaseDescription {
|
|
name: "index.db",
|
|
init_script:Some( "CREATE TABLE IF NOT EXISTS envelopes (
|
|
id INTEGER PRIMARY KEY,
|
|
account_id INTEGER REFERENCES accounts ON UPDATE CASCADE,
|
|
hash BLOB NOT NULL UNIQUE,
|
|
date TEXT NOT NULL,
|
|
_from TEXT NOT NULL,
|
|
_to TEXT NOT NULL,
|
|
cc TEXT NOT NULL,
|
|
bcc TEXT NOT NULL,
|
|
subject TEXT NOT NULL,
|
|
message_id TEXT NOT NULL,
|
|
in_reply_to TEXT NOT NULL,
|
|
_references TEXT NOT NULL,
|
|
flags INTEGER NOT NULL,
|
|
has_attachments BOOLEAN NOT NULL,
|
|
body_text TEXT NOT NULL,
|
|
timestamp BLOB NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS folders (
|
|
id INTEGER PRIMARY KEY,
|
|
account_id INTEGER NOT NULL REFERENCES accounts ON UPDATE CASCADE,
|
|
hash BLOB NOT NULL,
|
|
date TEXT NOT NULL,
|
|
name TEXT NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS accounts (
|
|
id INTEGER PRIMARY KEY,
|
|
name TEXT NOT NULL UNIQUE
|
|
);
|
|
CREATE TABLE IF NOT EXISTS folder_and_envelope (
|
|
folder_id INTEGER NOT NULL,
|
|
envelope_id INTEGER NOT NULL,
|
|
PRIMARY KEY (folder_id, envelope_id),
|
|
FOREIGN KEY(folder_id) REFERENCES folders(id) ON UPDATE CASCADE,
|
|
FOREIGN KEY(envelope_id) REFERENCES envelopes(id) ON UPDATE CASCADE
|
|
);
|
|
CREATE INDEX IF NOT EXISTS folder_env_idx ON folder_and_envelope(folder_id);
|
|
CREATE INDEX IF NOT EXISTS env_folder_idx ON folder_and_envelope(envelope_id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS acc_idx ON accounts(name);
|
|
|
|
|
|
CREATE INDEX IF NOT EXISTS envelope_timestamp_index ON envelopes (timestamp);
|
|
CREATE INDEX IF NOT EXISTS envelope__from_index ON envelopes (_from);
|
|
CREATE INDEX IF NOT EXISTS envelope__to_index ON envelopes (_to);
|
|
CREATE INDEX IF NOT EXISTS envelope_cc_index ON envelopes (cc);
|
|
CREATE INDEX IF NOT EXISTS envelope_bcc_index ON envelopes (bcc);
|
|
CREATE INDEX IF NOT EXISTS envelope_message_id_index ON envelopes (message_id);
|
|
|
|
CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(subject, body_text, content=envelopes, content_rowid=id);
|
|
|
|
-- Triggers to keep the FTS index up to date.
|
|
CREATE TRIGGER IF NOT EXISTS envelopes_ai AFTER INSERT ON envelopes BEGIN
|
|
INSERT INTO fts(rowid, subject, body_text) VALUES (new.id, new.subject, new.body_text);
|
|
END;
|
|
|
|
CREATE TRIGGER IF NOT EXISTS envelopes_ad AFTER DELETE ON envelopes BEGIN
|
|
INSERT INTO fts(fts, rowid, subject, body_text) VALUES('delete', old.id, old.subject, old.body_text);
|
|
END;
|
|
|
|
CREATE TRIGGER IF NOT EXISTS envelopes_au AFTER UPDATE ON envelopes BEGIN
|
|
INSERT INTO fts(fts, rowid, subject, body_text) VALUES('delete', old.id, old.subject, old.body_text);
|
|
INSERT INTO fts(rowid, subject, body_text) VALUES (new.id, new.subject, new.body_text);
|
|
END; "),
|
|
version: 1,
|
|
};
|
|
|
|
pub fn db_path() -> Result<PathBuf> {
|
|
melib_sqlite3::db_path(DB.name)
|
|
}
|
|
|
|
//#[inline(always)]
|
|
//fn fts5_bareword(w: &str) -> Cow<str> {
|
|
// if w == "AND" || w == "OR" || w == "NOT" {
|
|
// Cow::from(w)
|
|
// } else {
|
|
// if !w.is_ascii() {
|
|
// Cow::from(format!("\"{}\"", escape_double_quote(w)))
|
|
// } else {
|
|
// for &b in w.as_bytes() {
|
|
// if !(b > 0x2f && b < 0x3a)
|
|
// || !(b > 0x40 && b < 0x5b)
|
|
// || !(b > 0x60 && b < 0x7b)
|
|
// || b != 0x60
|
|
// || b != 26
|
|
// {
|
|
// return Cow::from(format!("\"{}\"", escape_double_quote(w)));
|
|
// }
|
|
// }
|
|
// Cow::from(w)
|
|
// }
|
|
// }
|
|
//}
|
|
//
|
|
//
|
|
pub async fn insert(
|
|
envelope: Envelope,
|
|
backend: Arc<RwLock<Box<dyn MailBackend>>>,
|
|
acc_name: String,
|
|
) -> Result<()> {
|
|
let db_path = db_path()?;
|
|
if !db_path.exists() {
|
|
return Err(MeliError::new(
|
|
"Database hasn't been initialised. Run `reindex` command",
|
|
));
|
|
}
|
|
|
|
let conn = melib_sqlite3::open_db(db_path)?;
|
|
|
|
let op = backend
|
|
.read()
|
|
.unwrap()
|
|
.operation(envelope.hash())?
|
|
.as_bytes()?;
|
|
|
|
let body = match op.await.map(|bytes| envelope.body_bytes(&bytes)) {
|
|
Ok(body) => body.text(),
|
|
Err(err) => {
|
|
debug!(
|
|
"{}",
|
|
format!(
|
|
"Failed to open envelope {}: {}",
|
|
envelope.message_id_display(),
|
|
err.to_string()
|
|
)
|
|
);
|
|
log(
|
|
format!(
|
|
"Failed to open envelope {}: {}",
|
|
envelope.message_id_display(),
|
|
err.to_string()
|
|
),
|
|
ERROR,
|
|
);
|
|
return Err(err);
|
|
}
|
|
};
|
|
|
|
if let Err(err) = conn.execute(
|
|
"INSERT OR IGNORE INTO accounts (name) VALUES (?1)",
|
|
params![acc_name,],
|
|
) {
|
|
debug!(
|
|
"Failed to insert envelope {}: {}",
|
|
envelope.message_id_display(),
|
|
err.to_string()
|
|
);
|
|
log(
|
|
format!(
|
|
"Failed to insert envelope {}: {}",
|
|
envelope.message_id_display(),
|
|
err.to_string()
|
|
),
|
|
ERROR,
|
|
);
|
|
return Err(MeliError::new(err.to_string()));
|
|
}
|
|
let account_id: i32 = {
|
|
let mut stmt = conn
|
|
.prepare("SELECT id FROM accounts WHERE name = ?")
|
|
.unwrap();
|
|
let x = stmt
|
|
.query_map(params![acc_name], |row| row.get(0))
|
|
.unwrap()
|
|
.next()
|
|
.unwrap()
|
|
.unwrap();
|
|
x
|
|
};
|
|
if let Err(err) = conn.execute(
|
|
"INSERT OR REPLACE INTO envelopes (account_id, hash, date, _from, _to, cc, bcc, subject, message_id, in_reply_to, _references, flags, has_attachments, body_text, timestamp)
|
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)",
|
|
params![account_id, envelope.hash().to_be_bytes().to_vec(), envelope.date_as_str(), envelope.field_from_to_string(), envelope.field_to_to_string(), envelope.field_cc_to_string(), envelope.field_bcc_to_string(), envelope.subject().into_owned().trim_end_matches('\u{0}'), envelope.message_id_display().to_string(), envelope.in_reply_to_display().map(|f| f.to_string()).unwrap_or(String::new()), envelope.field_references_to_string(), i64::from(envelope.flags().bits()), if envelope.has_attachments() { 1 } else { 0 }, body, envelope.date().to_be_bytes().to_vec()],
|
|
)
|
|
.map_err(|e| MeliError::new(e.to_string())) {
|
|
debug!(
|
|
"Failed to insert envelope {}: {}",
|
|
envelope.message_id_display(),
|
|
err.to_string()
|
|
);
|
|
log(
|
|
format!(
|
|
"Failed to insert envelope {}: {}",
|
|
envelope.message_id_display(),
|
|
err.to_string()
|
|
),
|
|
ERROR,
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub fn remove(env_hash: EnvelopeHash) -> Result<()> {
|
|
let db_path = db_path()?;
|
|
if !db_path.exists() {
|
|
return Err(MeliError::new(
|
|
"Database hasn't been initialised. Run `reindex` command",
|
|
));
|
|
}
|
|
|
|
let conn = melib_sqlite3::open_db(db_path)?;
|
|
if let Err(err) = conn
|
|
.execute(
|
|
"DELETE FROM envelopes WHERE hash = ?",
|
|
params![env_hash.to_be_bytes().to_vec(),],
|
|
)
|
|
.map_err(|e| MeliError::new(e.to_string()))
|
|
{
|
|
debug!(
|
|
"Failed to remove envelope {}: {}",
|
|
env_hash,
|
|
err.to_string()
|
|
);
|
|
log(
|
|
format!(
|
|
"Failed to remove envelope {}: {}",
|
|
env_hash,
|
|
err.to_string()
|
|
),
|
|
ERROR,
|
|
);
|
|
return Err(err);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub fn index(context: &mut crate::state::Context, account_index: usize) -> ResultFuture<()> {
|
|
let account = &context.accounts[account_index];
|
|
let (acc_name, acc_mutex, backend_mutex): (String, Arc<RwLock<_>>, Arc<_>) = (
|
|
account.name().to_string(),
|
|
account.collection.envelopes.clone(),
|
|
account.backend.clone(),
|
|
);
|
|
let conn = melib_sqlite3::open_or_create_db(&DB, None)?;
|
|
let env_hashes = acc_mutex
|
|
.read()
|
|
.unwrap()
|
|
.keys()
|
|
.cloned()
|
|
.collect::<Vec<_>>();
|
|
|
|
/* Sleep, index and repeat in order not to block the main process */
|
|
Ok(Box::pin(async move {
|
|
conn.execute(
|
|
"INSERT OR REPLACE INTO accounts (name) VALUES (?1)",
|
|
params![acc_name.as_str(),],
|
|
)
|
|
.chain_err_summary(|| "Failed to update index:")?;
|
|
let account_id: i32 = {
|
|
let mut stmt = conn
|
|
.prepare("SELECT id FROM accounts WHERE name = ?")
|
|
.unwrap();
|
|
let x = stmt
|
|
.query_map(params![acc_name.as_str()], |row| row.get(0))
|
|
.unwrap()
|
|
.next()
|
|
.unwrap()
|
|
.unwrap();
|
|
x
|
|
};
|
|
let mut ctr = 0;
|
|
debug!(
|
|
"{}",
|
|
format!(
|
|
"Rebuilding {} index. {}/{}",
|
|
acc_name,
|
|
ctr,
|
|
env_hashes.len()
|
|
)
|
|
);
|
|
for chunk in env_hashes.chunks(200) {
|
|
ctr += chunk.len();
|
|
for env_hash in chunk {
|
|
let mut op = backend_mutex.read().unwrap().operation(*env_hash)?;
|
|
let bytes = op
|
|
.as_bytes()?
|
|
.await
|
|
.chain_err_summary(|| format!("Failed to open envelope {}", env_hash))?;
|
|
let envelopes_lck = acc_mutex.read().unwrap();
|
|
if let Some(e) = envelopes_lck.get(&env_hash) {
|
|
let body = e.body_bytes(&bytes).text().replace('\0', "");
|
|
conn.execute("INSERT OR REPLACE INTO envelopes (account_id, hash, date, _from, _to, cc, bcc, subject, message_id, in_reply_to, _references, flags, has_attachments, body_text, timestamp)
|
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)",
|
|
params![account_id, e.hash().to_be_bytes().to_vec(), e.date_as_str(), e.field_from_to_string(), e.field_to_to_string(), e.field_cc_to_string(), e.field_bcc_to_string(), e.subject().into_owned().trim_end_matches('\u{0}'), e.message_id_display().to_string(), e.in_reply_to_display().map(|f| f.to_string()).unwrap_or(String::new()), e.field_references_to_string(), i64::from(e.flags().bits()), if e.has_attachments() { 1 } else { 0 }, body, e.date().to_be_bytes().to_vec()],
|
|
).chain_err_summary(|| format!( "Failed to insert envelope {}", e.message_id_display()))?;
|
|
}
|
|
}
|
|
let sleep_dur = std::time::Duration::from_millis(20);
|
|
std::thread::sleep(sleep_dur);
|
|
}
|
|
Ok(())
|
|
}))
|
|
}
|
|
|
|
pub fn search(
|
|
query: &Query,
|
|
(sort_field, sort_order): (SortField, SortOrder),
|
|
) -> ResultFuture<SmallVec<[EnvelopeHash; 512]>> {
|
|
let db_path = db_path()?;
|
|
if !db_path.exists() {
|
|
return Err(MeliError::new(
|
|
"Database hasn't been initialised. Run `reindex` command",
|
|
));
|
|
}
|
|
|
|
let conn = melib_sqlite3::open_db(db_path)?;
|
|
|
|
let sort_field = match debug!(sort_field) {
|
|
SortField::Subject => "subject",
|
|
SortField::Date => "timestamp",
|
|
};
|
|
|
|
let sort_order = match debug!(sort_order) {
|
|
SortOrder::Asc => "ASC",
|
|
SortOrder::Desc => "DESC",
|
|
};
|
|
|
|
let mut stmt = conn
|
|
.prepare(
|
|
debug!(format!(
|
|
"SELECT hash FROM envelopes WHERE {} ORDER BY {} {};",
|
|
query_to_sql(&query),
|
|
sort_field,
|
|
sort_order
|
|
))
|
|
.as_str(),
|
|
)
|
|
.map_err(|e| MeliError::new(e.to_string()))?;
|
|
|
|
let results = stmt
|
|
.query_map(rusqlite::NO_PARAMS, |row| Ok(row.get(0)?))
|
|
.map_err(|e| MeliError::new(e.to_string()))?
|
|
.map(|r: std::result::Result<Vec<u8>, rusqlite::Error>| {
|
|
Ok(u64::from_be_bytes(
|
|
r.map_err(|e| MeliError::new(e.to_string()))?
|
|
.as_slice()
|
|
.try_into()
|
|
.map_err(|e: std::array::TryFromSliceError| MeliError::new(e.to_string()))?,
|
|
))
|
|
})
|
|
.collect::<Result<SmallVec<[EnvelopeHash; 512]>>>();
|
|
Ok(Box::pin(async { results }))
|
|
}
|
|
|
|
/// Translates a `Query` to an Sqlite3 expression in a `String`.
|
|
pub fn query_to_sql(q: &Query) -> String {
|
|
fn rec(q: &Query, s: &mut String) {
|
|
match q {
|
|
Subject(t) => {
|
|
s.push_str("subject LIKE \"%");
|
|
s.extend(escape_double_quote(t).chars());
|
|
s.push_str("%\" ");
|
|
}
|
|
From(t) => {
|
|
s.push_str("_from LIKE \"%");
|
|
s.extend(escape_double_quote(t).chars());
|
|
s.push_str("%\" ");
|
|
}
|
|
To(t) => {
|
|
s.push_str("_to LIKE \"%");
|
|
s.extend(escape_double_quote(t).chars());
|
|
s.push_str("%\" ");
|
|
}
|
|
Cc(t) => {
|
|
s.push_str("cc LIKE \"%");
|
|
s.extend(escape_double_quote(t).chars());
|
|
s.push_str("%\" ");
|
|
}
|
|
Bcc(t) => {
|
|
s.push_str("bcc LIKE \"%");
|
|
s.extend(escape_double_quote(t).chars());
|
|
s.push_str("%\" ");
|
|
}
|
|
AllText(t) => {
|
|
s.push_str("body_text LIKE \"%");
|
|
s.extend(escape_double_quote(t).chars());
|
|
s.push_str("%\" ");
|
|
}
|
|
And(q1, q2) => {
|
|
s.push_str("(");
|
|
rec(q1, s);
|
|
s.push_str(") AND (");
|
|
rec(q2, s);
|
|
s.push_str(") ");
|
|
}
|
|
Or(q1, q2) => {
|
|
s.push_str("(");
|
|
rec(q1, s);
|
|
s.push_str(") OR (");
|
|
rec(q2, s);
|
|
s.push_str(") ");
|
|
}
|
|
Not(q) => {
|
|
s.push_str("NOT (");
|
|
rec(q, s);
|
|
s.push_str(") ");
|
|
}
|
|
Flags(v) => {
|
|
let total = v.len();
|
|
if total > 1 {
|
|
s.push_str("(");
|
|
}
|
|
for (i, f) in v.iter().enumerate() {
|
|
match f.as_str() {
|
|
"draft" => {
|
|
s.push_str(" (flags & 8 > 0) ");
|
|
}
|
|
"deleted" | "trashed" => {
|
|
s.push_str(" (flags & 6 > 0) ");
|
|
}
|
|
"flagged" => {
|
|
s.push_str(" (flags & 16 > 0) ");
|
|
}
|
|
"recent" => {
|
|
s.push_str(" (flags & 4 == 0) ");
|
|
}
|
|
"seen" | "read" => {
|
|
s.push_str(" (flags & 4 > 0) ");
|
|
}
|
|
"unseen" | "unread" => {
|
|
s.push_str(" (flags & 4 == 0) ");
|
|
}
|
|
"answered" | "replied" => {
|
|
s.push_str(" (flags & 2 > 0) ");
|
|
}
|
|
"unanswered" => {
|
|
s.push_str(" (flags & 2 == 0) ");
|
|
}
|
|
_ => {
|
|
continue;
|
|
}
|
|
}
|
|
if total > 1 && i != total - 1 {
|
|
s.push_str(" AND ");
|
|
}
|
|
}
|
|
if total > 1 {
|
|
s.push_str(") ");
|
|
}
|
|
}
|
|
HasAttachment => {
|
|
s.push_str("has_attachments == 1 ");
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
let mut ret = String::new();
|
|
rec(q, &mut ret);
|
|
ret
|
|
}
|
|
|
|
#[test]
|
|
fn test_query_to_sql() {
|
|
use melib::parsec::Parser;
|
|
use melib::search::query;
|
|
assert_eq!(
|
|
"(subject LIKE \"%test%\" ) AND (body_text LIKE \"%i%\" ) ",
|
|
&query_to_sql(&query().parse_complete("subject: test and i").unwrap().1)
|
|
);
|
|
assert_eq!(
|
|
"(subject LIKE \"%github%\" ) OR ((_from LIKE \"%epilys%\" ) AND ((subject LIKE \"%lib%\" ) OR (subject LIKE \"%meli%\" ) ) ) ",
|
|
&query_to_sql(
|
|
&query()
|
|
.parse_complete(
|
|
"subject: github or (from: epilys and (subject:lib or subject: meli))"
|
|
)
|
|
.unwrap()
|
|
.1
|
|
)
|
|
);
|
|
}
|