🐝 I really like where this mua is(was?) headed, but it seems as though there has not been much activity recently.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

548 lines
22 KiB

  1. /*
  2. * meli - sqlite3.rs
  3. *
  4. * Copyright 2019 Manos Pitsidianakis
  5. *
  6. * This file is part of meli.
  7. *
  8. * meli is free software: you can redistribute it and/or modify
  9. * it under the terms of the GNU General Public License as published by
  10. * the Free Software Foundation, either version 3 of the License, or
  11. * (at your option) any later version.
  12. *
  13. * meli is distributed in the hope that it will be useful,
  14. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16. * GNU General Public License for more details.
  17. *
  18. * You should have received a copy of the GNU General Public License
  19. * along with meli. If not, see <http://www.gnu.org/licenses/>.
  20. */
  21. /*! Use an sqlite3 database for fast searching.
  22. */
  23. use smallvec::SmallVec;
  24. use crate::cache::{escape_double_quote, query, Query::{self, *}};
  25. use crate::melib::parsec::Parser;
  26. use melib::{
  27. backends::MailBackend,
  28. email::{Envelope, EnvelopeHash},
  29. log,
  30. thread::{SortField, SortOrder},
  31. MeliError, Result, ERROR,
  32. };
  33. use rusqlite::{params, Connection};
  34. use std::path::PathBuf;
  35. use std::convert::TryInto;
  36. use std::sync::{Arc, RwLock};
  37. pub fn db_path() -> Result<PathBuf> {
  38. let data_dir =
  39. xdg::BaseDirectories::with_prefix("meli").map_err(|e| MeliError::new(e.to_string()))?;
  40. Ok(data_dir
  41. .place_data_file("index.db")
  42. .map_err(|e| MeliError::new(e.to_string()))?)
  43. }
  44. //#[inline(always)]
  45. //fn fts5_bareword(w: &str) -> Cow<str> {
  46. // if w == "AND" || w == "OR" || w == "NOT" {
  47. // Cow::from(w)
  48. // } else {
  49. // if !w.is_ascii() {
  50. // Cow::from(format!("\"{}\"", escape_double_quote(w)))
  51. // } else {
  52. // for &b in w.as_bytes() {
  53. // if !(b > 0x2f && b < 0x3a)
  54. // || !(b > 0x40 && b < 0x5b)
  55. // || !(b > 0x60 && b < 0x7b)
  56. // || b != 0x60
  57. // || b != 26
  58. // {
  59. // return Cow::from(format!("\"{}\"", escape_double_quote(w)));
  60. // }
  61. // }
  62. // Cow::from(w)
  63. // }
  64. // }
  65. //}
  66. //
  67. pub fn open_db() -> Result<Connection> {
  68. let db_path = db_path()?;
  69. if !db_path.exists() {
  70. return Err(MeliError::new("Database hasn't been initialised. Run `reindex` command"));
  71. }
  72. Connection::open(&db_path).map_err(|e| MeliError::new(e.to_string()))
  73. }
  74. pub fn open_or_create_db() -> Result<Connection> {
  75. let db_path = db_path()?;
  76. let mut set_mode = false;
  77. if !db_path.exists() {
  78. log(
  79. format!("Creating index database in {}", db_path.display()),
  80. melib::INFO,
  81. );
  82. set_mode = true;
  83. }
  84. let conn = Connection::open(&db_path).map_err(|e| MeliError::new(e.to_string()))?;
  85. if set_mode {
  86. use std::os::unix::fs::PermissionsExt;
  87. let file = std::fs::File::open(&db_path)?;
  88. let metadata = file.metadata()?;
  89. let mut permissions = metadata.permissions();
  90. permissions.set_mode(0o600); // Read/write for owner only.
  91. file.set_permissions(permissions)?;
  92. }
  93. conn.execute_batch(
  94. "CREATE TABLE IF NOT EXISTS envelopes (
  95. id INTEGER PRIMARY KEY,
  96. account_id INTEGER REFERENCES accounts ON UPDATE CASCADE,
  97. hash BLOB NOT NULL UNIQUE,
  98. date TEXT NOT NULL,
  99. _from TEXT NOT NULL,
  100. _to TEXT NOT NULL,
  101. cc TEXT NOT NULL,
  102. bcc TEXT NOT NULL,
  103. subject TEXT NOT NULL,
  104. message_id TEXT NOT NULL,
  105. in_reply_to TEXT NOT NULL,
  106. _references TEXT NOT NULL,
  107. flags INTEGER NOT NULL,
  108. has_attachments BOOLEAN NOT NULL,
  109. body_text TEXT NOT NULL,
  110. timestamp BLOB NOT NULL
  111. );
  112. CREATE TABLE IF NOT EXISTS folders (
  113. id INTEGER PRIMARY KEY,
  114. account_id INTEGER NOT NULL REFERENCES accounts ON UPDATE CASCADE,
  115. hash BLOB NOT NULL,
  116. date TEXT NOT NULL,
  117. name TEXT NOT NULL
  118. );
  119. CREATE TABLE IF NOT EXISTS accounts (
  120. id INTEGER PRIMARY KEY,
  121. name TEXT NOT NULL UNIQUE
  122. );
  123. CREATE TABLE IF NOT EXISTS folder_and_envelope (
  124. folder_id INTEGER NOT NULL,
  125. envelope_id INTEGER NOT NULL,
  126. PRIMARY KEY (folder_id, envelope_id),
  127. FOREIGN KEY(folder_id) REFERENCES folders(id) ON UPDATE CASCADE,
  128. FOREIGN KEY(envelope_id) REFERENCES envelopes(id) ON UPDATE CASCADE
  129. );
  130. CREATE INDEX IF NOT EXISTS folder_env_idx ON folder_and_envelope(folder_id);
  131. CREATE INDEX IF NOT EXISTS env_folder_idx ON folder_and_envelope(envelope_id);
  132. CREATE UNIQUE INDEX IF NOT EXISTS acc_idx ON accounts(name);
  133. CREATE INDEX IF NOT EXISTS envelope_timestamp_index ON envelopes (timestamp);
  134. CREATE INDEX IF NOT EXISTS envelope__from_index ON envelopes (_from);
  135. CREATE INDEX IF NOT EXISTS envelope__to_index ON envelopes (_to);
  136. CREATE INDEX IF NOT EXISTS envelope_cc_index ON envelopes (cc);
  137. CREATE INDEX IF NOT EXISTS envelope_bcc_index ON envelopes (bcc);
  138. CREATE INDEX IF NOT EXISTS envelope_message_id_index ON envelopes (message_id);
  139. CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(subject, body_text, content=envelopes, content_rowid=id);
  140. -- Triggers to keep the FTS index up to date.
  141. CREATE TRIGGER IF NOT EXISTS envelopes_ai AFTER INSERT ON envelopes BEGIN
  142. INSERT INTO fts(rowid, subject, body_text) VALUES (new.id, new.subject, new.body_text);
  143. END;
  144. CREATE TRIGGER IF NOT EXISTS envelopes_ad AFTER DELETE ON envelopes BEGIN
  145. INSERT INTO fts(fts, rowid, subject, body_text) VALUES('delete', old.id, old.subject, old.body_text);
  146. END;
  147. CREATE TRIGGER IF NOT EXISTS envelopes_au AFTER UPDATE ON envelopes BEGIN
  148. INSERT INTO fts(fts, rowid, subject, body_text) VALUES('delete', old.id, old.subject, old.body_text);
  149. INSERT INTO fts(rowid, subject, body_text) VALUES (new.id, new.subject, new.body_text);
  150. END; ",
  151. )
  152. .map_err(|e| MeliError::new(e.to_string()))?;
  153. Ok(conn)
  154. }
  155. pub fn insert(envelope: &Envelope, backend: &Arc<RwLock<Box<dyn MailBackend>>>, acc_name: &str) -> Result<()> {
  156. let conn = open_db()?;
  157. let backend_lck = backend.read().unwrap();
  158. let op = backend_lck.operation(envelope.hash());
  159. let body = match envelope.body(op) {
  160. Ok(body) => body.text(),
  161. Err(err) => {
  162. debug!(
  163. "{}",
  164. format!(
  165. "Failed to open envelope {}: {}",
  166. envelope.message_id_display(),
  167. err.to_string()
  168. )
  169. );
  170. log(
  171. format!(
  172. "Failed to open envelope {}: {}",
  173. envelope.message_id_display(),
  174. err.to_string()
  175. ),
  176. ERROR,
  177. );
  178. return Err(err);
  179. }
  180. };
  181. if let Err(err) = conn.execute("INSERT OR IGNORE INTO accounts (name) VALUES (?1)", params![acc_name, ]) {
  182. debug!(
  183. "Failed to insert envelope {}: {}",
  184. envelope.message_id_display(),
  185. err.to_string()
  186. );
  187. log(
  188. format!(
  189. "Failed to insert envelope {}: {}",
  190. envelope.message_id_display(),
  191. err.to_string()
  192. ),
  193. ERROR,
  194. );
  195. return Err(MeliError::new(err.to_string()));
  196. }
  197. let account_id: i32 = {
  198. let mut stmt = conn.prepare("SELECT id FROM accounts WHERE name = ?").unwrap();
  199. let x = stmt.query_map(params![acc_name], |row| row.get(0)).unwrap().next().unwrap().unwrap();
  200. x
  201. };
  202. if let Err(err) = conn.execute(
  203. "INSERT OR REPLACE INTO envelopes (account_id, hash, date, _from, _to, cc, bcc, subject, message_id, in_reply_to, _references, flags, has_attachments, body_text, timestamp)
  204. VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)",
  205. params![account_id, envelope.hash().to_be_bytes().to_vec(), envelope.date_as_str(), envelope.field_from_to_string(), envelope.field_to_to_string(), envelope.field_cc_to_string(), envelope.field_bcc_to_string(), envelope.subject().into_owned().trim_end_matches('\u{0}'), envelope.message_id_display().to_string(), envelope.in_reply_to_display().map(|f| f.to_string()).unwrap_or(String::new()), envelope.field_references_to_string(), i64::from(envelope.flags().bits()), if envelope.has_attachments() { 1 } else { 0 }, body, envelope.date().to_be_bytes().to_vec()],
  206. )
  207. .map_err(|e| MeliError::new(e.to_string())) {
  208. debug!(
  209. "Failed to insert envelope {}: {}",
  210. envelope.message_id_display(),
  211. err.to_string()
  212. );
  213. log(
  214. format!(
  215. "Failed to insert envelope {}: {}",
  216. envelope.message_id_display(),
  217. err.to_string()
  218. ),
  219. ERROR,
  220. );
  221. }
  222. Ok(())
  223. }
  224. pub fn remove(env_hash: EnvelopeHash) -> Result<()> {
  225. let conn = open_db()?;
  226. if let Err(err) = conn.execute(
  227. "DELETE FROM envelopes WHERE hash = ?",
  228. params![env_hash.to_be_bytes().to_vec(), ])
  229. .map_err(|e| MeliError::new(e.to_string())) {
  230. debug!(
  231. "Failed to remove envelope {}: {}",
  232. env_hash,
  233. err.to_string()
  234. );
  235. log(
  236. format!(
  237. "Failed to remove envelope {}: {}",
  238. env_hash,
  239. err.to_string()
  240. ),
  241. ERROR,
  242. );
  243. return Err(err);
  244. }
  245. Ok(())
  246. }
  247. pub fn index(context: &mut crate::state::Context, account_name: &str) -> Result<()> {
  248. let account = if let Some(a) = context.accounts.iter().find(|acc| acc.name() == account_name) {
  249. a} else {
  250. return Err(MeliError::new(format!("Account {} was not found.", account_name)));
  251. };
  252. let (acc_name, acc_mutex, backend_mutex):( String, Arc<RwLock<_>>, Arc<_>) = if *account.settings.conf.cache_type() != crate::conf::CacheType::Sqlite3 {
  253. return Err(MeliError::new(format!("Account {} doesn't have an sqlite3 search backend.", account_name)));
  254. } else {
  255. (
  256. account.name().to_string(),
  257. account.collection.envelopes.clone(),
  258. account.backend.clone(),
  259. )};
  260. let conn = open_or_create_db()?;
  261. let work_context = context.work_controller().get_context();
  262. let env_hashes =
  263. acc_mutex.read().unwrap().keys().cloned().collect::<Vec<_>>();
  264. /* Sleep, index and repeat in order not to block the main process */
  265. let handle = std::thread::Builder::new().name(String::from("rebuilding index")).spawn(move || {
  266. let thread_id = std::thread::current().id();
  267. let sleep_dur = std::time::Duration::from_millis(20);
  268. if let Err(err) = conn.execute(
  269. "INSERT OR REPLACE INTO accounts (name) VALUES (?1)", params![acc_name.as_str(),],).map_err(|e| MeliError::new(e.to_string())) {
  270. debug!("{}",
  271. format!(
  272. "Failed to update index: {}",
  273. err.to_string()
  274. ));
  275. log(
  276. format!(
  277. "Failed to update index: {}",
  278. err.to_string()
  279. ),
  280. ERROR,
  281. );
  282. }
  283. let account_id: i32 = {
  284. let mut stmt = conn.prepare("SELECT id FROM accounts WHERE name = ?").unwrap();
  285. let x = stmt.query_map(params![acc_name.as_str()], |row| row.get(0)).unwrap().next().unwrap().unwrap();
  286. x
  287. };
  288. let mut ctr = 0;
  289. debug!("{}", format!("Rebuilding {} index. {}/{}", acc_name, ctr, env_hashes.len()));
  290. work_context
  291. .set_status
  292. .send((thread_id, format!("Rebuilding {} index. {}/{}", acc_name, ctr, env_hashes.len())))
  293. .unwrap();
  294. for chunk in env_hashes.chunks(200) {
  295. ctr += chunk.len();
  296. let envelopes_lck = acc_mutex.read().unwrap();
  297. let backend_lck = backend_mutex.read().unwrap();
  298. for env_hash in chunk {
  299. if let Some(e) = envelopes_lck.get(&env_hash) {
  300. let op = backend_lck.operation(e.hash());
  301. let body = match e.body(op) {
  302. Ok(body) => body.text(),
  303. Err(err) => {
  304. debug!("{}",
  305. format!(
  306. "Failed to open envelope {}: {}",
  307. e.message_id_display(),
  308. err.to_string()
  309. ));
  310. log(
  311. format!(
  312. "Failed to open envelope {}: {}",
  313. e.message_id_display(),
  314. err.to_string()
  315. ),
  316. ERROR,
  317. );
  318. return;
  319. }
  320. };
  321. if let Err(err) = conn.execute(
  322. "INSERT OR REPLACE INTO envelopes (account_id, hash, date, _from, _to, cc, bcc, subject, message_id, in_reply_to, _references, flags, has_attachments, body_text, timestamp)
  323. VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)",
  324. params![account_id, e.hash().to_be_bytes().to_vec(), e.date_as_str(), e.field_from_to_string(), e.field_to_to_string(), e.field_cc_to_string(), e.field_bcc_to_string(), e.subject().into_owned().trim_end_matches('\u{0}'), e.message_id_display().to_string(), e.in_reply_to_display().map(|f| f.to_string()).unwrap_or(String::new()), e.field_references_to_string(), i64::from(e.flags().bits()), if e.has_attachments() { 1 } else { 0 }, body, e.date().to_be_bytes().to_vec()],
  325. )
  326. .map_err(|e| MeliError::new(e.to_string())) {
  327. debug!("{}",
  328. format!(
  329. "Failed to insert envelope {}: {}",
  330. e.message_id_display(),
  331. err.to_string()
  332. ));
  333. log(
  334. format!(
  335. "Failed to insert envelope {}: {}",
  336. e.message_id_display(),
  337. err.to_string()
  338. ),
  339. ERROR,
  340. );
  341. }
  342. }
  343. }
  344. drop(envelopes_lck);
  345. work_context
  346. .set_status
  347. .send((thread_id, format!("Rebuilding {} index. {}/{}", acc_name, ctr, env_hashes.len())))
  348. .unwrap();
  349. std::thread::sleep(sleep_dur);
  350. }
  351. work_context.finished.send(thread_id).unwrap();
  352. })?;
  353. context.work_controller().static_threads.lock()?.insert(
  354. handle.thread().id(),
  355. String::from("Rebuilding sqlite3 index").into(),
  356. );
  357. Ok(())
  358. }
  359. pub fn search(
  360. term: &str,
  361. (sort_field, sort_order): (SortField, SortOrder),
  362. ) -> Result<SmallVec<[EnvelopeHash; 512]>> {
  363. let conn = open_db()?;
  364. let sort_field = match debug!(sort_field) {
  365. SortField::Subject => "subject",
  366. SortField::Date => "timestamp",
  367. };
  368. let sort_order = match debug!(sort_order) {
  369. SortOrder::Asc => "ASC",
  370. SortOrder::Desc => "DESC",
  371. };
  372. let mut stmt = conn
  373. .prepare(
  374. debug!(format!(
  375. "SELECT hash FROM envelopes WHERE {} ORDER BY {} {};",
  376. query_to_sql(&query().parse(term)?.1),
  377. sort_field,
  378. sort_order
  379. ))
  380. .as_str(),
  381. )
  382. .map_err(|e| MeliError::new(e.to_string()))?;
  383. let results = stmt
  384. .query_map(rusqlite::NO_PARAMS, |row| Ok(row.get(0)?))
  385. .map_err(|e| MeliError::new(e.to_string()))?
  386. .map(|r: std::result::Result<Vec<u8>, rusqlite::Error>| {
  387. Ok(u64::from_be_bytes(
  388. r.map_err(|e| MeliError::new(e.to_string()))?
  389. .as_slice()
  390. .try_into()
  391. .map_err(|e: std::array::TryFromSliceError| MeliError::new(e.to_string()))?,
  392. ))
  393. })
  394. .collect::<Result<SmallVec<[EnvelopeHash; 512]>>>();
  395. results
  396. }
  397. /// Translates a `Query` to an Sqlite3 expression in a `String`.
  398. pub fn query_to_sql(q: &Query) -> String {
  399. fn rec(q: &Query, s: &mut String) {
  400. match q {
  401. Subject(t) => {
  402. s.push_str("subject LIKE \"%");
  403. s.extend(escape_double_quote(t).chars());
  404. s.push_str("%\" ");
  405. }
  406. From(t) => {
  407. s.push_str("_from LIKE \"%");
  408. s.extend(escape_double_quote(t).chars());
  409. s.push_str("%\" ");
  410. }
  411. To(t) => {
  412. s.push_str("_to LIKE \"%");
  413. s.extend(escape_double_quote(t).chars());
  414. s.push_str("%\" ");
  415. }
  416. Cc(t) => {
  417. s.push_str("cc LIKE \"%");
  418. s.extend(escape_double_quote(t).chars());
  419. s.push_str("%\" ");
  420. }
  421. Bcc(t) => {
  422. s.push_str("bcc LIKE \"%");
  423. s.extend(escape_double_quote(t).chars());
  424. s.push_str("%\" ");
  425. }
  426. AllText(t) => {
  427. s.push_str("body_text LIKE \"%");
  428. s.extend(escape_double_quote(t).chars());
  429. s.push_str("%\" ");
  430. }
  431. And(q1, q2) => {
  432. s.push_str("(");
  433. rec(q1, s);
  434. s.push_str(") AND (");
  435. rec(q2, s);
  436. s.push_str(") ");
  437. }
  438. Or(q1, q2) => {
  439. s.push_str("(");
  440. rec(q1, s);
  441. s.push_str(") OR (");
  442. rec(q2, s);
  443. s.push_str(") ");
  444. }
  445. Not(q) => {
  446. s.push_str("NOT (");
  447. rec(q, s);
  448. s.push_str(") ");
  449. }
  450. Flags(v) => {
  451. let total = v.len();
  452. if total > 1 {
  453. s.push_str("(");
  454. }
  455. for (i, f) in v.iter().enumerate() {
  456. match f.as_str() {
  457. "draft" => {
  458. s.push_str(" (flags & 8 > 0) ");
  459. }
  460. "deleted" | "trashed" => {
  461. s.push_str(" (flags & 6 > 0) ");
  462. }
  463. "flagged" => {
  464. s.push_str(" (flags & 16 > 0) ");
  465. }
  466. "recent" => {
  467. s.push_str(" (flags & 4 == 0) ");
  468. }
  469. "seen" | "read" => {
  470. s.push_str(" (flags & 4 > 0) ");
  471. }
  472. "unseen" | "unread" => {
  473. s.push_str(" (flags & 4 == 0) ");
  474. }
  475. "answered" | "replied" => {
  476. s.push_str(" (flags & 2 > 0) ");
  477. }
  478. "unanswered" => {
  479. s.push_str(" (flags & 2 == 0) ");
  480. }
  481. _ => {
  482. continue;
  483. }
  484. }
  485. if total > 1 && i != total - 1 {
  486. s.push_str(" AND ");
  487. }
  488. }
  489. if total > 1 {
  490. s.push_str(") ");
  491. }
  492. }
  493. HasAttachment => {
  494. s.push_str("has_attachments == 1 ");
  495. }
  496. _ => {}
  497. }
  498. }
  499. let mut ret = String::new();
  500. rec(q, &mut ret);
  501. ret
  502. }
  503. #[test]
  504. fn test_query_to_sql() {
  505. assert_eq!(
  506. "(subject LIKE \"%test%\" ) AND (body_text LIKE \"%i%\" ) ",
  507. &query_to_sql(&query().parse_complete("subject: test and i").unwrap().1)
  508. );
  509. assert_eq!(
  510. "(subject LIKE \"%github%\" ) OR ((_from LIKE \"%epilys%\" ) AND ((subject LIKE \"%lib%\" ) OR (subject LIKE \"%meli%\" ) ) ) ",
  511. &query_to_sql(
  512. &query()
  513. .parse_complete(
  514. "subject: github or (from: epilys and (subject:lib or subject: meli))"
  515. )
  516. .unwrap()
  517. .1
  518. )
  519. );
  520. }