From 657b58c4aed4cc89c4a1b6613ed955d4a4b70fcb Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Thu, 18 May 2023 13:57:51 +0300 Subject: [PATCH] core: add migration test --- cli/src/args.rs | 1 - core/build.rs | 77 +--------- core/make_migrations.rs | 92 ++++++++++++ core/src/connection.rs | 2 +- core/tests/migrations.rs | 312 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 412 insertions(+), 72 deletions(-) create mode 100644 core/make_migrations.rs diff --git a/cli/src/args.rs b/cli/src/args.rs index 5cc26e8..98dc512 100644 --- a/cli/src/args.rs +++ b/cli/src/args.rs @@ -506,7 +506,6 @@ pub enum ListCommand { pub struct QueueValueParser; impl QueueValueParser { - /// Implementation for [`ValueParser::path_buf`] pub fn new() -> Self { Self } diff --git a/core/build.rs b/core/build.rs index 23a9bcd..c902648 100644 --- a/core/build.rs +++ b/core/build.rs @@ -18,10 +18,8 @@ */ use std::{ - fs::{metadata, read_dir, OpenOptions}, + fs::{metadata, OpenOptions}, io, - io::Write, - path::Path, process::{Command, Stdio}, }; @@ -46,7 +44,12 @@ where } } +include!("make_migrations.rs"); + +const MIGRATION_RS: &str = "src/migrations.rs.inc"; + fn main() { + println!("cargo:rerun-if-changed=src/migrations.rs.inc"); println!("cargo:rerun-if-changed=migrations"); println!("cargo:rerun-if-changed=src/schema.sql.m4"); @@ -88,71 +91,5 @@ fn main() { file.write_all(&output.stdout).unwrap(); } - const MIGRATION_RS: &str = "src/migrations.rs.inc"; - - let mut regen = false; - let mut paths = vec![]; - let mut undo_paths = vec![]; - for entry in read_dir("migrations").unwrap() { - let entry = entry.unwrap(); - let path = entry.path(); - if path.is_dir() || path.extension().map(|os| os.to_str().unwrap()) != Some("sql") { - continue; - } - if is_output_file_outdated(&path, MIGRATION_RS).unwrap() { - regen = true; - } - if path - .file_name() - .unwrap() - .to_str() - .unwrap() - .ends_with("undo.sql") - { - undo_paths.push(path); - } else { - paths.push(path); - } - } - - if regen { - paths.sort(); - undo_paths.sort(); - let mut migr_rs = OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .open(MIGRATION_RS) - .unwrap(); - migr_rs - .write_all(b"\n//(user_version, redo sql, undo sql\n&[") - .unwrap(); - for (p, u) in paths.iter().zip(undo_paths.iter()) { - // This should be a number string, padded with 2 zeros if it's less than 3 - // digits. e.g. 001, \d{3} - let num = p.file_stem().unwrap().to_str().unwrap(); - if !u.file_name().unwrap().to_str().unwrap().starts_with(num) { - panic!("Undo file {u:?} should match with {p:?}"); - } - if num.parse::().is_err() { - panic!("Migration file {p:?} should start with a number"); - } - migr_rs.write_all(b"(").unwrap(); - migr_rs - .write_all(num.trim_start_matches('0').as_bytes()) - .unwrap(); - migr_rs.write_all(b",\"").unwrap(); - - migr_rs - .write_all(std::fs::read_to_string(p).unwrap().as_bytes()) - .unwrap(); - migr_rs.write_all(b"\",\"").unwrap(); - migr_rs - .write_all(std::fs::read_to_string(u).unwrap().as_bytes()) - .unwrap(); - migr_rs.write_all(b"\"),").unwrap(); - } - migr_rs.write_all(b"]").unwrap(); - migr_rs.flush().unwrap(); - } + make_migrations("migrations", MIGRATION_RS); } diff --git a/core/make_migrations.rs b/core/make_migrations.rs new file mode 100644 index 0000000..8cf372d --- /dev/null +++ b/core/make_migrations.rs @@ -0,0 +1,92 @@ +/* + * This file is part of mailpot + * + * Copyright 2023 - Manos Pitsidianakis + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +use std::{fs::read_dir, io::Write, path::Path}; + +pub fn make_migrations, O: AsRef>(migrations_path: M, output_file: O) { + let migrations_folder_path = migrations_path.as_ref(); + let output_file_path = output_file.as_ref(); + + let mut regen = false; + let mut paths = vec![]; + let mut undo_paths = vec![]; + for entry in read_dir(migrations_folder_path).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + if path.is_dir() || path.extension().map(|os| os.to_str().unwrap()) != Some("sql") { + continue; + } + if is_output_file_outdated(&path, output_file_path).unwrap() { + regen = true; + } + if path + .file_name() + .unwrap() + .to_str() + .unwrap() + .ends_with("undo.sql") + { + undo_paths.push(path); + } else { + paths.push(path); + } + } + + if regen { + paths.sort(); + undo_paths.sort(); + let mut migr_rs = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(output_file_path) + .unwrap(); + migr_rs + .write_all(b"\n//(user_version, redo sql, undo sql\n&[") + .unwrap(); + for (i, (p, u)) in paths.iter().zip(undo_paths.iter()).enumerate() { + // This should be a number string, padded with 2 zeros if it's less than 3 + // digits. e.g. 001, \d{3} + let num = p.file_stem().unwrap().to_str().unwrap(); + if !u.file_name().unwrap().to_str().unwrap().starts_with(num) { + panic!("Undo file {u:?} should match with {p:?}"); + } + if num.parse::().is_err() { + panic!("Migration file {p:?} should start with a number"); + } + assert_eq!(num.parse::().unwrap(), i + 1, "migration sql files should start with 1, not zero, and no intermediate numbers should be missing. Panicked on file: {}", p.display()); + migr_rs.write_all(b"(").unwrap(); + migr_rs + .write_all(num.trim_start_matches('0').as_bytes()) + .unwrap(); + migr_rs.write_all(b",\"").unwrap(); + + migr_rs + .write_all(std::fs::read_to_string(p).unwrap().as_bytes()) + .unwrap(); + migr_rs.write_all(b"\",\"").unwrap(); + migr_rs + .write_all(std::fs::read_to_string(u).unwrap().as_bytes()) + .unwrap(); + migr_rs.write_all(b"\"),").unwrap(); + } + migr_rs.write_all(b"]").unwrap(); + migr_rs.flush().unwrap(); + } +} diff --git a/core/src/connection.rs b/core/src/connection.rs index 05ad84c..e6d6907 100644 --- a/core/src/connection.rs +++ b/core/src/connection.rs @@ -184,7 +184,7 @@ impl Connection { return Err("Database doesn't exist".into()); } INIT_SQLITE_LOGGING.call_once(|| { - unsafe { rusqlite::trace::config_log(Some(log_callback)).unwrap() }; + _ = unsafe { rusqlite::trace::config_log(Some(log_callback)) }; }); let conn = DbConnection::open(conf.db_path.to_str().unwrap())?; rusqlite::vtab::array::load_module(&conn)?; diff --git a/core/tests/migrations.rs b/core/tests/migrations.rs index f5464ec..841baf7 100644 --- a/core/tests/migrations.rs +++ b/core/tests/migrations.rs @@ -17,10 +17,34 @@ * along with this program. If not, see . */ +use std::fs::{metadata, File, OpenOptions}; + use mailpot::{Configuration, Connection, SendMail}; use mailpot_tests::init_stderr_logging; use tempfile::TempDir; +// Source: https://stackoverflow.com/a/64535181 +fn is_output_file_outdated(input: P1, output: P2) -> std::io::Result +where + P1: AsRef, + P2: AsRef, +{ + let out_meta = metadata(output); + if let Ok(meta) = out_meta { + let output_mtime = meta.modified()?; + + // if input file is more recent than our output, we are outdated + let input_meta = metadata(input)?; + let input_mtime = input_meta.modified()?; + + Ok(input_mtime > output_mtime) + } else { + // output file not found, we are outdated + Ok(true) + } +} +include!("../make_migrations.rs"); + #[test] fn test_init_empty() { init_stderr_logging(); @@ -49,3 +73,291 @@ fn test_init_empty() { db.migrate(migrations[0].0, version).unwrap(); } + +trait ConnectionExt { + fn schema_version(&self) -> Result; + fn migrate( + &mut self, + from: u32, + to: u32, + migrations: &[(u32, &str, &str)], + ) -> Result<(), rusqlite::Error>; +} + +impl ConnectionExt for rusqlite::Connection { + fn schema_version(&self) -> Result { + self.prepare("SELECT user_version FROM pragma_user_version;")? + .query_row([], |row| { + let v: u32 = row.get(0)?; + Ok(v) + }) + } + + fn migrate( + &mut self, + mut from: u32, + to: u32, + migrations: &[(u32, &str, &str)], + ) -> Result<(), rusqlite::Error> { + if from == to { + return Ok(()); + } + + let undo = from > to; + let tx = self.transaction()?; + + loop { + log::trace!( + "exec migration from {from} to {to}, type: {}do", + if undo { "un" } else { "re" } + ); + if undo { + log::trace!("{}", migrations[from as usize - 1].2); + tx.execute_batch(migrations[from as usize - 1].2)?; + from -= 1; + if from == to { + break; + } + } else { + if from != 0 { + log::trace!("{}", migrations[from as usize - 1].1); + tx.execute_batch(migrations[from as usize - 1].1)?; + } + from += 1; + if from == to + 1 { + break; + } + } + } + tx.pragma_update( + None, + "user_version", + if to == 0 { + 0 + } else { + migrations[to as usize - 1].0 + }, + )?; + + tx.commit()?; + Ok(()) + } +} + +#[test] +fn test_migration_gen() { + init_stderr_logging(); + let tmp_dir = TempDir::new().unwrap(); + let in_path = tmp_dir.path().join("migrations"); + std::fs::create_dir(&in_path).unwrap(); + let out_path = tmp_dir.path().join("migrations.txt"); + for (num, redo, undo) in MIGRATIONS.iter() { + let mut redo_file = File::options() + .write(true) + .create(true) + .truncate(true) + .open(&in_path.join(&format!("{num:03}.sql"))) + .unwrap(); + redo_file.write_all(redo.as_bytes()).unwrap(); + redo_file.flush().unwrap(); + + let mut undo_file = File::options() + .write(true) + .create(true) + .truncate(true) + .open(&in_path.join(&format!("{num:03}.undo.sql"))) + .unwrap(); + undo_file.write_all(undo.as_bytes()).unwrap(); + undo_file.flush().unwrap(); + } + + make_migrations(&in_path, &out_path); + let output = std::fs::read_to_string(&out_path).unwrap(); + assert_eq!(&output.replace([' ', '\n'], ""), &r#"//(user_version, redo sql, undo sql +&[(1,"ALTER TABLE PERSON ADD COLUMN interests TEXT;","ALTER TABLE PERSON DROP COLUMN interests;"),(2,"CREATE TABLE hobby ( pk INTEGER PRIMARY KEY NOT NULL,title TEXT NOT NULL);","DROP TABLE hobby;"),(3,"ALTER TABLE PERSON ADD COLUMN main_hobby INTEGER REFERENCES hobby(pk) ON DELETE SET NULL;","ALTER TABLE PERSON DROP COLUMN main_hobby;"),]"#.replace([' ', '\n'], "")); +} + +#[test] +#[should_panic] +fn test_migration_gen_panic() { + init_stderr_logging(); + let tmp_dir = TempDir::new().unwrap(); + let in_path = tmp_dir.path().join("migrations"); + std::fs::create_dir(&in_path).unwrap(); + let out_path = tmp_dir.path().join("migrations.txt"); + for (num, redo, undo) in MIGRATIONS.iter().skip(1) { + let mut redo_file = File::options() + .write(true) + .create(true) + .truncate(true) + .open(&in_path.join(&format!("{num:03}.sql"))) + .unwrap(); + redo_file.write_all(redo.as_bytes()).unwrap(); + redo_file.flush().unwrap(); + + let mut undo_file = File::options() + .write(true) + .create(true) + .truncate(true) + .open(&in_path.join(&format!("{num:03}.undo.sql"))) + .unwrap(); + undo_file.write_all(undo.as_bytes()).unwrap(); + undo_file.flush().unwrap(); + } + + make_migrations(&in_path, &out_path); + let output = std::fs::read_to_string(&out_path).unwrap(); + assert_eq!(&output.replace([' ','\n'], ""), &r#"//(user_version, redo sql, undo sql +&[(1,"ALTER TABLE PERSON ADD COLUMN interests TEXT;","ALTER TABLE PERSON DROP COLUMN interests;"),(2,"CREATE TABLE hobby ( pk INTEGER PRIMARY KEY NOT NULL,title TEXT NOT NULL);","DROP TABLE hobby;"),(3,"ALTER TABLE PERSON ADD COLUMN main_hobby INTEGER REFERENCES hobby(pk) ON DELETE SET NULL;","ALTER TABLE PERSON DROP COLUMN main_hobby;"),]"#.replace([' ', '\n'], "")); +} + +#[test] +fn test_migration() { + init_stderr_logging(); + let tmp_dir = TempDir::new().unwrap(); + let db_path = tmp_dir.path().join("migr.db"); + + let mut conn = rusqlite::Connection::open(db_path.to_str().unwrap()).unwrap(); + conn.execute_batch(FIRST_SCHEMA).unwrap(); + + conn.execute_batch( + "INSERT INTO person(name,address) VALUES('John Doe', 'johndoe@example.com');", + ) + .unwrap(); + + let version = conn.schema_version().unwrap(); + log::trace!("initial schema version is {}", version); + + //assert_eq!(version, migrations[migrations.len() - 1].0); + + conn.migrate(version, MIGRATIONS.last().unwrap().0, MIGRATIONS) + .unwrap(); + /* + * CREATE TABLE sqlite_schema ( + type text, + name text, + tbl_name text, + rootpage integer, + sql text + ); + */ + let get_sql = |table: &str, conn: &rusqlite::Connection| -> String { + conn.prepare("SELECT sql FROM sqlite_schema WHERE name = ?;") + .unwrap() + .query_row([table], |row| { + let sql: String = row.get(0)?; + Ok(sql) + }) + .unwrap() + }; + + let strip_ws = |sql: &str| -> String { sql.replace([' ', '\n'], "") }; + + let person_sql: String = get_sql("person", &conn); + assert_eq!( + &strip_ws(&person_sql), + &strip_ws( + r#" +CREATE TABLE person ( + pk INTEGER PRIMARY KEY NOT NULL, + name TEXT, + address TEXT NOT NULL, + created INTEGER NOT NULL DEFAULT (unixepoch()), + last_modified INTEGER NOT NULL DEFAULT (unixepoch()), + interests TEXT, + main_hobby INTEGER REFERENCES hobby(pk) ON DELETE SET NULL +)"# + ) + ); + let hobby_sql: String = get_sql("hobby", &conn); + assert_eq!( + &strip_ws(&hobby_sql), + &strip_ws( + r#"CREATE TABLE hobby ( + pk INTEGER PRIMARY KEY NOT NULL, + title TEXT NOT NULL +)"# + ) + ); + conn.execute_batch( + r#" + INSERT INTO hobby(title) VALUES('fishing'); + INSERT INTO hobby(title) VALUES('reading books'); + INSERT INTO hobby(title) VALUES('running'); + INSERT INTO hobby(title) VALUES('forest walks'); + UPDATE person SET main_hobby = hpk FROM (SELECT pk AS hpk FROM hobby LIMIT 1) WHERE name = 'John Doe'; + "# + ) + .unwrap(); + log::trace!( + "John Doe's main hobby is {:?}", + conn.prepare( + "SELECT pk, title FROM hobby WHERE EXISTS (SELECT 1 FROM person AS p WHERE \ + p.main_hobby = pk);" + ) + .unwrap() + .query_row([], |row| { + let pk: i64 = row.get(0)?; + let title: String = row.get(1)?; + Ok((pk, title)) + }) + .unwrap() + ); + + conn.migrate(MIGRATIONS.last().unwrap().0, 0, MIGRATIONS) + .unwrap(); + + assert_eq!( + conn.prepare("SELECT sql FROM sqlite_schema WHERE name = 'hobby';") + .unwrap() + .query_row([], |row| { row.get::<_, String>(0) }) + .unwrap_err(), + rusqlite::Error::QueryReturnedNoRows + ); + let person_sql: String = get_sql("person", &conn); + assert_eq!( + &strip_ws(&person_sql), + &strip_ws( + r#" +CREATE TABLE person ( + pk INTEGER PRIMARY KEY NOT NULL, + name TEXT, + address TEXT NOT NULL, + created INTEGER NOT NULL DEFAULT (unixepoch()), + last_modified INTEGER NOT NULL DEFAULT (unixepoch()) +)"# + ) + ); +} + +const FIRST_SCHEMA: &str = r#" +PRAGMA foreign_keys = true; +PRAGMA encoding = 'UTF-8'; +PRAGMA schema_version = 0; + +CREATE TABLE IF NOT EXISTS person ( + pk INTEGER PRIMARY KEY NOT NULL, + name TEXT, + address TEXT NOT NULL, + created INTEGER NOT NULL DEFAULT (unixepoch()), + last_modified INTEGER NOT NULL DEFAULT (unixepoch()) +); +"#; + +const MIGRATIONS: &[(u32, &str, &str)] = &[ + ( + 1, + "ALTER TABLE PERSON ADD COLUMN interests TEXT;", + "ALTER TABLE PERSON DROP COLUMN interests;", + ), + ( + 2, + "CREATE TABLE hobby ( pk INTEGER PRIMARY KEY NOT NULL,title TEXT NOT NULL);", + "DROP TABLE hobby;", + ), + ( + 3, + "ALTER TABLE PERSON ADD COLUMN main_hobby INTEGER REFERENCES hobby(pk) ON DELETE SET NULL;", + "ALTER TABLE PERSON DROP COLUMN main_hobby;", + ), +];