core: add migration test

topics
Manos Pitsidianakis 2 weeks ago
parent e8120c75db
commit 657b58c4ae
Signed by: Manos Pitsidianakis
GPG Key ID: 7729C7707F7E09D0

@ -506,7 +506,6 @@ pub enum ListCommand {
pub struct QueueValueParser;
impl QueueValueParser {
/// Implementation for [`ValueParser::path_buf`]
pub fn new() -> Self {
Self
}

@ -18,10 +18,8 @@
*/
use std::{
fs::{metadata, read_dir, OpenOptions},
fs::{metadata, OpenOptions},
io,
io::Write,
path::Path,
process::{Command, Stdio},
};
@ -46,7 +44,12 @@ where
}
}
include!("make_migrations.rs");
const MIGRATION_RS: &str = "src/migrations.rs.inc";
fn main() {
println!("cargo:rerun-if-changed=src/migrations.rs.inc");
println!("cargo:rerun-if-changed=migrations");
println!("cargo:rerun-if-changed=src/schema.sql.m4");
@ -88,71 +91,5 @@ fn main() {
file.write_all(&output.stdout).unwrap();
}
const MIGRATION_RS: &str = "src/migrations.rs.inc";
let mut regen = false;
let mut paths = vec![];
let mut undo_paths = vec![];
for entry in read_dir("migrations").unwrap() {
let entry = entry.unwrap();
let path = entry.path();
if path.is_dir() || path.extension().map(|os| os.to_str().unwrap()) != Some("sql") {
continue;
}
if is_output_file_outdated(&path, MIGRATION_RS).unwrap() {
regen = true;
}
if path
.file_name()
.unwrap()
.to_str()
.unwrap()
.ends_with("undo.sql")
{
undo_paths.push(path);
} else {
paths.push(path);
}
}
if regen {
paths.sort();
undo_paths.sort();
let mut migr_rs = OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.open(MIGRATION_RS)
.unwrap();
migr_rs
.write_all(b"\n//(user_version, redo sql, undo sql\n&[")
.unwrap();
for (p, u) in paths.iter().zip(undo_paths.iter()) {
// This should be a number string, padded with 2 zeros if it's less than 3
// digits. e.g. 001, \d{3}
let num = p.file_stem().unwrap().to_str().unwrap();
if !u.file_name().unwrap().to_str().unwrap().starts_with(num) {
panic!("Undo file {u:?} should match with {p:?}");
}
if num.parse::<u32>().is_err() {
panic!("Migration file {p:?} should start with a number");
}
migr_rs.write_all(b"(").unwrap();
migr_rs
.write_all(num.trim_start_matches('0').as_bytes())
.unwrap();
migr_rs.write_all(b",\"").unwrap();
migr_rs
.write_all(std::fs::read_to_string(p).unwrap().as_bytes())
.unwrap();
migr_rs.write_all(b"\",\"").unwrap();
migr_rs
.write_all(std::fs::read_to_string(u).unwrap().as_bytes())
.unwrap();
migr_rs.write_all(b"\"),").unwrap();
}
migr_rs.write_all(b"]").unwrap();
migr_rs.flush().unwrap();
}
make_migrations("migrations", MIGRATION_RS);
}

@ -0,0 +1,92 @@
/*
* This file is part of mailpot
*
* Copyright 2023 - Manos Pitsidianakis
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
use std::{fs::read_dir, io::Write, path::Path};
pub fn make_migrations<M: AsRef<Path>, O: AsRef<Path>>(migrations_path: M, output_file: O) {
let migrations_folder_path = migrations_path.as_ref();
let output_file_path = output_file.as_ref();
let mut regen = false;
let mut paths = vec![];
let mut undo_paths = vec![];
for entry in read_dir(migrations_folder_path).unwrap() {
let entry = entry.unwrap();
let path = entry.path();
if path.is_dir() || path.extension().map(|os| os.to_str().unwrap()) != Some("sql") {
continue;
}
if is_output_file_outdated(&path, output_file_path).unwrap() {
regen = true;
}
if path
.file_name()
.unwrap()
.to_str()
.unwrap()
.ends_with("undo.sql")
{
undo_paths.push(path);
} else {
paths.push(path);
}
}
if regen {
paths.sort();
undo_paths.sort();
let mut migr_rs = OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.open(output_file_path)
.unwrap();
migr_rs
.write_all(b"\n//(user_version, redo sql, undo sql\n&[")
.unwrap();
for (i, (p, u)) in paths.iter().zip(undo_paths.iter()).enumerate() {
// This should be a number string, padded with 2 zeros if it's less than 3
// digits. e.g. 001, \d{3}
let num = p.file_stem().unwrap().to_str().unwrap();
if !u.file_name().unwrap().to_str().unwrap().starts_with(num) {
panic!("Undo file {u:?} should match with {p:?}");
}
if num.parse::<u32>().is_err() {
panic!("Migration file {p:?} should start with a number");
}
assert_eq!(num.parse::<usize>().unwrap(), i + 1, "migration sql files should start with 1, not zero, and no intermediate numbers should be missing. Panicked on file: {}", p.display());
migr_rs.write_all(b"(").unwrap();
migr_rs
.write_all(num.trim_start_matches('0').as_bytes())
.unwrap();
migr_rs.write_all(b",\"").unwrap();
migr_rs
.write_all(std::fs::read_to_string(p).unwrap().as_bytes())
.unwrap();
migr_rs.write_all(b"\",\"").unwrap();
migr_rs
.write_all(std::fs::read_to_string(u).unwrap().as_bytes())
.unwrap();
migr_rs.write_all(b"\"),").unwrap();
}
migr_rs.write_all(b"]").unwrap();
migr_rs.flush().unwrap();
}
}

@ -184,7 +184,7 @@ impl Connection {
return Err("Database doesn't exist".into());
}
INIT_SQLITE_LOGGING.call_once(|| {
unsafe { rusqlite::trace::config_log(Some(log_callback)).unwrap() };
_ = unsafe { rusqlite::trace::config_log(Some(log_callback)) };
});
let conn = DbConnection::open(conf.db_path.to_str().unwrap())?;
rusqlite::vtab::array::load_module(&conn)?;

@ -17,10 +17,34 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
use std::fs::{metadata, File, OpenOptions};
use mailpot::{Configuration, Connection, SendMail};
use mailpot_tests::init_stderr_logging;
use tempfile::TempDir;
// Source: https://stackoverflow.com/a/64535181
fn is_output_file_outdated<P1, P2>(input: P1, output: P2) -> std::io::Result<bool>
where
P1: AsRef<Path>,
P2: AsRef<Path>,
{
let out_meta = metadata(output);
if let Ok(meta) = out_meta {
let output_mtime = meta.modified()?;
// if input file is more recent than our output, we are outdated
let input_meta = metadata(input)?;
let input_mtime = input_meta.modified()?;
Ok(input_mtime > output_mtime)
} else {
// output file not found, we are outdated
Ok(true)
}
}
include!("../make_migrations.rs");
#[test]
fn test_init_empty() {
init_stderr_logging();
@ -49,3 +73,291 @@ fn test_init_empty() {
db.migrate(migrations[0].0, version).unwrap();
}
trait ConnectionExt {
fn schema_version(&self) -> Result<u32, rusqlite::Error>;
fn migrate(
&mut self,
from: u32,
to: u32,
migrations: &[(u32, &str, &str)],
) -> Result<(), rusqlite::Error>;
}
impl ConnectionExt for rusqlite::Connection {
fn schema_version(&self) -> Result<u32, rusqlite::Error> {
self.prepare("SELECT user_version FROM pragma_user_version;")?
.query_row([], |row| {
let v: u32 = row.get(0)?;
Ok(v)
})
}
fn migrate(
&mut self,
mut from: u32,
to: u32,
migrations: &[(u32, &str, &str)],
) -> Result<(), rusqlite::Error> {
if from == to {
return Ok(());
}
let undo = from > to;
let tx = self.transaction()?;
loop {
log::trace!(
"exec migration from {from} to {to}, type: {}do",
if undo { "un" } else { "re" }
);
if undo {
log::trace!("{}", migrations[from as usize - 1].2);
tx.execute_batch(migrations[from as usize - 1].2)?;
from -= 1;
if from == to {
break;
}
} else {
if from != 0 {
log::trace!("{}", migrations[from as usize - 1].1);
tx.execute_batch(migrations[from as usize - 1].1)?;
}
from += 1;
if from == to + 1 {
break;
}
}
}
tx.pragma_update(
None,
"user_version",
if to == 0 {
0
} else {
migrations[to as usize - 1].0
},
)?;
tx.commit()?;
Ok(())
}
}
#[test]
fn test_migration_gen() {
init_stderr_logging();
let tmp_dir = TempDir::new().unwrap();
let in_path = tmp_dir.path().join("migrations");
std::fs::create_dir(&in_path).unwrap();
let out_path = tmp_dir.path().join("migrations.txt");
for (num, redo, undo) in MIGRATIONS.iter() {
let mut redo_file = File::options()
.write(true)
.create(true)
.truncate(true)
.open(&in_path.join(&format!("{num:03}.sql")))
.unwrap();
redo_file.write_all(redo.as_bytes()).unwrap();
redo_file.flush().unwrap();
let mut undo_file = File::options()
.write(true)
.create(true)
.truncate(true)
.open(&in_path.join(&format!("{num:03}.undo.sql")))
.unwrap();
undo_file.write_all(undo.as_bytes()).unwrap();
undo_file.flush().unwrap();
}
make_migrations(&in_path, &out_path);
let output = std::fs::read_to_string(&out_path).unwrap();
assert_eq!(&output.replace([' ', '\n'], ""), &r#"//(user_version, redo sql, undo sql
&[(1,"ALTER TABLE PERSON ADD COLUMN interests TEXT;","ALTER TABLE PERSON DROP COLUMN interests;"),(2,"CREATE TABLE hobby ( pk INTEGER PRIMARY KEY NOT NULL,title TEXT NOT NULL);","DROP TABLE hobby;"),(3,"ALTER TABLE PERSON ADD COLUMN main_hobby INTEGER REFERENCES hobby(pk) ON DELETE SET NULL;","ALTER TABLE PERSON DROP COLUMN main_hobby;"),]"#.replace([' ', '\n'], ""));
}
#[test]
#[should_panic]
fn test_migration_gen_panic() {
init_stderr_logging();
let tmp_dir = TempDir::new().unwrap();
let in_path = tmp_dir.path().join("migrations");
std::fs::create_dir(&in_path).unwrap();
let out_path = tmp_dir.path().join("migrations.txt");
for (num, redo, undo) in MIGRATIONS.iter().skip(1) {
let mut redo_file = File::options()
.write(true)
.create(true)
.truncate(true)
.open(&in_path.join(&format!("{num:03}.sql")))
.unwrap();
redo_file.write_all(redo.as_bytes()).unwrap();
redo_file.flush().unwrap();
let mut undo_file = File::options()
.write(true)
.create(true)
.truncate(true)
.open(&in_path.join(&format!("{num:03}.undo.sql")))
.unwrap();
undo_file.write_all(undo.as_bytes()).unwrap();
undo_file.flush().unwrap();
}
make_migrations(&in_path, &out_path);
let output = std::fs::read_to_string(&out_path).unwrap();
assert_eq!(&output.replace([' ','\n'], ""), &r#"//(user_version, redo sql, undo sql
&[(1,"ALTER TABLE PERSON ADD COLUMN interests TEXT;","ALTER TABLE PERSON DROP COLUMN interests;"),(2,"CREATE TABLE hobby ( pk INTEGER PRIMARY KEY NOT NULL,title TEXT NOT NULL);","DROP TABLE hobby;"),(3,"ALTER TABLE PERSON ADD COLUMN main_hobby INTEGER REFERENCES hobby(pk) ON DELETE SET NULL;","ALTER TABLE PERSON DROP COLUMN main_hobby;"),]"#.replace([' ', '\n'], ""));
}
#[test]
fn test_migration() {
init_stderr_logging();
let tmp_dir = TempDir::new().unwrap();
let db_path = tmp_dir.path().join("migr.db");
let mut conn = rusqlite::Connection::open(db_path.to_str().unwrap()).unwrap();
conn.execute_batch(FIRST_SCHEMA).unwrap();
conn.execute_batch(
"INSERT INTO person(name,address) VALUES('John Doe', 'johndoe@example.com');",
)
.unwrap();
let version = conn.schema_version().unwrap();
log::trace!("initial schema version is {}", version);
//assert_eq!(version, migrations[migrations.len() - 1].0);
conn.migrate(version, MIGRATIONS.last().unwrap().0, MIGRATIONS)
.unwrap();
/*
* CREATE TABLE sqlite_schema (
type text,
name text,
tbl_name text,
rootpage integer,
sql text
);
*/
let get_sql = |table: &str, conn: &rusqlite::Connection| -> String {
conn.prepare("SELECT sql FROM sqlite_schema WHERE name = ?;")
.unwrap()
.query_row([table], |row| {
let sql: String = row.get(0)?;
Ok(sql)
})
.unwrap()
};
let strip_ws = |sql: &str| -> String { sql.replace([' ', '\n'], "") };
let person_sql: String = get_sql("person", &conn);
assert_eq!(
&strip_ws(&person_sql),
&strip_ws(
r#"
CREATE TABLE person (
pk INTEGER PRIMARY KEY NOT NULL,
name TEXT,
address TEXT NOT NULL,
created INTEGER NOT NULL DEFAULT (unixepoch()),
last_modified INTEGER NOT NULL DEFAULT (unixepoch()),
interests TEXT,
main_hobby INTEGER REFERENCES hobby(pk) ON DELETE SET NULL
)"#
)
);
let hobby_sql: String = get_sql("hobby", &conn);
assert_eq!(
&strip_ws(&hobby_sql),
&strip_ws(
r#"CREATE TABLE hobby (
pk INTEGER PRIMARY KEY NOT NULL,
title TEXT NOT NULL
)"#
)
);
conn.execute_batch(
r#"
INSERT INTO hobby(title) VALUES('fishing');
INSERT INTO hobby(title) VALUES('reading books');
INSERT INTO hobby(title) VALUES('running');
INSERT INTO hobby(title) VALUES('forest walks');
UPDATE person SET main_hobby = hpk FROM (SELECT pk AS hpk FROM hobby LIMIT 1) WHERE name = 'John Doe';
"#
)
.unwrap();
log::trace!(
"John Doe's main hobby is {:?}",
conn.prepare(
"SELECT pk, title FROM hobby WHERE EXISTS (SELECT 1 FROM person AS p WHERE \
p.main_hobby = pk);"
)
.unwrap()
.query_row([], |row| {
let pk: i64 = row.get(0)?;
let title: String = row.get(1)?;
Ok((pk, title))
})
.unwrap()
);
conn.migrate(MIGRATIONS.last().unwrap().0, 0, MIGRATIONS)
.unwrap();
assert_eq!(
conn.prepare("SELECT sql FROM sqlite_schema WHERE name = 'hobby';")
.unwrap()
.query_row([], |row| { row.get::<_, String>(0) })
.unwrap_err(),
rusqlite::Error::QueryReturnedNoRows
);
let person_sql: String = get_sql("person", &conn);
assert_eq!(
&strip_ws(&person_sql),
&strip_ws(
r#"
CREATE TABLE person (
pk INTEGER PRIMARY KEY NOT NULL,
name TEXT,
address TEXT NOT NULL,
created INTEGER NOT NULL DEFAULT (unixepoch()),
last_modified INTEGER NOT NULL DEFAULT (unixepoch())
)"#
)
);
}
const FIRST_SCHEMA: &str = r#"
PRAGMA foreign_keys = true;
PRAGMA encoding = 'UTF-8';
PRAGMA schema_version = 0;
CREATE TABLE IF NOT EXISTS person (
pk INTEGER PRIMARY KEY NOT NULL,
name TEXT,
address TEXT NOT NULL,
created INTEGER NOT NULL DEFAULT (unixepoch()),
last_modified INTEGER NOT NULL DEFAULT (unixepoch())
);
"#;
const MIGRATIONS: &[(u32, &str, &str)] = &[
(
1,
"ALTER TABLE PERSON ADD COLUMN interests TEXT;",
"ALTER TABLE PERSON DROP COLUMN interests;",
),
(
2,
"CREATE TABLE hobby ( pk INTEGER PRIMARY KEY NOT NULL,title TEXT NOT NULL);",
"DROP TABLE hobby;",
),
(
3,
"ALTER TABLE PERSON ADD COLUMN main_hobby INTEGER REFERENCES hobby(pk) ON DELETE SET NULL;",
"ALTER TABLE PERSON DROP COLUMN main_hobby;",
),
];

Loading…
Cancel
Save