diff --git a/Cargo.lock b/Cargo.lock index 99f08f7..03a666a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -142,86 +142,6 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" -[[package]] -name = "async-attributes" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5" -dependencies = [ - "quote", - "syn 1.0.109", -] - -[[package]] -name = "async-channel" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" -dependencies = [ - "concurrent-queue", - "event-listener", - "futures-core", -] - -[[package]] -name = "async-executor" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fa3dc5f2a8564f07759c008b9109dc0d39de92a88d5588b8a5036d286383afb" -dependencies = [ - "async-lock", - "async-task", - "concurrent-queue", - "fastrand", - "futures-lite", - "slab", -] - -[[package]] -name = "async-global-executor" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" -dependencies = [ - "async-channel", - "async-executor", - "async-io", - "async-lock", - "blocking", - "futures-lite", - "once_cell", - "tokio", -] - -[[package]] -name = "async-io" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" -dependencies = [ - "async-lock", - "autocfg", - "cfg-if", - "concurrent-queue", - "futures-lite", - "log", - "parking", - "polling", - "rustix 0.37.23", - "slab", - "socket2", - "waker-fn", -] - -[[package]] -name = "async-lock" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa24f727524730b077666307f2734b4a1a1c57acb79193127dcc8914d5242dd7" -dependencies = [ - "event-listener", -] - [[package]] name = "async-recursion" version = "1.0.4" @@ -233,33 +153,6 @@ dependencies = [ "syn 2.0.26", ] -[[package]] -name = "async-std" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" -dependencies = [ - "async-attributes", - "async-channel", - "async-global-executor", - "async-io", - "async-lock", - "crossbeam-utils", - "futures-channel", - "futures-core", - "futures-io", - "futures-lite", - "gloo-timers", - "kv-log-macro", - "log", - "memchr", - "once_cell", - "pin-project-lite", - "pin-utils", - "slab", - "wasm-bindgen-futures", -] - [[package]] name = "async-stream" version = "0.3.5" @@ -282,12 +175,6 @@ dependencies = [ "syn 2.0.26", ] -[[package]] -name = "async-task" -version = "4.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc7ab41815b3c653ccd2978ec3255c81349336702dfdf62ee6f7069b12a3aae" - [[package]] name = "async-trait" version = "0.1.71" @@ -308,12 +195,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "atomic-waker" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1181e1e0d1fce796a03db1ae795d67167da795f9cf4a39c37589e85ef57f26d3" - [[package]] name = "autocfg" version = "1.1.0" @@ -498,21 +379,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "blocking" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77231a1c8f801696fc0123ec6150ce92cffb8e164a02afb9c8ddee0e9b65ad65" -dependencies = [ - "async-channel", - "async-lock", - "async-task", - "atomic-waker", - "fastrand", - "futures-lite", - "log", -] - [[package]] name = "borsh" version = "0.10.3" @@ -771,15 +637,6 @@ dependencies = [ "yansi", ] -[[package]] -name = "concurrent-queue" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "config" version = "0.13.3" @@ -986,7 +843,6 @@ dependencies = [ "config", "cron", "cronback-dto", - "cronback-migration", "cronback-proto", "derive_more", "futures", @@ -1018,14 +874,6 @@ dependencies = [ "validator", ] -[[package]] -name = "cronback-migration" -version = "0.1.0" -dependencies = [ - "async-std", - "sea-orm-migration", -] - [[package]] name = "cronback-proto" version = "0.1.0" @@ -1070,6 +918,7 @@ dependencies = [ "regex", "reqwest", "sea-orm", + "sea-orm-migration", "sea-query", "sea-query-binder", "serde", @@ -1502,21 +1351,6 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" -[[package]] -name = "futures-lite" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" -dependencies = [ - "fastrand", - "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite", - "waker-fn", -] - [[package]] name = "futures-macro" version = "0.3.28" @@ -1591,18 +1425,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" -[[package]] -name = "gloo-timers" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "h2" version = "0.3.20" @@ -1952,15 +1774,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "kv-log-macro" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" -dependencies = [ - "log", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -2011,9 +1824,6 @@ name = "log" version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" -dependencies = [ - "value-bag", -] [[package]] name = "mach2" @@ -2386,12 +2196,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" -[[package]] -name = "parking" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e" - [[package]] name = "parking_lot" version = "0.11.2" @@ -2580,22 +2384,6 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" -[[package]] -name = "polling" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" -dependencies = [ - "autocfg", - "bitflags 1.3.2", - "cfg-if", - "concurrent-queue", - "libc", - "log", - "pin-project-lite", - "windows-sys", -] - [[package]] name = "portable-atomic" version = "1.4.0" @@ -4272,12 +4060,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" -[[package]] -name = "value-bag" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d92ccd67fb88503048c01b59152a04effd0782d035a83a6d256ce6085f08f4a3" - [[package]] name = "vcpkg" version = "0.2.15" @@ -4290,12 +4072,6 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" -[[package]] -name = "waker-fn" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" - [[package]] name = "want" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index 7aa85eb..3a39b31 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,6 @@ cronback-api-model = { path = "cronback-api-model", version = "0.1.0" } cronback-client = { path = "clients/rust", package = "cronback-client", version = "0.1.0" } dto = { path = "cronback-dto", package = "cronback-dto", version = "0.1.0" } lib = { path = "cronback-lib", package = "cronback-lib", version = "0.1.0" } -migration = { path = "cronback-migration", package = "cronback-migration", version = "0.1.0" } proto = { path = "cronback-proto", package = "cronback-proto", version = "0.1.0" } anyhow = "1.0.69" diff --git a/cronback-lib/Cargo.toml b/cronback-lib/Cargo.toml index 056857d..7414086 100644 --- a/cronback-lib/Cargo.toml +++ b/cronback-lib/Cargo.toml @@ -17,7 +17,6 @@ path = "lib.rs" # Internal Dependencies proto = { workspace = true } dto = { workspace = true } -migration = { workspace = true } # Dependencies from workspace anyhow = { workspace = true } diff --git a/cronback-lib/database/mod.rs b/cronback-lib/database/mod.rs index 4816533..637f5b2 100644 --- a/cronback-lib/database/mod.rs +++ b/cronback-lib/database/mod.rs @@ -2,9 +2,7 @@ mod errors; mod pagination; pub use errors::DatabaseError; -use migration::{Migrator, MigratorTrait}; pub use pagination::*; -use sea_orm::TransactionTrait; #[derive(Clone)] pub struct Database { @@ -12,22 +10,13 @@ pub struct Database { } impl Database { - pub async fn connect(conn_string: &str) -> Result { + pub async fn connect(conn_string: &str) -> Result { Ok(Self { orm: sea_orm::Database::connect(conn_string).await?, }) } - pub async fn in_memory() -> Result { - let conn = Self::connect("sqlite::memory:").await?; - conn.migrate().await?; - Ok(conn) - } - - pub async fn migrate(&self) -> Result<(), sea_orm::DbErr> { - let conn = self.orm.begin().await?; - Migrator::up(&conn, None).await?; - conn.commit().await?; - Ok(()) + pub async fn in_memory() -> Result { + Self::connect("sqlite::memory:").await } } diff --git a/cronback-lib/default.toml b/cronback-lib/default.toml index 5f2c794..8dac8ad 100644 --- a/cronback-lib/default.toml +++ b/cronback-lib/default.toml @@ -19,7 +19,7 @@ prometheus_port = 9000 [api] address = "0.0.0.0" port = 8888 -database_uri = "sqlite://database.sqlite?mode=rwc" +database_uri = "sqlite://api.sqlite?mode=rwc" admin_api_keys = [] log_request_body = true log_response_body = true @@ -29,7 +29,7 @@ cell_id = 0 address = "0.0.0.0" port = 9999 request_processing_timeout_s = 30 -database_uri = "sqlite://database.sqlite?mode=rwc" +database_uri = "sqlite://dispatcher.sqlite?mode=rwc" [scheduler] cell_id = 0 @@ -38,7 +38,7 @@ port = 9811 request_processing_timeout_s = 30 spinner_yield_max_ms = 250 max_triggers_per_tick = 100000 -database_uri = "sqlite://database.sqlite?mode=rwc" +database_uri = "sqlite://scheduler.sqlite?mode=rwc" db_flush_s = 10 dangerous_fast_forward = false @@ -47,4 +47,4 @@ cell_id = 0 address = "0.0.0.0" port = 9998 request_processing_timeout_s = 30 -database_uri = "sqlite://database.sqlite?mode=rwc" +database_uri = "sqlite://metadata.sqlite?mode=rwc" diff --git a/cronback-migration/Cargo.toml b/cronback-migration/Cargo.toml deleted file mode 100644 index 6f39d40..0000000 --- a/cronback-migration/Cargo.toml +++ /dev/null @@ -1,28 +0,0 @@ -[package] -name = "cronback-migration" -description = "Cronback database migration" -version.workspace = true -edition.workspace = true -authors.workspace = true -rust-version.workspace = true -license.workspace = true -homepage.workspace = true -documentation.workspace = true -repository.workspace = true - -[lib] -name = "migration" -path = "src/lib.rs" - -[dependencies] -async-std = { version = "1", features = ["attributes", "tokio1"] } - -[dependencies.sea-orm-migration] -version = "0.11.0" -features = [ - # Enable at least one `ASYNC_RUNTIME` and `DATABASE_DRIVER` feature if you want to run migration via CLI. - # View the list of supported features at https://www.sea-ql.org/SeaORM/docs/install-and-config/database-and-async-runtime. - # e.g. - "runtime-tokio-native-tls", # `ASYNC_RUNTIME` feature - "sqlx-sqlite", # `DATABASE_DRIVER` feature -] diff --git a/cronback-migration/README.md b/cronback-migration/README.md deleted file mode 100644 index b3ea53e..0000000 --- a/cronback-migration/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# Running Migrator CLI - -- Generate a new migration file - ```sh - cargo run -- migrate generate MIGRATION_NAME - ``` -- Apply all pending migrations - ```sh - cargo run - ``` - ```sh - cargo run -- up - ``` -- Apply first 10 pending migrations - ```sh - cargo run -- up -n 10 - ``` -- Rollback last applied migrations - ```sh - cargo run -- down - ``` -- Rollback last 10 applied migrations - ```sh - cargo run -- down -n 10 - ``` -- Drop all tables from the database, then reapply all migrations - ```sh - cargo run -- fresh - ``` -- Rollback all applied migrations, then reapply all migrations - ```sh - cargo run -- refresh - ``` -- Rollback all applied migrations - ```sh - cargo run -- reset - ``` -- Check the status of all migrations - ```sh - cargo run -- status - ``` diff --git a/cronback-migration/src/main.rs b/cronback-migration/src/main.rs deleted file mode 100644 index c6b6e48..0000000 --- a/cronback-migration/src/main.rs +++ /dev/null @@ -1,6 +0,0 @@ -use sea_orm_migration::prelude::*; - -#[async_std::main] -async fn main() { - cli::run_cli(migration::Migrator).await; -} diff --git a/cronback-services/Cargo.toml b/cronback-services/Cargo.toml index e81a4d1..b93d3be 100644 --- a/cronback-services/Cargo.toml +++ b/cronback-services/Cargo.toml @@ -50,7 +50,7 @@ ulid = { workspace = true } once_cell = { workspace = true } # Unique Dependencies -async-recursion = "1.0.4" +async-recursion = { version = "1.0.4" } axum-extra = { version = "0.7", features = ["query"] } dashmap = { version = "5.5.0" } hyper = "0.14.26" @@ -60,4 +60,11 @@ sha2 = "0.10.6" uuid = { version = "1.2.2", features = ["v4"] } validator = { version = "0.16.0", features = ["derive"] } regex = { version = "1.9.1" } - +sea-orm-migration = { version = "0.11.0", features = [ + # Enable at least one `ASYNC_RUNTIME` and `DATABASE_DRIVER` feature if you want to run migration via CLI. + # View the list of supported features at https://www.sea-ql.org/SeaORM/docs/install-and-config/database-and-async-runtime. + # e.g. + "runtime-tokio-native-tls", # `ASYNC_RUNTIME` feature + "sqlx-sqlite", # `DATABASE_DRIVER` feature + "sqlx-postgres", # `DATABASE_DRIVER` feature +] } diff --git a/cronback-services/src/api/auth.rs b/cronback-services/src/api/auth.rs index cbeee79..fbe597c 100644 --- a/cronback-services/src/api/auth.rs +++ b/cronback-services/src/api/auth.rs @@ -236,6 +236,7 @@ mod tests { use super::*; use crate::api::auth_store::SqlAuthStore; + use crate::api::migrate_up; #[test] fn test_api_key() { @@ -266,6 +267,7 @@ mod tests { #[tokio::test] async fn test_sql_auth_store() -> anyhow::Result<()> { let db = Database::in_memory().await?; + migrate_up(&db).await?; let store = SqlAuthStore::new(db); let prj1 = ProjectId::generate(); diff --git a/cronback-services/src/api/auth_store.rs b/cronback-services/src/api/auth_store.rs index e0660d7..3b9cbf5 100644 --- a/cronback-services/src/api/auth_store.rs +++ b/cronback-services/src/api/auth_store.rs @@ -84,6 +84,7 @@ mod tests { use chrono::Utc; use super::*; + use crate::api::migrate_up; fn build_model( key_id: &str, @@ -105,6 +106,7 @@ mod tests { #[tokio::test] async fn test_sql_auth_store() -> anyhow::Result<()> { let db = Database::in_memory().await?; + migrate_up(&db).await?; let store = SqlAuthStore::new(db); let owner1 = ProjectId::generate(); diff --git a/cronback-migration/src/m20220101_000001_create_api_keys_table.rs b/cronback-services/src/api/migration/m20220101_000001_create_api_keys_table.rs similarity index 100% rename from cronback-migration/src/m20220101_000001_create_api_keys_table.rs rename to cronback-services/src/api/migration/m20220101_000001_create_api_keys_table.rs diff --git a/cronback-services/src/api/migration/mod.rs b/cronback-services/src/api/migration/mod.rs new file mode 100644 index 0000000..e2afcc1 --- /dev/null +++ b/cronback-services/src/api/migration/mod.rs @@ -0,0 +1,12 @@ +pub use sea_orm_migration::prelude::*; + +mod m20220101_000001_create_api_keys_table; + +pub struct Migrator; + +#[async_trait::async_trait] +impl MigratorTrait for Migrator { + fn migrations() -> Vec> { + vec![Box::new(m20220101_000001_create_api_keys_table::Migration)] + } +} diff --git a/cronback-services/src/api/mod.rs b/cronback-services/src/api/mod.rs index f0dd028..eb7b0dd 100644 --- a/cronback-services/src/api/mod.rs +++ b/cronback-services/src/api/mod.rs @@ -7,6 +7,7 @@ pub mod errors; pub(crate) mod extractors; mod handlers; mod logging; +mod migration; mod paginated; use std::net::SocketAddr; @@ -30,6 +31,8 @@ use lib::prelude::*; use lib::{netutils, service, Config, GrpcClientFactory, GrpcClientProvider}; use logging::{trace_request_response, ApiMakeSpan}; use metrics::{histogram, increment_counter}; +use sea_orm::TransactionTrait; +use sea_orm_migration::MigratorTrait; use thiserror::Error; use tokio::select; use tower_http::cors::{AllowOrigin, CorsLayer}; @@ -62,6 +65,14 @@ async fn fallback() -> (StatusCode, &'static str) { (StatusCode::NOT_FOUND, "Not Found") } +// TODO: Move database migration into a new service trait. +pub async fn migrate_up(db: &Database) -> Result<(), DatabaseError> { + let conn = db.orm.begin().await?; + migration::Migrator::up(&conn, None).await?; + conn.commit().await?; + Ok(()) +} + #[tracing::instrument(skip_all, fields(service = context.service_name()))] pub async fn start_api_server( mut context: service::ServiceContext, @@ -71,7 +82,7 @@ pub async fn start_api_server( netutils::parse_addr(&config.api.address, config.api.port).unwrap(); let db = Database::connect(&config.api.database_uri).await?; - db.migrate().await?; + migrate_up(&db).await?; let shared_state = Arc::new(AppState { _context: context.clone(), diff --git a/cronback-services/src/dispatcher/attempt_store.rs b/cronback-services/src/dispatcher/attempt_store.rs index 4dd5d63..e367378 100644 --- a/cronback-services/src/dispatcher/attempt_store.rs +++ b/cronback-services/src/dispatcher/attempt_store.rs @@ -89,6 +89,7 @@ mod tests { AttemptStatus, WebhookAttemptDetails, }; + use crate::dispatcher::migrate_up; fn build_attempt( project: &ValidShardedId, @@ -119,6 +120,7 @@ mod tests { #[tokio::test] async fn test_sql_trigger_store() -> anyhow::Result<()> { let db = Database::in_memory().await?; + migrate_up(&db).await?; let store = SqlAttemptLogStore::new(db); let project = ProjectId::generate(); diff --git a/cronback-migration/src/m20230520_213613_create_attempts.rs b/cronback-services/src/dispatcher/migration/m20230520_213613_create_attempts.rs similarity index 100% rename from cronback-migration/src/m20230520_213613_create_attempts.rs rename to cronback-services/src/dispatcher/migration/m20230520_213613_create_attempts.rs diff --git a/cronback-migration/src/m20230521_221728_create_runs.rs b/cronback-services/src/dispatcher/migration/m20230521_221728_create_runs.rs similarity index 100% rename from cronback-migration/src/m20230521_221728_create_runs.rs rename to cronback-services/src/dispatcher/migration/m20230521_221728_create_runs.rs diff --git a/cronback-migration/src/lib.rs b/cronback-services/src/dispatcher/migration/mod.rs similarity index 55% rename from cronback-migration/src/lib.rs rename to cronback-services/src/dispatcher/migration/mod.rs index 40228a6..eda891a 100644 --- a/cronback-migration/src/lib.rs +++ b/cronback-services/src/dispatcher/migration/mod.rs @@ -1,10 +1,7 @@ pub use sea_orm_migration::prelude::*; -mod m20220101_000001_create_api_keys_table; mod m20230520_213613_create_attempts; mod m20230521_221728_create_runs; -mod m20230521_233041_create_triggers; -mod m20230712_205649_add_projects_model; pub struct Migrator; @@ -12,11 +9,8 @@ pub struct Migrator; impl MigratorTrait for Migrator { fn migrations() -> Vec> { vec![ - Box::new(m20220101_000001_create_api_keys_table::Migration), Box::new(m20230520_213613_create_attempts::Migration), Box::new(m20230521_221728_create_runs::Migration), - Box::new(m20230521_233041_create_triggers::Migration), - Box::new(m20230712_205649_add_projects_model::Migration), ] } } diff --git a/cronback-services/src/dispatcher/mod.rs b/cronback-services/src/dispatcher/mod.rs index 14138af..9ac3a60 100644 --- a/cronback-services/src/dispatcher/mod.rs +++ b/cronback-services/src/dispatcher/mod.rs @@ -2,6 +2,7 @@ mod attempt_store; mod db_model; mod dispatch_manager; mod handler; +mod migration; mod retry; mod run_store; mod webhook_action; @@ -14,8 +15,18 @@ use lib::prelude::*; use lib::{netutils, service}; use proto::dispatcher_svc::dispatcher_svc_server::DispatcherSvcServer; use run_store::{RunStore, SqlRunStore}; +use sea_orm::TransactionTrait; +use sea_orm_migration::MigratorTrait; use tracing::info; +// TODO: Move database migration into a new service trait. +pub async fn migrate_up(db: &Database) -> Result<(), DatabaseError> { + let conn = db.orm.begin().await?; + migration::Migrator::up(&conn, None).await?; + conn.commit().await?; + Ok(()) +} + #[tracing::instrument(skip_all, fields(service = context.service_name()))] pub async fn start_dispatcher_server( mut context: service::ServiceContext, @@ -28,7 +39,8 @@ pub async fn start_dispatcher_server( .unwrap(); let db = Database::connect(&config.dispatcher.database_uri).await?; - db.migrate().await?; + migrate_up(&db).await?; + let attempt_store: Arc = Arc::new(SqlAttemptLogStore::new(db.clone())); diff --git a/cronback-services/src/dispatcher/run_store.rs b/cronback-services/src/dispatcher/run_store.rs index 8c206b2..01bfd05 100644 --- a/cronback-services/src/dispatcher/run_store.rs +++ b/cronback-services/src/dispatcher/run_store.rs @@ -160,6 +160,7 @@ mod tests { use sea_orm::DbErr; use super::*; + use crate::dispatcher::migrate_up; fn build_run( trigger_id: ValidShardedId, @@ -190,6 +191,7 @@ mod tests { #[tokio::test] async fn test_sql_run_store() -> anyhow::Result<()> { let db = Database::in_memory().await?; + migrate_up(&db).await?; let store = SqlRunStore::new(db); let project1 = ProjectId::generate(); diff --git a/cronback-services/src/metadata/metadata_store.rs b/cronback-services/src/metadata/metadata_store.rs index 32f2c89..b797b55 100644 --- a/cronback-services/src/metadata/metadata_store.rs +++ b/cronback-services/src/metadata/metadata_store.rs @@ -90,6 +90,7 @@ impl MetadataStore for SqlMetadataStore { mod tests { use super::*; + use crate::metadata::migrate_up; fn build_project(status: ProjectStatus) -> Project { let now = Utc::now(); @@ -105,6 +106,7 @@ mod tests { #[tokio::test] async fn test_sql_project_store() -> anyhow::Result<()> { let db = Database::in_memory().await?; + migrate_up(&db).await?; let store = SqlMetadataStore::new(db); let project1 = build_project(ProjectStatus::Enabled); diff --git a/cronback-migration/src/m20230712_205649_add_projects_model.rs b/cronback-services/src/metadata/migration/m20230712_205649_add_projects_model.rs similarity index 100% rename from cronback-migration/src/m20230712_205649_add_projects_model.rs rename to cronback-services/src/metadata/migration/m20230712_205649_add_projects_model.rs diff --git a/cronback-services/src/metadata/migration/mod.rs b/cronback-services/src/metadata/migration/mod.rs new file mode 100644 index 0000000..011bb34 --- /dev/null +++ b/cronback-services/src/metadata/migration/mod.rs @@ -0,0 +1,12 @@ +pub use sea_orm_migration::prelude::*; + +mod m20230712_205649_add_projects_model; + +pub struct Migrator; + +#[async_trait::async_trait] +impl MigratorTrait for Migrator { + fn migrations() -> Vec> { + vec![Box::new(m20230712_205649_add_projects_model::Migration)] + } +} diff --git a/cronback-services/src/metadata/mod.rs b/cronback-services/src/metadata/mod.rs index aaa7643..3a252d6 100644 --- a/cronback-services/src/metadata/mod.rs +++ b/cronback-services/src/metadata/mod.rs @@ -1,6 +1,7 @@ mod db_model; mod handler; mod metadata_store; +mod migration; use std::sync::Arc; @@ -8,8 +9,18 @@ use lib::prelude::*; use lib::{netutils, service}; use metadata_store::{MetadataStore, SqlMetadataStore}; use proto::metadata_svc::metadata_svc_server::MetadataSvcServer; +use sea_orm::TransactionTrait; +use sea_orm_migration::MigratorTrait; use tracing::info; +// TODO: Move database migration into a new service trait. +pub async fn migrate_up(db: &Database) -> Result<(), DatabaseError> { + let conn = db.orm.begin().await?; + migration::Migrator::up(&conn, None).await?; + conn.commit().await?; + Ok(()) +} + #[tracing::instrument(skip_all, fields(service = context.service_name()))] pub async fn start_metadata_server( mut context: service::ServiceContext, @@ -20,7 +31,7 @@ pub async fn start_metadata_server( .unwrap(); let db = Database::connect(&config.metadata.database_uri).await?; - db.migrate().await?; + migrate_up(&db).await?; let store: Arc = Arc::new(SqlMetadataStore::new(db)); diff --git a/cronback-migration/src/m20230521_233041_create_triggers.rs b/cronback-services/src/scheduler/migration/m20230521_233041_create_triggers.rs similarity index 100% rename from cronback-migration/src/m20230521_233041_create_triggers.rs rename to cronback-services/src/scheduler/migration/m20230521_233041_create_triggers.rs diff --git a/cronback-services/src/scheduler/migration/mod.rs b/cronback-services/src/scheduler/migration/mod.rs new file mode 100644 index 0000000..52b4b0d --- /dev/null +++ b/cronback-services/src/scheduler/migration/mod.rs @@ -0,0 +1,12 @@ +pub use sea_orm_migration::prelude::*; + +mod m20230521_233041_create_triggers; + +pub struct Migrator; + +#[async_trait::async_trait] +impl MigratorTrait for Migrator { + fn migrations() -> Vec> { + vec![Box::new(m20230521_233041_create_triggers::Migration)] + } +} diff --git a/cronback-services/src/scheduler/mod.rs b/cronback-services/src/scheduler/mod.rs index ad0b0fc..9a878ad 100644 --- a/cronback-services/src/scheduler/mod.rs +++ b/cronback-services/src/scheduler/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod db_model; pub(crate) mod error; pub(crate) mod handler; +mod migration; pub(crate) mod spinner; pub(crate) mod trigger_store; @@ -11,9 +12,19 @@ use handler::SchedulerSvcHandler; use lib::prelude::*; use lib::{netutils, service, GrpcClientProvider}; use proto::scheduler_svc::scheduler_svc_server::SchedulerSvcServer; +use sea_orm::TransactionTrait; +use sea_orm_migration::MigratorTrait; use spinner::controller::SpinnerController; use trigger_store::SqlTriggerStore; +// TODO: Move database migration into a new service trait. +pub async fn migrate_up(db: &Database) -> Result<(), DatabaseError> { + let conn = db.orm.begin().await?; + migration::Migrator::up(&conn, None).await?; + conn.commit().await?; + Ok(()) +} + #[tracing::instrument(skip_all, fields(service = context.service_name()))] pub async fn start_scheduler_server( mut context: service::ServiceContext, @@ -21,7 +32,8 @@ pub async fn start_scheduler_server( let config = context.load_config(); let db = Database::connect(&config.scheduler.database_uri).await?; - db.migrate().await?; + migrate_up(&db).await?; + let trigger_store = Arc::new(SqlTriggerStore::new(db)); let dispatcher_clients = Arc::new(GrpcClientProvider::new(context.clone())); @@ -89,6 +101,8 @@ pub mod test_helpers { Arc::new(GrpcClientProvider::new(context.clone())); let db = Database::in_memory().await.unwrap(); + migrate_up(&db).await.unwrap(); + let trigger_store = Arc::new(SqlTriggerStore::new(db)); let controller = Arc::new(SpinnerController::new( context.clone(), diff --git a/cronback-services/src/scheduler/trigger_store.rs b/cronback-services/src/scheduler/trigger_store.rs index 45e4c85..7cfbcd0 100644 --- a/cronback-services/src/scheduler/trigger_store.rs +++ b/cronback-services/src/scheduler/trigger_store.rs @@ -211,6 +211,7 @@ mod tests { use chrono::{Timelike, Utc}; use super::*; + use crate::scheduler::migrate_up; fn build_trigger( name: &str, @@ -245,6 +246,8 @@ mod tests { #[tokio::test] async fn test_sql_trigger_store() -> anyhow::Result<()> { let db = Database::in_memory().await?; + migrate_up(&db).await?; + let store = SqlTriggerStore::new(db); let project1 = ProjectId::generate();