diff --git a/Gemfile.lock b/Gemfile.lock index 1420e55..8cfedb9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - umbrellio-sequel-plugins (0.16.1) + umbrellio-sequel-plugins (0.17.0) sequel GEM diff --git a/README.md b/README.md index 8b029c7..6655983 100644 --- a/README.md +++ b/README.md @@ -485,6 +485,130 @@ Overrides Rails default `dbconsole` and `db` commands. In order to use it, you h require "umbrellio_sequel_plugins/rails_db_command" ``` +# Database Tasks for ClickHouse and Sequel + +## ClickHouse Rake Tasks + +We have added a set of Rake tasks to manage ClickHouse database migrations and database operations. These tasks are located in the `namespace :ch`. + +### Task: `ch:create` + +Creates a ClickHouse database in the specified cluster. + +```bash +rake ch:create +``` +This task will create a ClickHouse database as defined in the configuration file with the option to specify the cluster using the ClickHouse.config.database. + +Example: +```ruby +ClickHouse.config do |config| + config.assign Rails.application.config_for(:clickhouse) +end +``` + +### Task: `ch:create_migration_table` + +Creates a migration tracking table for ClickHouse in PostgreSQL. This table will be used to track applied migrations for the ClickHouse database. + +```bash +rake ch:create_migration_table +``` + +### Task: `ch:drop` + +Drops the ClickHouse database and truncates the migration tracking table. + +```bash +rake ch:drop +``` + +### Task: `ch:migrate` + +Runs the migrations for the ClickHouse database from the db/migrate/clickhouse directory. + +```bash +rake ch:migrate +``` + +You can specify a version to migrate to using the VERSION environment variable. + +### Task: `ch:rollback` + +Rollbacks the migrations for the ClickHouse database to a specified version. + +```bash +rake ch:rollback VERSION= +``` + +If no version is provided, it rolls back the last migration. + +### Task: `ch:reset` + +Drops, recreates, and runs all migrations for the ClickHouse database. This is useful for resetting the entire ClickHouse setup. + +```bash +rake ch:reset +``` + +### Task: `ch:rollback_missing_migrations` + +Rollbacks any missing migrations for the ClickHouse database by comparing applied migrations to the available migration files. + +```bash +rake ch:rollback_missing_migrations +``` + +### Sequel Rake Tasks + +Several tasks have been added under the namespace :sequel to provide better management of migrations and rollbacks in Sequel. These tasks help in managing PostgreSQL and ClickHouse migrations. + +### Task: `sequel:archive_migrations` + +Archives migration source code into a PostgreSQL table for tracking purposes. This task can now accept custom paths for migrations and source tables. + +```bash +rake sequel:archive_migrations[migrations_path, migration_table_source] +``` + +- `migrations_path`: Path to the migration files (default is `db/migrate/*.rb`). +- `migration_table_source`: Table to store migration source code (default is `:schema_migrations_sources`). + +### `Task: sequel:rollback_archived_migrations` + +Rollbacks migrations that were applied but are no longer present in the current release. The task supports additional options such as custom migration paths, tables, and transaction settings. + +```bash +rake sequel:rollback_archived_migrations[migrations_path, migration_table, migration_table_source, use_transactions] +``` + +- `migrations_path`: Path to the migration files (default is `db/migrate/*.rb`). +- `migration_table`: Table used to track applied migrations (default is `:schema_migrations`). +- `migration_table_source`: Table storing migration source code (default is `:schema_migrations_sources`). +- `use_transactions`: Whether to use transactions for rolling back (default is `false`). + +### Task: `sequel:rollback_missing_migrations` + +Rollbacks migrations that are absent in the current release when deploying to staging or production. This task helps ensure consistency between different versions. + +```bash +rake sequel:rollback_missing_migrations[table, use_transactions] +``` + +- `table`: The table used to track migrations (optional). +- `use_transactions`: Whether to use transactions during rollback (default is `false`). + +### Task: `sequel:rollback_missing_migrations` + +This task specifically helps during deployment by rolling back any migrations that are not present in the current release. + +```bash +rake sequel:rollback_missing_migrations[table, use_transactions] +``` + +- `table`: The table used to track migrations (optional). +- `use_transactions`: Whether or not to use transactions when rolling back (optional). + ## License Released under MIT License. diff --git a/lib/clickhouse/migrator.rb b/lib/clickhouse/migrator.rb new file mode 100644 index 0000000..c571450 --- /dev/null +++ b/lib/clickhouse/migrator.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +# :nocov: +module Clickhouse + module Migrator + module_function + + def migrate(to: nil) + if to.present? + migrator(target: to.to_i).run + else + migrator.run + end + end + + def rollback(to: nil) + target = to || migrator.applied_migrations.reverse[1] + migrator(target: target.to_i).run + end + + def migrator(**opts) + Sequel::TimestampMigrator.new( + DB, + Rails.root.join("db/migrate/clickhouse"), + table: :clickhouse_migrations, + use_transactions: false, + **opts, + ) + end + end +end +# :nocov: diff --git a/lib/tasks/clickhouse.rake b/lib/tasks/clickhouse.rake new file mode 100644 index 0000000..123c5f4 --- /dev/null +++ b/lib/tasks/clickhouse.rake @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require "clickhouse/migrator" + +namespace :ch do + desc "Create a ClickHouse database in the specified cluster" + task create: :environment do + CH.create_database(ClickHouse.config.database, cluster: "click_cluster") + end + + desc "Create a migration tracking table for ClickHouse in PostgreSQL" + task create_migration_table: :environment do + DB.create_table Sequel[:public][:clickhouse_migrations] do + column :filename, :text, null: false, primary_key: true + end + end + + desc "Drop the ClickHouse database and truncate the migration tracking table" + task drop: :environment do + CH.drop_database(ClickHouse.config.database, cluster: "click_cluster") + DB.from(Sequel[:public][:clickhouse_migrations]).truncate + DB.from(Sequel[:public][:clickhouse_migrations_sources]).truncate + end + + desc "Run migrations for the ClickHouse database" + task migrate: :environment do + path = "db/migrate/clickhouse/*.rb" + migrations_table = :clickhouse_migrations + migrations_sources_table = :clickhouse_migrations_sources + use_transactions = "false" + + Rake::Task["sequel:archive_migrations"].reenable + Rake::Task["sequel:archive_migrations"].invoke(path, migrations_sources_table) + + Rake::Task["sequel:rollback_archived_migrations"].reenable + Rake::Task["sequel:rollback_archived_migrations"] + .invoke(path, migrations_table, migrations_sources_table, use_transactions) + + Clickhouse::Migrator.migrate(to: ENV.fetch("VERSION", nil)) + end + + desc "Rollback migrations for the ClickHouse database" + task rollback: :environment do + Clickhouse::Migrator.rollback(to: ENV.fetch("VERSION", nil)) + end + + desc "Reset the ClickHouse database: drop, recreate, and run all migrations" + task reset: :environment do + Rake::Task["ch:drop"].invoke + Rake::Task["ch:create"].invoke + Rake::Task["ch:migrate"].invoke + end + + desc "Rollback any missing migrations for ClickHouse" + task rollback_missing_migrations: :environment do + Rake::Task["sequel:rollback_missing_migrations"].reenable + Rake::Task["sequel:rollback_missing_migrations"].invoke(:clickhouse_migrations, "false") + end +end diff --git a/lib/tasks/sequel/archive_migrations.rake b/lib/tasks/sequel/archive_migrations.rake index e01ef9f..ae06611 100644 --- a/lib/tasks/sequel/archive_migrations.rake +++ b/lib/tasks/sequel/archive_migrations.rake @@ -2,14 +2,18 @@ namespace :sequel do desc "Archive migrations source code" - task archive_migrations: :environment do - DB.create_table?(:schema_migrations_sources) do + task :archive_migrations, + [:migrations_path, :migration_table_source] => :environment do |_t, args| + migrations_path = args[:migrations_path] || "db/migrate/*.rb" + migration_table_source = args[:migration_table_source]&.to_sym || :schema_migrations_sources + + DB.create_table?(migration_table_source) do column :version, "numeric", primary_key: true column :filename, "text", null: false column :source, "text", null: false end - migrations = Rails.root.glob("db/migrate/*.rb").map do |file| + migrations = Rails.root.glob(migrations_path).map do |file| filename = file.basename.to_s { version: filename.to_i, filename: filename, source: file.read } end @@ -19,6 +23,6 @@ namespace :sequel do update: { filename: Sequel[:excluded][:filename], source: Sequel[:excluded][:source] }, } - DB[:schema_migrations_sources].insert_conflict(**conflict_options).multi_insert(migrations) + DB[migration_table_source].insert_conflict(**conflict_options).multi_insert(migrations) end end diff --git a/lib/tasks/sequel/rollback_archived_migrations.rake b/lib/tasks/sequel/rollback_archived_migrations.rake index ee99c1a..edf37d2 100644 --- a/lib/tasks/sequel/rollback_archived_migrations.rake +++ b/lib/tasks/sequel/rollback_archived_migrations.rake @@ -4,19 +4,30 @@ require "sequel/timestamp_migrator_undo_extension" namespace :sequel do desc "Rollback migrations that were applied earlier but are not present in current release" - task rollback_archived_migrations: :environment do + task :rollback_archived_migrations, + [:migrations_path, :migration_table, :migration_table_source, + :use_transactions] => :environment do |_t, args| + migrations_path = args[:migrations_path] || "db/migrate/*.rb" + migration_table_source = args[:migration_table_source]&.to_sym || :schema_migrations_sources + use_transactions = args[:use_transactions].nil? ? nil : args[:use_transactions] == "true" + DB.log_info("Finding applied migrations not present in current release...") Dir.mktmpdir do |tmpdir| - DB[:schema_migrations_sources].each do |migration| + DB[migration_table_source].each do |migration| path = File.join(tmpdir, migration.fetch(:filename)) File.write(path, migration.fetch(:source)) end - migrator = Sequel::TimestampMigrator.new(DB, tmpdir, allow_missing_migration_files: true) + migrator_args = { + table: args[:migration_table], + use_transactions: use_transactions, + allow_missing_migration_files: false, + }.compact + migrator = Sequel::TimestampMigrator.new(DB, tmpdir, migrator_args) applied_migrations = migrator.applied_migrations.map(&:to_i) - filesystem_migrations = Rails.root.glob("db/migrate/*.rb").map { |x| File.basename(x).to_i } + filesystem_migrations = Rails.root.glob(migrations_path).map { |x| File.basename(x).to_i } missing_migrations = applied_migrations - filesystem_migrations if missing_migrations.any? diff --git a/lib/tasks/sequel/rollback_missing_migrations.rake b/lib/tasks/sequel/rollback_missing_migrations.rake index 52efded..39110a1 100644 --- a/lib/tasks/sequel/rollback_missing_migrations.rake +++ b/lib/tasks/sequel/rollback_missing_migrations.rake @@ -4,7 +4,9 @@ require "sequel/timestamp_migrator_undo_extension" namespace :sequel do desc "Rollback migrations that are absent in revision when deploying on staging" - task rollback_missing_migrations: :environment do + task :rollback_missing_migrations, [:table, :use_transactions] => :environment do |_t, args| + use_transactions = args[:use_transactions].nil? ? nil : args[:use_transactions] == "true" + extract_migrations = lambda do |path| Dir.glob("#{path}/db/migrate/*.rb").map { |filename| File.basename(filename).to_i } end @@ -19,7 +21,12 @@ namespace :sequel do puts migrations_to_rollback path = Rails.root.join("db/migrate") - migrator = Sequel::TimestampMigrator.new(DB, path, allow_missing_migration_files: true) + migrator_args = { + table: args[:table], + use_transactions: use_transactions, + allow_missing_migration_files: false, + }.compact + migrator = Sequel::TimestampMigrator.new(DB, path, migrator_args) applied_migrations = migrator.applied_migrations.map(&:to_i) migrations = applied_migrations.select { |m| m.in?(migrations_to_rollback) }.sort.reverse diff --git a/umbrellio-sequel-plugins.gemspec b/umbrellio-sequel-plugins.gemspec index 99d3832..27216e0 100644 --- a/umbrellio-sequel-plugins.gemspec +++ b/umbrellio-sequel-plugins.gemspec @@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) Gem::Specification.new do |spec| spec.name = "umbrellio-sequel-plugins" - spec.version = "0.16.1" + spec.version = "0.17.0" spec.required_ruby_version = ">= 3.0" spec.authors = ["Team Umbrellio"]