From 4e9cd3fd30203d55dc01274db76ef94c683157fd Mon Sep 17 00:00:00 2001 From: James Smith Date: Sat, 1 Jul 2023 23:48:48 +0100 Subject: [PATCH] rough first attempt at db:prepare:with_data Lots of code copied out of rails, needs DRYing up a lot and the commits refactoring. --- lib/data_migrate/database_tasks.rb | 120 ++++++++++++++++++++++++++--- tasks/databases.rake | 63 +++++---------- 2 files changed, 127 insertions(+), 56 deletions(-) diff --git a/lib/data_migrate/database_tasks.rb b/lib/data_migrate/database_tasks.rb index ee96a7d6..6dd0db72 100644 --- a/lib/data_migrate/database_tasks.rb +++ b/lib/data_migrate/database_tasks.rb @@ -29,7 +29,7 @@ def dump_filename(spec_name, format = ActiveRecord::Base.schema_format) def check_schema_file(filename) unless File.exist?(filename) - message = +%{#{filename} doesn't exist yet. Run `rake data:migrate` to create it, then try again.} + message = +%(#{filename} doesn't exist yet. Run `rake data:migrate` to create it, then try again.) Kernel.abort message end end @@ -42,11 +42,11 @@ def pending_migrations end def sort_migrations(*migrations) - migrations.flatten.sort { |a, b| sort_string(a) <=> sort_string(b) } + migrations.flatten.sort { |a, b| sort_string(a) <=> sort_string(b) } end def sort_string migration - "#{migration[:version]}_#{migration[:kind] == :data ? 1 : 0}" + "#{migration[:version]}_#{(migration[:kind] == :data) ? 1 : 0}" end def data_migrations_path @@ -55,10 +55,10 @@ def data_migrations_path def run_migration(migration, direction) if migration[:kind] == :data - ::ActiveRecord::Migration.write("== %s %s" % ['Data', "=" * 71]) + ::ActiveRecord::Migration.write("== %s %s" % ["Data", "=" * 71]) ::DataMigrate::DataMigrator.run(direction, data_migrations_path, migration[:version]) else - ::ActiveRecord::Migration.write("== %s %s" % ['Schema', "=" * 69]) + ::ActiveRecord::Migration.write("== %s %s" % ["Schema", "=" * 69]) ::DataMigrate::SchemaMigration.run( direction, ::DataMigrate::SchemaMigration.migrations_paths, @@ -80,14 +80,30 @@ def schema_dump_path(db_config, format = ActiveRecord.schema_format) # to ensure that the sha saved in ar_internal_metadata table # is from the original schema.rb file def schema_sha1(file) - super(file.gsub(/data_schema.rb\z/, 'schema.rb')) + super(file.gsub(/data_schema.rb\z/, "schema.rb")) + end + + def with_temporary_pool(db_config) + original_db_config = migration_class.connection_db_config + pool = migration_class.establish_connection(db_config) + yield pool + ensure + migration_class.establish_connection(original_db_config) + end + + def migration_class # :nodoc: + ActiveRecord::Base + end + + def migration_connection # :nodoc: + migration_class.connection end end def self.forward(step = 1) DataMigrate::DataMigrator.create_data_schema_table migrations = pending_migrations.reverse.pop(step).reverse - migrations.each do | pending_migration | + migrations.each do |pending_migration| if pending_migration[:kind] == :data ActiveRecord::Migration.write("== %s %s" % ["Data", "=" * 71]) DataMigrate::DataMigrator.run(:up, data_migrations_path, pending_migration[:version]) @@ -100,8 +116,8 @@ def self.forward(step = 1) def self.pending_data_migrations data_migrations = DataMigrate::DataMigrator.migrations(data_migrations_path) - sort_migrations(DataMigrate::DataMigrator.new(:up, data_migrations ). - pending_migrations.map {|m| { version: m.version, name: m.name, kind: :data }}) + sort_migrations(DataMigrate::DataMigrator.new(:up, data_migrations) + .pending_migrations.map { |m| {version: m.version, name: m.name, kind: :data} }) end def self.pending_schema_migrations @@ -111,9 +127,91 @@ def self.pending_schema_migrations def self.past_migrations(sort = nil) data_versions = DataMigrate::DataSchemaMigration.table_exists? ? DataMigrate::DataSchemaMigration.normalized_versions : [] schema_versions = ActiveRecord::SchemaMigration.normalized_versions - migrations = data_versions.map { |v| { version: v.to_i, kind: :data } } + schema_versions.map { |v| { version: v.to_i, kind: :schema } } + migrations = data_versions.map { |v| {version: v.to_i, kind: :data} } + schema_versions.map { |v| {version: v.to_i, kind: :schema} } + + (sort&.downcase == "asc") ? sort_migrations(migrations) : sort_migrations(migrations).reverse + end + + def self.migrate_with_data + DataMigrate::DataMigrator.create_data_schema_table + + ActiveRecord::Migration.verbose = ENV["VERBOSE"] ? ENV["VERBOSE"] == "true" : true + target_version = ENV["VERSION"] ? ENV["VERSION"].to_i : nil + migrations = [] + + if target_version.nil? + migrations = DataMigrate::DatabaseTasks.pending_migrations.map { |m| m.merge(direction: :up) } + else + current_schema_version = ActiveRecord::Migrator.current_version + schema_migrations = if target_version > current_schema_version + DataMigrate::DatabaseTasks.pending_schema_migrations.keep_if { |m| m[:version] <= target_version }.map { |m| m.merge(direction: :up) } + elsif target_version < current_schema_version + DataMigrate::DatabaseTasks.past_migrations.keep_if { |m| m[:version] > target_version }.map { |m| m.merge(direction: :down) } + else # == + [] + end + + current_data_version = DataMigrate::DataMigrator.current_version + data_migrations = if target_version > current_data_version + DataMigrate::DatabaseTasks.pending_data_migrations.keep_if { |m| m[:version] <= target_version }.map { |m| m.merge(direction: :up) } + elsif target_version < current_data_version + DataMigrate::DatabaseTasks.past_migrations.keep_if { |m| m[:version] > target_version }.map { |m| m.merge(direction: :down) } + else # == + [] + end + migrations = if schema_migrations.empty? + data_migrations + elsif data_migrations.empty? + schema_migrations + elsif target_version > current_data_version && target_version > current_schema_version + DataMigrate::DatabaseTasks.sort_migrations data_migrations, schema_migrations + elsif target_version < current_data_version && target_version < current_schema_version + DataMigrate::DatabaseTasks.sort_migrations(data_migrations, schema_migrations).reverse + elsif target_version > current_data_version && target_version < current_schema_version + schema_migrations + data_migrations + elsif target_version < current_data_version && target_version > current_schema_version + schema_migrations + data_migrations + end + end + + migrations.each do |migration| + DataMigrate::DatabaseTasks.run_migration(migration, migration[:direction]) + end + end + + def self.prepare_all_with_data + seed = false + + each_current_configuration(env) do |db_config| + with_temporary_pool(db_config) do + begin + database_initialized = migration_connection.schema_migration.table_exists? + rescue ActiveRecord::NoDatabaseError + create(db_config) + retry + end + + unless database_initialized + if File.exist?(schema_dump_path(db_config)) + load_schema(db_config, ActiveRecord.schema_format, nil) + load_schema_current( + :ruby, + ENV["DATA_SCHEMA"] + ) + end + + seed = true + end + + migrate_with_data + if ActiveRecord.dump_schema_after_migration + dump_schema(db_config) + DataMigrate::Tasks::DataMigrateTasks.dump + end + end + end - sort&.downcase == "asc" ? sort_migrations(migrations) : sort_migrations(migrations).reverse + load_seed if seed end end end diff --git a/tasks/databases.rake b/tasks/databases.rake index e02e7f7b..31f26302 100644 --- a/tasks/databases.rake +++ b/tasks/databases.rake @@ -2,55 +2,21 @@ require 'data_migrate/tasks/data_migrate_tasks' +# From https://gist.github.com/raggi/232966 +def alias_task(name, old_name) + t = Rake::Task[old_name] + desc t.full_comment if t.full_comment + task name, *t.arg_names do |_, args| + args = t.arg_names.map { |a| args[a] } + t.invoke(args) + end +end + namespace :db do namespace :migrate do desc "Migrate the database data and schema (options: VERSION=x, VERBOSE=false)." task :with_data => :environment do - DataMigrate::DataMigrator.create_data_schema_table - - ActiveRecord::Migration.verbose = ENV["VERBOSE"] ? ENV["VERBOSE"] == "true" : true - target_version = ENV["VERSION"] ? ENV["VERSION"].to_i : nil - migrations = [] - - if target_version.nil? - migrations = DataMigrate::DatabaseTasks.pending_migrations.map{ |m| m.merge(:direction =>:up) } - else - current_schema_version = ActiveRecord::Migrator.current_version - schema_migrations = if target_version > current_schema_version - DataMigrate::DatabaseTasks.pending_schema_migrations.keep_if{ |m| m[:version] <= target_version }.map{ |m| m.merge(:direction =>:up) } - elsif target_version < current_schema_version - DataMigrate::DatabaseTasks.past_migrations.keep_if{ |m| m[:version] > target_version }.map{ |m| m.merge(:direction =>:down) } - else # == - [] - end - - current_data_version = DataMigrate::DataMigrator.current_version - data_migrations = if target_version > current_data_version - DataMigrate::DatabaseTasks.pending_data_migrations.keep_if{ |m| m[:version] <= target_version }.map{ |m| m.merge(:direction =>:up) } - elsif target_version < current_data_version - DataMigrate::DatabaseTasks.past_migrations.keep_if{ |m| m[:version] > target_version }.map{ |m| m.merge(:direction =>:down) } - else # == - [] - end - migrations = if schema_migrations.empty? - data_migrations - elsif data_migrations.empty? - schema_migrations - elsif target_version > current_data_version && target_version > current_schema_version - DataMigrate::DatabaseTasks.sort_migrations data_migrations, schema_migrations - elsif target_version < current_data_version && target_version < current_schema_version - DataMigrate::DatabaseTasks.sort_migrations(data_migrations, schema_migrations).reverse - elsif target_version > current_data_version && target_version < current_schema_version - schema_migrations + data_migrations - elsif target_version < current_data_version && target_version > current_schema_version - schema_migrations + data_migrations - end - end - - migrations.each do |migration| - DataMigrate::DatabaseTasks.run_migration(migration, migration[:direction]) - end - + DataMigrate::DatabaseTasks.migrate_with_data Rake::Task["db:_dump"].invoke Rake::Task["data:dump"].invoke end @@ -190,6 +156,13 @@ namespace :db do end end end + + namespace :prepare do + desc "Runs setup if database does not exist, or runs data and schema migrations if it does" + task with_data: :environment do + DataMigrate::DatabaseTasks.prepare_all_with_data + end + end end namespace :data do