diff --git a/db/migrations/20250930135451_bigint_migration_delayed_jobs_step3a.rb b/db/migrations/20250930135451_bigint_migration_delayed_jobs_step3a.rb new file mode 100644 index 00000000000..874edf78734 --- /dev/null +++ b/db/migrations/20250930135451_bigint_migration_delayed_jobs_step3a.rb @@ -0,0 +1,6 @@ +require 'database/bigint_migration_step3ab' + +Sequel.migration do + no_transaction + VCAP::BigintMigration.step3a_migration(self, :delayed_jobs) +end diff --git a/db/migrations/20250930135459_bigint_migration_delayed_jobs_step3b.rb b/db/migrations/20250930135459_bigint_migration_delayed_jobs_step3b.rb new file mode 100644 index 00000000000..4253ca0d347 --- /dev/null +++ b/db/migrations/20250930135459_bigint_migration_delayed_jobs_step3b.rb @@ -0,0 +1,6 @@ +require 'database/bigint_migration_step3ab' + +Sequel.migration do + no_transaction + VCAP::BigintMigration.step3b_migration(self, :delayed_jobs) +end diff --git a/db/migrations/20250930135517_bigint_migration_jobs_step3a.rb b/db/migrations/20250930135517_bigint_migration_jobs_step3a.rb new file mode 100644 index 00000000000..1d126d2c439 --- /dev/null +++ b/db/migrations/20250930135517_bigint_migration_jobs_step3a.rb @@ -0,0 +1,6 @@ +require 'database/bigint_migration_step3ab' + +Sequel.migration do + no_transaction + VCAP::BigintMigration.step3a_migration(self, :jobs) +end diff --git a/db/migrations/20250930135527_bigint_migration_jobs_step3b.rb b/db/migrations/20250930135527_bigint_migration_jobs_step3b.rb new file mode 100644 index 00000000000..1104090d7b6 --- /dev/null +++ b/db/migrations/20250930135527_bigint_migration_jobs_step3b.rb @@ -0,0 +1,6 @@ +require 'database/bigint_migration_step3ab' + +Sequel.migration do + no_transaction + VCAP::BigintMigration.step3b_migration(self, :jobs) +end diff --git a/db/migrations/20250930135548_bigint_migration_app_usage_events_step3a.rb b/db/migrations/20250930135548_bigint_migration_app_usage_events_step3a.rb new file mode 100644 index 00000000000..3df2beee37f --- /dev/null +++ b/db/migrations/20250930135548_bigint_migration_app_usage_events_step3a.rb @@ -0,0 +1,6 @@ +require 'database/bigint_migration_step3ab' + +Sequel.migration do + no_transaction + VCAP::BigintMigration.step3a_migration(self, :app_usage_events) +end diff --git a/db/migrations/20250930135554_bigint_migration_app_usage_events_step3b.rb b/db/migrations/20250930135554_bigint_migration_app_usage_events_step3b.rb new file mode 100644 index 00000000000..85c07e862c1 --- /dev/null +++ b/db/migrations/20250930135554_bigint_migration_app_usage_events_step3b.rb @@ -0,0 +1,6 @@ +require 'database/bigint_migration_step3ab' + +Sequel.migration do + no_transaction + VCAP::BigintMigration.step3b_migration(self, :app_usage_events) +end diff --git a/db/migrations/20250930135612_bigint_migration_service_usage_events_step3a.rb b/db/migrations/20250930135612_bigint_migration_service_usage_events_step3a.rb new file mode 100644 index 00000000000..218fc70553b --- /dev/null +++ b/db/migrations/20250930135612_bigint_migration_service_usage_events_step3a.rb @@ -0,0 +1,6 @@ +require 'database/bigint_migration_step3ab' + +Sequel.migration do + no_transaction + VCAP::BigintMigration.step3a_migration(self, :service_usage_events) +end diff --git a/db/migrations/20250930135619_bigint_migration_service_usage_events_step3b.rb b/db/migrations/20250930135619_bigint_migration_service_usage_events_step3b.rb new file mode 100644 index 00000000000..d91001904d7 --- /dev/null +++ b/db/migrations/20250930135619_bigint_migration_service_usage_events_step3b.rb @@ -0,0 +1,6 @@ +require 'database/bigint_migration_step3ab' + +Sequel.migration do + no_transaction + VCAP::BigintMigration.step3b_migration(self, :service_usage_events) +end diff --git a/lib/database/bigint_migration_step3ab.rb b/lib/database/bigint_migration_step3ab.rb new file mode 100644 index 00000000000..50ad7c8adb2 --- /dev/null +++ b/lib/database/bigint_migration_step3ab.rb @@ -0,0 +1,78 @@ +require 'database/bigint_migration' + +module VCAP::BigintMigration + class << self + def step3a_migration(migration, table) + migration.up do + if database_type == :postgres && + !VCAP::BigintMigration.migration_completed?(self, table) && + !VCAP::BigintMigration.migration_skipped?(self, table) + transaction { VCAP::BigintMigration.add_check_constraint(self, table) } + begin + VCAP::Migration.with_concurrent_timeout(self) do + VCAP::BigintMigration.validate_check_constraint(self, table) + end + rescue Sequel::CheckConstraintViolation + raise "Failed to add check constraint on '#{table}' table!\n" \ + "There are rows where 'id_bigint' does not match 'id', " \ + "thus step 3 of the bigint migration cannot be executed.\n" \ + "Consider running rake task 'db:bigint_backfill[#{table}]'." + end + end + end + + migration.down do + transaction { VCAP::BigintMigration.drop_check_constraint(self, table) if database_type == :postgres } + end + end + + def step3b_migration(migration, table) + migration.up do + if database_type == :postgres && VCAP::BigintMigration.has_check_constraint?(self, table) + transaction do + # Drop check constraint and trigger function + VCAP::BigintMigration.drop_check_constraint(self, table) + VCAP::BigintMigration.drop_trigger_function(self, table) + + # Drop old id column + VCAP::BigintMigration.drop_pk_column(self, table) + + # Switch id_bigint -> id + VCAP::BigintMigration.rename_bigint_column(self, table) + VCAP::BigintMigration.add_pk_constraint(self, table) + VCAP::BigintMigration.set_pk_as_identity_with_correct_start_value(self, table) + end + end + end + + migration.down do + if database_type == :postgres && VCAP::BigintMigration.migration_completed?(self, table) + transaction do + # Revert id -> id_bigint + VCAP::BigintMigration.drop_identity(self, table) + VCAP::BigintMigration.drop_timestamp_pk_index(self, table) + VCAP::BigintMigration.drop_pk_constraint(self, table) + VCAP::BigintMigration.revert_bigint_column_name(self, table) + + # Restore old id column + VCAP::BigintMigration.add_id_column(self, table) + + # To restore the previous state it is necessary to backfill the id column. In case there is a lot of data in the + # table this might be problematic, e.g. take a longer time. + # + # Ideally this down migration SHOULD NEVER BE EXECUTED IN A PRODUCTION SYSTEM! (It's there for proper integration + # testing of the bigint migration steps.) + VCAP::BigintMigration.backfill_id(self, table) + + VCAP::BigintMigration.add_pk_constraint(self, table) + VCAP::BigintMigration.set_pk_as_identity_with_correct_start_value(self, table) + + # Recreate trigger function and check constraint + VCAP::BigintMigration.create_trigger_function(self, table) + VCAP::BigintMigration.add_check_constraint(self, table) + end + end + end + end + end +end diff --git a/spec/migrations/20250729143100_bigint_migration_service_usage_events_step3a_spec.rb b/spec/migrations/20250729143100_bigint_migration_service_usage_events_step3a_spec.rb new file mode 100644 index 00000000000..a47236a26e8 --- /dev/null +++ b/spec/migrations/20250729143100_bigint_migration_service_usage_events_step3a_spec.rb @@ -0,0 +1,19 @@ +require 'spec_helper' +require 'migrations/helpers/bigint_migration_step3_shared_context' + +RSpec.describe 'bigint migration - service_usage_events table - step3a', isolation: :truncation, type: :migration do + include_context 'bigint migration step3a' do + let(:migration_filename_step1) { '20250729143100_bigint_migration_service_usage_events_step1.rb' } + let(:migration_filename_step3a) { '20250930135612_bigint_migration_service_usage_events_step3a.rb' } + let(:table) { :service_usage_events } + let(:insert) do + lambda do |db| + db[:service_usage_events].insert(guid: SecureRandom.uuid, created_at: Time.now.utc, + state: 'teststate', org_guid: SecureRandom.uuid, + space_guid: SecureRandom.uuid, space_name: 'testspace', + service_instance_guid: SecureRandom.uuid, service_instance_name: 'testinstance', + service_instance_type: 'testtype') + end + end + end +end diff --git a/spec/migrations/20250930135451_bigint_migration_delayed_jobs_step3a_spec.rb b/spec/migrations/20250930135451_bigint_migration_delayed_jobs_step3a_spec.rb new file mode 100644 index 00000000000..89134c4393c --- /dev/null +++ b/spec/migrations/20250930135451_bigint_migration_delayed_jobs_step3a_spec.rb @@ -0,0 +1,15 @@ +require 'spec_helper' +require 'migrations/helpers/bigint_migration_step3_shared_context' + +RSpec.describe 'bigint migration - delayed jobs table - step3a', isolation: :truncation, type: :migration do + include_context 'bigint migration step3a' do + let(:migration_filename_step1) { '20250729142800_bigint_migration_delayed_jobs_step1.rb' } + let(:migration_filename_step3a) { '20250930135451_bigint_migration_delayed_jobs_step3a_spec.rb' } + let(:table) { :delayed_jobs } + let(:insert) do + lambda do |db| + db[:delayed_jobs].insert(guid: SecureRandom.uuid, created_at: Time.now.utc, updated_at: Time.now.utc) + end + end + end +end diff --git a/spec/migrations/20250930135459_bigint_migration_delayed_jobs_step3b_spec.rb b/spec/migrations/20250930135459_bigint_migration_delayed_jobs_step3b_spec.rb new file mode 100644 index 00000000000..0e78cd3c929 --- /dev/null +++ b/spec/migrations/20250930135459_bigint_migration_delayed_jobs_step3b_spec.rb @@ -0,0 +1,16 @@ +require 'spec_helper' +require 'migrations/helpers/bigint_migration_step3_shared_context' + +RSpec.describe 'bigint migration - delayed jobs table - step3b', isolation: :truncation, type: :migration do + include_context 'bigint migration step3b' do + let(:migration_filename_step1) { '20250729142800_bigint_migration_delayed_jobs_step1.rb' } + let(:migration_filename_step3a) { '20250930135451_bigint_migration_delayed_jobs_step3a.rb' } + let(:migration_filename_step3b) { '20250930135459_bigint_migration_delayed_jobs_step3b.rb' } + let(:table) { :delayed_jobs } + let(:insert) do + lambda do |db| + db[:delayed_jobs].insert(guid: SecureRandom.uuid, created_at: Time.now.utc, updated_at: Time.now.utc) + end + end + end +end diff --git a/spec/migrations/20250930135517_bigint_migration_jobs_step3a_spec.rb b/spec/migrations/20250930135517_bigint_migration_jobs_step3a_spec.rb new file mode 100644 index 00000000000..7b5cc6f0ab1 --- /dev/null +++ b/spec/migrations/20250930135517_bigint_migration_jobs_step3a_spec.rb @@ -0,0 +1,15 @@ +require 'spec_helper' +require 'migrations/helpers/bigint_migration_step3_shared_context' + +RSpec.describe 'bigint migration - jobs table - step3a', isolation: :truncation, type: :migration do + include_context 'bigint migration step3a' do + let(:migration_filename_step1) { '20250729142900_bigint_migration_jobs_step1.rb' } + let(:migration_filename_step3a) { '20250930135517_bigint_migration_jobs_step3a.rb' } + let(:table) { :jobs } + let(:insert) do + lambda do |db| + db[:jobs].insert(guid: SecureRandom.uuid, created_at: Time.now.utc, updated_at: Time.now.utc) + end + end + end +end diff --git a/spec/migrations/20250930135527_bigint_migration_jobs_step3b_spec.rb b/spec/migrations/20250930135527_bigint_migration_jobs_step3b_spec.rb new file mode 100644 index 00000000000..8d34d120865 --- /dev/null +++ b/spec/migrations/20250930135527_bigint_migration_jobs_step3b_spec.rb @@ -0,0 +1,16 @@ +require 'spec_helper' +require 'migrations/helpers/bigint_migration_step3_shared_context' + +RSpec.describe 'bigint migration - jobs table - step3b', isolation: :truncation, type: :migration do + include_context 'bigint migration step3b' do + let(:migration_filename_step1) { '20250729142900_bigint_migration_jobs_step1.rb' } + let(:migration_filename_step3a) { '20250930135517_bigint_migration_jobs_step3a.rb' } + let(:migration_filename_step3b) { '20250930135527_bigint_migration_jobs_step3b.rb' } + let(:table) { :jobs } + let(:insert) do + lambda do |db| + db[:jobs].insert(guid: SecureRandom.uuid, created_at: Time.now.utc, updated_at: Time.now.utc) + end + end + end +end diff --git a/spec/migrations/20250930135548_bigint_migration_app_usage_events_step3a_spec.rb b/spec/migrations/20250930135548_bigint_migration_app_usage_events_step3a_spec.rb new file mode 100644 index 00000000000..03bd22b4f0f --- /dev/null +++ b/spec/migrations/20250930135548_bigint_migration_app_usage_events_step3a_spec.rb @@ -0,0 +1,19 @@ +require 'spec_helper' +require 'migrations/helpers/bigint_migration_step3_shared_context' + +RSpec.describe 'bigint migration - app_usage_events table - step3a', isolation: :truncation, type: :migration do + include_context 'bigint migration step3a' do + let(:migration_filename_step1) { '20250729143000_bigint_migration_app_usage_events_step1.rb' } + let(:migration_filename_step3a) { '20250930135548_bigint_migration_app_usage_events_step3a.rb' } + let(:table) { :app_usage_events } + let(:insert) do + lambda do |db| + db[:app_usage_events].insert(guid: SecureRandom.uuid, created_at: Time.now.utc, instance_count: 1, + memory_in_mb_per_instance: 512, state: 'teststate', + app_guid: SecureRandom.uuid, app_name: 'testapp', + space_guid: SecureRandom.uuid, space_name: 'testspace', + org_guid: SecureRandom.uuid) + end + end + end +end diff --git a/spec/migrations/20250930135554_bigint_migration_app_usage_events_step3b_spec.rb b/spec/migrations/20250930135554_bigint_migration_app_usage_events_step3b_spec.rb new file mode 100644 index 00000000000..cea59d787de --- /dev/null +++ b/spec/migrations/20250930135554_bigint_migration_app_usage_events_step3b_spec.rb @@ -0,0 +1,20 @@ +require 'spec_helper' +require 'migrations/helpers/bigint_migration_step3_shared_context' + +RSpec.describe 'bigint migration - app_usage_events table - step3b', isolation: :truncation, type: :migration do + include_context 'bigint migration step3b' do + let(:migration_filename_step1) { '20250729143000_bigint_migration_app_usage_events_step1.rb' } + let(:migration_filename_step3a) { '20250930135548_bigint_migration_app_usage_events_step3a.rb' } + let(:migration_filename_step3b) { '20250930135554_bigint_migration_app_usage_events_step3b.rb' } + let(:table) { :app_usage_events } + let(:insert) do + lambda do |db| + db[:app_usage_events].insert(guid: SecureRandom.uuid, created_at: Time.now.utc, instance_count: 1, + memory_in_mb_per_instance: 512, state: 'teststate', + app_guid: SecureRandom.uuid, app_name: 'testapp', + space_guid: SecureRandom.uuid, space_name: 'testspace', + org_guid: SecureRandom.uuid) + end + end + end +end diff --git a/spec/migrations/20250930135619_bigint_migration_service_usage_events_step3b_spec.rb b/spec/migrations/20250930135619_bigint_migration_service_usage_events_step3b_spec.rb new file mode 100644 index 00000000000..3cead228538 --- /dev/null +++ b/spec/migrations/20250930135619_bigint_migration_service_usage_events_step3b_spec.rb @@ -0,0 +1,20 @@ +require 'spec_helper' +require 'migrations/helpers/bigint_migration_step3_shared_context' + +RSpec.describe 'bigint migration - service_usage_events table - step3b', isolation: :truncation, type: :migration do + include_context 'bigint migration step3b' do + let(:migration_filename_step1) { '20250729143100_bigint_migration_service_usage_events_step1.rb' } + let(:migration_filename_step3a) { '20250930135612_bigint_migration_service_usage_events_step3a.rb' } + let(:migration_filename_step3b) { '20250930135619_bigint_migration_service_usage_events_step3b.rb' } + let(:table) { :service_usage_events } + let(:insert) do + lambda do |db| + db[:service_usage_events].insert(guid: SecureRandom.uuid, created_at: Time.now.utc, + state: 'teststate', org_guid: SecureRandom.uuid, + space_guid: SecureRandom.uuid, space_name: 'testspace', + service_instance_guid: SecureRandom.uuid, service_instance_name: 'testinstance', + service_instance_type: 'testtype') + end + end + end +end diff --git a/spec/migrations/helpers/bigint_migration_step3_shared_context.rb b/spec/migrations/helpers/bigint_migration_step3_shared_context.rb index 8bd06c2db16..064e3a5292a 100644 --- a/spec/migrations/helpers/bigint_migration_step3_shared_context.rb +++ b/spec/migrations/helpers/bigint_migration_step3_shared_context.rb @@ -51,7 +51,7 @@ it 'fails ...' do expect do run_migration_step3a - end.to raise_error(/Failed to add check constraint on 'events' table!/) + end.to raise_error(/Failed to add check constraint on '#{table}' table!/) end end end @@ -178,11 +178,14 @@ end it 'has an index on timestamp + (bigint) id column' do - expect(db).to have_table_with_index_on_columns(table, %i[timestamp id]) + if db.schema(table).any? { |col| col[0] == :timestamp } - run_migration_step3b + expect(db).to have_table_with_index_on_columns(table, %i[timestamp id]) + + run_migration_step3b - expect(db).to have_table_with_index_on_columns(table, %i[timestamp id]) + expect(db).to have_table_with_index_on_columns(table, %i[timestamp id]) + end end it 'uses an identity with correct start value for the (bigint) id column' do @@ -218,12 +221,15 @@ end it 'has an index on timestamp + (integer) id column' do - expect(db).to have_table_with_index_on_columns(table, %i[timestamp id]) + if db.schema(table).any? { |col| col[0] == :timestamp } - run_rollback_step3b + expect(db).to have_table_with_index_on_columns(table, %i[timestamp id]) + + run_rollback_step3b - expect(db).to have_table_with_index_on_columns(table, %i[timestamp id]) - expect(db).not_to have_table_with_index_on_columns(table, %i[timestamp id_bigint]) + expect(db).to have_table_with_index_on_columns(table, %i[timestamp id]) + expect(db).not_to have_table_with_index_on_columns(table, %i[timestamp id_bigint]) + end end it 'uses the (integer) id column as primary key' do