diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a2c9105..08687d93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fixed the `aws_postgresql` ActiveRecord adapter passing ActiveRecord-only `database.yml` keys, such as the Rails 8.1 pool settings (`max_connections`, `min_connections`, `keepalive`, `max_age`) and multi-database settings (`replica`, `migrations_paths`, `database_tasks`), to `pg`, which rejected them with `PG::Error: invalid connection option` ([PR #191](https://github.com/aws/aws-advanced-ruby-driver-wrapper/pull/191)). - Fixed the `aws_mysql2` ActiveRecord adapter dropping the `flags`, `encoding`, `socket`, and `reconnect` client options, which removed ActiveRecord's `FOUND_ROWS` flag so that statements such as `update_all` reported changed rows instead of matched rows ([PR #193](https://github.com/aws/aws-advanced-ruby-driver-wrapper/pull/193)). - Fixed the `aws_postgresql` and `aws_mysql2` ActiveRecord adapters not translating connection errors the way the standard adapters do, so a missing database was not reported as `ActiveRecord::NoDatabaseError` and `db:prepare` (and `bin/setup`) failed instead of creating it ([PR #193](https://github.com/aws/aws-advanced-ruby-driver-wrapper/pull/193)). +- Fixed the `aws_postgresql` and `aws_mysql2` ActiveRecord adapters keeping ActiveRecord's prepared statement cache after a successful failover, so every query prepared before the failover failed on that connection with `Method invoked against old connection` until the process restarted ([PR #194](https://github.com/aws/aws-advanced-ruby-driver-wrapper/pull/194), [documentation](https://aws.github.io/aws-advanced-wrapper-docs/ruby/enhanced-failover)). ## [1.0.0] - 2026-10-05 diff --git a/lib/aws_advanced_ruby_driver_wrapper/active_record/aws_mysql2_adapter.rb b/lib/aws_advanced_ruby_driver_wrapper/active_record/aws_mysql2_adapter.rb index bd5c444a..5ec71b8e 100644 --- a/lib/aws_advanced_ruby_driver_wrapper/active_record/aws_mysql2_adapter.rb +++ b/lib/aws_advanced_ruby_driver_wrapper/active_record/aws_mysql2_adapter.rb @@ -66,6 +66,9 @@ def translate_exception(exception, message:, sql:, binds:) return super unless exception.is_a?(AwsAdvancedRubyDriverWrapper::Errors::AwsError) if exception.needs_reconfiguration + # The wrapper has reconnected this connection object to a new physical connection, where the + # statements prepared on the old one do not exist. Forget them so they are prepared again. + clear_cache!(new_connection: true) configure_connection exception elsif exception.is_a?(AwsAdvancedRubyDriverWrapper::Errors::FailoverFailedError) diff --git a/lib/aws_advanced_ruby_driver_wrapper/active_record/aws_postgresql_adapter.rb b/lib/aws_advanced_ruby_driver_wrapper/active_record/aws_postgresql_adapter.rb index c596854d..9687f080 100644 --- a/lib/aws_advanced_ruby_driver_wrapper/active_record/aws_postgresql_adapter.rb +++ b/lib/aws_advanced_ruby_driver_wrapper/active_record/aws_postgresql_adapter.rb @@ -94,6 +94,9 @@ def translate_exception(exception, message:, sql:, binds:) return super unless exception.is_a?(AwsAdvancedRubyDriverWrapper::Errors::AwsError) if exception.needs_reconfiguration + # The wrapper has reconnected this connection object to a new physical connection, where the + # statements prepared on the old one do not exist. Forget them so they are prepared again. + clear_cache!(new_connection: true) configure_connection exception elsif exception.is_a?(AwsAdvancedRubyDriverWrapper::Errors::FailoverFailedError) diff --git a/spec/integration/container/failover_activerecord_spec.rb b/spec/integration/container/failover_activerecord_spec.rb index b9ffdc14..f61daa60 100644 --- a/spec/integration/container/failover_activerecord_spec.rb +++ b/spec/integration/container/failover_activerecord_spec.rb @@ -49,7 +49,8 @@ # AR connection config pointing at the cluster/instance endpoints through the proxy, with the # failover plugin enabled. Wrapper property keys pass through the adapter into the wrapper unchanged. # An optional :variables hash is applied by AR's configure_connection via SET SESSION statements. - def failover_adapter_config(host:, port:, variables: nil) + # :prepared_statements overrides the adapter default (on for PostgreSQL, off for MySQL). + def failover_adapter_config(host:, port:, variables: nil, prepared_statements: nil) adapter = case drv when Integration::TestDriver::PG then 'aws_postgresql' when Integration::TestDriver::MYSQL then 'aws_mysql2' @@ -70,6 +71,7 @@ def failover_adapter_config(host:, port:, variables: nil) "?.#{proxy_info.instance_endpoint_suffix}:#{proxy_info.instance_endpoint_port}" } config[:variables] = variables if variables + config[:prepared_statements] = prepared_statements unless prepared_statements.nil? config end @@ -91,10 +93,10 @@ def session_probe end end - def establish_failover_connection(host:, port:, variables: nil) + def establish_failover_connection(host:, port:, variables: nil, prepared_statements: nil) ActiveRecord::Base.connection_handler.clear_all_connections! ActiveRecord::Base.establish_connection( - failover_adapter_config(host: host, port: port, variables: variables) + failover_adapter_config(host: host, port: port, variables: variables, prepared_statements: prepared_statements) ) end @@ -189,6 +191,34 @@ def instance_id_via(conn) expect(adapter.select_value(probe[:read_sql])).to eq(probe[:expected]) end + it 'runs queries ActiveRecord prepared before failover again on the new writer', + features: [Integration::TestEnvironmentFeatures::NETWORK_OUTAGES_ENABLED] do + enable_on_num_instances(min_instances: 2) + + establish_failover_connection( + host: proxy_info.cluster_endpoint, port: proxy_info.cluster_endpoint_port, prepared_statements: true + ) + conn = ActiveRecord::Base.connection + conn.execute('DROP TABLE IF EXISTS ar_test_failover_prepared') + conn.execute('CREATE TABLE ar_test_failover_prepared (id int not null primary key, val varchar(255) not null)') + conn.execute("INSERT INTO ar_test_failover_prepared VALUES (1, 'value1')") + model = Class.new(ActiveRecord::Base) { self.table_name = 'ar_test_failover_prepared' } + expect(conn.prepared_statements?).to be true + + # A query with binds, which ActiveRecord prepares once and then reuses by statement name. + expect(model.find(1).val).to eq('value1') + + rds_util.crash_instance(current_writer) + expect { model.find(1) }.to raise_error(AwsAdvancedRubyDriverWrapper::Errors::FailoverSuccessError) + + # The statement prepared before failover existed only on the old physical connection. The same + # query must be prepared again on the new writer rather than failing on every later attempt. + 3.times { expect(model.find(1).val).to eq('value1') } + expect(Integration::RetryHelper.verify_writer(rds_util, current_instance_id)).to be true + ensure + ActiveRecord::Base.connection.execute('DROP TABLE IF EXISTS ar_test_failover_prepared') + end + it 'raises TransactionStateUnknownError and loses the open transaction when failover happens mid-transaction', features: [Integration::TestEnvironmentFeatures::NETWORK_OUTAGES_ENABLED] do enable_on_num_instances(min_instances: 2) diff --git a/spec/unit/aws_mysql2_adapter_spec.rb b/spec/unit/aws_mysql2_adapter_spec.rb index 12d07cb4..a04390ec 100644 --- a/spec/unit/aws_mysql2_adapter_spec.rb +++ b/spec/unit/aws_mysql2_adapter_spec.rb @@ -121,6 +121,24 @@ end end + # The wrapper reconnects the same connection object to a new physical connection, so statements + # ActiveRecord prepared on the old one no longer exist and must be forgotten before reuse. + reconfiguring_errors = [AwsAdvancedRubyDriverWrapper::Errors::FailoverSuccessError, + AwsAdvancedRubyDriverWrapper::Errors::TransactionStateUnknownError] + reconfiguring_errors.each do |error_class| + context "when exception is a #{error_class.name.split('::').last} and statements were prepared" do + it 'forgets the prepared statements before reconfiguring the connection' do + calls = [] + allow(adapter).to receive(:clear_cache!) { |**kwargs| calls << [:clear_cache!, kwargs] } + allow(adapter).to receive(:configure_connection) { calls << [:configure_connection] } + + adapter.translate_exception(error_class.new, message: message, sql: sql, binds: binds) + + expect(calls).to eq([[:clear_cache!, { new_connection: true }], [:configure_connection]]) + end + end + end + context 'when exception is a FailoverFailedError' do it 'returns a connection error and sets connection_broken' do exception = AwsAdvancedRubyDriverWrapper::Errors::FailoverFailedError.new('') diff --git a/spec/unit/aws_postgresql_adapter_spec.rb b/spec/unit/aws_postgresql_adapter_spec.rb index d1597a62..53815cdf 100644 --- a/spec/unit/aws_postgresql_adapter_spec.rb +++ b/spec/unit/aws_postgresql_adapter_spec.rb @@ -151,6 +151,24 @@ def connect_failing_with(message, client_config = config) end end + # The wrapper reconnects the same connection object to a new physical connection, so statements + # ActiveRecord prepared on the old one no longer exist and must be forgotten before reuse. + reconfiguring_errors = [AwsAdvancedRubyDriverWrapper::Errors::FailoverSuccessError, + AwsAdvancedRubyDriverWrapper::Errors::TransactionStateUnknownError] + reconfiguring_errors.each do |error_class| + context "when exception is a #{error_class.name.split('::').last} and statements were prepared" do + it 'forgets the prepared statements before reconfiguring the connection' do + calls = [] + allow(adapter).to receive(:clear_cache!) { |**kwargs| calls << [:clear_cache!, kwargs] } + allow(adapter).to receive(:configure_connection) { calls << [:configure_connection] } + + adapter.translate_exception(error_class.new, message: message, sql: sql, binds: binds) + + expect(calls).to eq([[:clear_cache!, { new_connection: true }], [:configure_connection]]) + end + end + end + context 'when exception is a FailoverFailedError' do it 'returns a connection error and sets connection_broken' do exception = AwsAdvancedRubyDriverWrapper::Errors::FailoverFailedError.new('')