From ddb5f75fbb5164449ebba45efcfb6498244efdc6 Mon Sep 17 00:00:00 2001 From: Jochen Ehret Date: Wed, 17 Dec 2025 15:17:03 +0100 Subject: [PATCH 1/2] Fix shutdown of periodic_updater * current implementation produces "error: can't be called from trap" --- lib/cloud_controller/runners/puma_runner.rb | 19 ++++++++++++++++--- .../runners/puma_runner_spec.rb | 1 + 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/cloud_controller/runners/puma_runner.rb b/lib/cloud_controller/runners/puma_runner.rb index a9e38ad51c..7ef00acd6b 100644 --- a/lib/cloud_controller/runners/puma_runner.rb +++ b/lib/cloud_controller/runners/puma_runner.rb @@ -62,9 +62,7 @@ def initialize(config, app, logger, periodic_updater, request_logs) prometheus_updater.update_gauge_metric(:cc_db_connection_pool_timeouts_total, 0, labels: { process_type: 'main' }) @periodic_updater.setup_updates end - events.after_stopped do - @periodic_updater.stop_updates unless @periodic_updater.nil? - end + events.after_stopped(&method(:shutdown_periodic_updater)) @puma_launcher = Puma::Launcher.new(puma_config, log_writer:, events:) end @@ -78,6 +76,21 @@ def start! private + def shutdown_periodic_updater + if @periodic_updater + Thread.new do + # Give the trap handler a moment to return before taking any locks. + sleep 0.05 + @periodic_updater.stop_updates + rescue StandardError => e + @logger.error "Failed to stop periodic updates cleanly: #{e}\n#{e.backtrace&.join("\n")}" + end + end + rescue StandardError => e + # Never let exceptions from the trap context bubble up and crash shutdown. + @logger.error "Failed to schedule periodic updates shutdown: #{e}\n#{e.backtrace&.join("\n")}" + end + def prometheus_updater CloudController::DependencyLocator.instance.prometheus_updater end diff --git a/spec/unit/lib/cloud_controller/runners/puma_runner_spec.rb b/spec/unit/lib/cloud_controller/runners/puma_runner_spec.rb index 40199689cc..2ed36d6556 100644 --- a/spec/unit/lib/cloud_controller/runners/puma_runner_spec.rb +++ b/spec/unit/lib/cloud_controller/runners/puma_runner_spec.rb @@ -208,6 +208,7 @@ module VCAP::CloudController expect(periodic_updater).to receive(:stop_updates) puma_launcher.events.fire(:after_stopped) + sleep 0.1 end end end From dd61383b3fe5c760ae70f2c3528008e49168c90f Mon Sep 17 00:00:00 2001 From: Jochen Ehret Date: Fri, 19 Dec 2025 14:50:47 +0100 Subject: [PATCH 2/2] Use at_exit hook to shut down periodic_updater * avoids use of sleep commands --- lib/cloud_controller/runners/puma_runner.rb | 24 +++++++++---------- .../runners/puma_runner_spec.rb | 1 - 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/lib/cloud_controller/runners/puma_runner.rb b/lib/cloud_controller/runners/puma_runner.rb index 7ef00acd6b..bc436062ed 100644 --- a/lib/cloud_controller/runners/puma_runner.rb +++ b/lib/cloud_controller/runners/puma_runner.rb @@ -62,7 +62,10 @@ def initialize(config, app, logger, periodic_updater, request_logs) prometheus_updater.update_gauge_metric(:cc_db_connection_pool_timeouts_total, 0, labels: { process_type: 'main' }) @periodic_updater.setup_updates end - events.after_stopped(&method(:shutdown_periodic_updater)) + + events.after_stopped do + stop_periodic_updates + end @puma_launcher = Puma::Launcher.new(puma_config, log_writer:, events:) end @@ -76,19 +79,16 @@ def start! private - def shutdown_periodic_updater - if @periodic_updater - Thread.new do - # Give the trap handler a moment to return before taking any locks. - sleep 0.05 - @periodic_updater.stop_updates - rescue StandardError => e - @logger.error "Failed to stop periodic updates cleanly: #{e}\n#{e.backtrace&.join("\n")}" - end + def stop_periodic_updates + @periodic_updater&.stop_updates + @logger.info('Successfully stopped periodic updates in after_stopped') + rescue ThreadError + at_exit do + @periodic_updater&.stop_updates + @logger.info('Successfully stopped periodic updates in at_exit') end rescue StandardError => e - # Never let exceptions from the trap context bubble up and crash shutdown. - @logger.error "Failed to schedule periodic updates shutdown: #{e}\n#{e.backtrace&.join("\n")}" + @logger.error("Failed to stop periodic updates: #{e}\n#{e.backtrace&.join("\n")}") end def prometheus_updater diff --git a/spec/unit/lib/cloud_controller/runners/puma_runner_spec.rb b/spec/unit/lib/cloud_controller/runners/puma_runner_spec.rb index 2ed36d6556..40199689cc 100644 --- a/spec/unit/lib/cloud_controller/runners/puma_runner_spec.rb +++ b/spec/unit/lib/cloud_controller/runners/puma_runner_spec.rb @@ -208,7 +208,6 @@ module VCAP::CloudController expect(periodic_updater).to receive(:stop_updates) puma_launcher.events.fire(:after_stopped) - sleep 0.1 end end end