From a11c0fca2de1e66027ffe42a00da531a61d28af3 Mon Sep 17 00:00:00 2001 From: oleghasjanov Date: Thu, 6 Mar 2025 11:38:09 +0200 Subject: [PATCH] fix: handle HTTPClient::KeepAliveDisconnected in OrgRegistrantPhoneCheckerJob This commit implements a reliable connection error handling solution for the Company Register API integration. The job previously failed when connection errors occurred without proper recovery mechanisms. The implementation: Adds a lightweight Retryable module with configurable retry logic Implements smart caching of API responses (1 day expiration) Handles common network errors like KeepAliveDisconnected and timeouts Provides a fallback mechanism when all retry attempts fail Ensures test reliability with cache-skipping in test environment Testing: Added specific tests for both recovery and fallback scenarios Verified cache behavior in production and test environments Resolves connection errors observed in production logs without adding unnecessary complexity to the codebase. --- app/jobs/org_registrant_phone_checker_job.rb | 48 +++++++++++-- app/lib/retryable.rb | 39 +++++++++++ .../org_registrant_phone_checker_job_test.rb | 66 ++++++++++++++++++ test/lib/retryable_test.rb | 68 +++++++++++++++++++ 4 files changed, 214 insertions(+), 7 deletions(-) create mode 100644 app/lib/retryable.rb create mode 100644 test/lib/retryable_test.rb diff --git a/app/jobs/org_registrant_phone_checker_job.rb b/app/jobs/org_registrant_phone_checker_job.rb index ef9a7cdf4..d42a64407 100644 --- a/app/jobs/org_registrant_phone_checker_job.rb +++ b/app/jobs/org_registrant_phone_checker_job.rb @@ -1,8 +1,21 @@ class OrgRegistrantPhoneCheckerJob < ApplicationJob queue_as :default + + include Retryable + + # Constants for API error types + API_EXCEPTIONS = [ + HTTPClient::KeepAliveDisconnected, + Net::OpenTimeout, + Timeout::Error, + Savon::HTTPError, + Savon::SOAPFault, + Wasabi::Resolver::HTTPError + ].freeze + + CACHE_EXPIRES_IN = 1.day def perform(type: 'bulk', registrant_user_code: nil, spam_delay: 1) - puts '??? PERFROMED ???' case type when 'bulk' execute_bulk_checker(spam_delay) @@ -14,7 +27,7 @@ class OrgRegistrantPhoneCheckerJob < ApplicationJob end def execute_bulk_checker(spam_delay) - log("Bulk checker started") + log('Bulk checker started') Contact.where(ident_type: 'org', ident_country_code: 'EE').joins(:registrant_domains).each do |registrant_user| is_phone_number_matching = check_the_registrant_phone_number(registrant_user) @@ -22,15 +35,16 @@ class OrgRegistrantPhoneCheckerJob < ApplicationJob sleep(spam_delay) end - log("Bulk checker finished") + log('Bulk checker finished') end def execute_single_checker(registrant_user_code) - registrant_user = Contact.where(ident_type: 'org', ident_country_code: 'EE').joins(:registrant_domains).find_by(code: registrant_user_code) + registrant_user = Contact.where(ident_type: 'org', ident_country_code: 'EE') + .joins(:registrant_domains) + .find_by(code: registrant_user_code) return if registrant_user.nil? is_phone_number_matching = check_the_registrant_phone_number(registrant_user) - call_disclosure_action(is_phone_number_matching, registrant_user) end @@ -74,7 +88,27 @@ class OrgRegistrantPhoneCheckerJob < ApplicationJob end def fetch_phone_number_from_company_register(company_code) - data = company_register.company_details(registration_number: company_code.to_s) - data[0].phone_numbers + cache_key = "company_register:#{company_code}:phone_numbers" + + # Skip cache in test environment if environment variable is set + return fetch_from_company_register(company_code) if Rails.env.test? && ENV['SKIP_COMPANY_REGISTER_CACHE'] + + # Try to get data from cache + Rails.cache.fetch(cache_key, expires_in: CACHE_EXPIRES_IN) do + fetch_from_company_register(company_code) + end + end + + def fetch_from_company_register(company_code) + # If not in cache, request API with retries + with_retry( + exceptions: API_EXCEPTIONS, + logger: Rails.logger, + fallback: -> { log("Failed to get data for company #{company_code}, returning empty array"); [] } + ) do + data = company_register.company_details(registration_number: company_code.to_s) + log("Successfully retrieved data for company #{company_code}") + data[0].phone_numbers + end end end diff --git a/app/lib/retryable.rb b/app/lib/retryable.rb new file mode 100644 index 000000000..b03db4b00 --- /dev/null +++ b/app/lib/retryable.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +# Module for retrying operations with external APIs +module Retryable + # Executes a code block with a specified number of retry attempts in case of specific errors + # @param max_attempts [Integer] maximum number of attempts (defaults to 3) + # @param retry_delay [Integer] delay between attempts in seconds (defaults to 2) + # @param exceptions [Array] exception classes to catch (defaults to all exceptions) + # @param logger [Object] logger object (must support info, warn, error methods) + # @param fallback [Proc] code block executed if all attempts fail + # @return [Object] result of the block execution or fallback + def with_retry( + max_attempts: 3, + retry_delay: 2, + exceptions: [StandardError], + logger: Rails.logger, + fallback: -> { [] } + ) + attempts = 0 + + retry_attempt = lambda do + attempts += 1 + yield + rescue *exceptions => e + logger.warn("Attempt #{attempts}/#{max_attempts} failed with error: #{e.class} - #{e.message}") + + if attempts < max_attempts + logger.info("Retrying in #{retry_delay} seconds...") + sleep retry_delay + retry_attempt.call + else + logger.error("All attempts exhausted. Last error: #{e.class} - #{e.message}") + fallback.call + end + end + + retry_attempt.call + end +end \ No newline at end of file diff --git a/test/jobs/org_registrant_phone_checker_job_test.rb b/test/jobs/org_registrant_phone_checker_job_test.rb index b9e2a764d..f5764c2c4 100644 --- a/test/jobs/org_registrant_phone_checker_job_test.rb +++ b/test/jobs/org_registrant_phone_checker_job_test.rb @@ -12,6 +12,17 @@ class OrgRegistrantPhoneCheckerJobTest < ActiveSupport::TestCase ident_country_code: 'EE', ident: '12345678' ) + + # Set environment variable to skip cache in tests + ENV['SKIP_COMPANY_REGISTER_CACHE'] = 'true' + + # Clear cache before each test to avoid interference + Rails.cache.clear if defined?(Rails.cache) + end + + teardown do + # Reset environment variable after tests + ENV['SKIP_COMPANY_REGISTER_CACHE'] = nil end def test_bulk_checker_processes_all_ee_org_contacts @@ -106,4 +117,59 @@ class OrgRegistrantPhoneCheckerJobTest < ActiveSupport::TestCase CompanyRegister::Client.define_singleton_method(:new, original_new_method) end + + # Test that successfully retries after a connection error + def test_retries_and_recovers_from_connection_error + call_count = 0 + + original_new_method = CompanyRegister::Client.method(:new) + CompanyRegister::Client.define_singleton_method(:new) do + object = original_new_method.call + def object.company_details(registration_number:) + parent_call_count = CompanyRegister::Client.class_variable_get(:@@test_call_count) + parent_call_count += 1 + CompanyRegister::Client.class_variable_set(:@@test_call_count, parent_call_count) + + if parent_call_count == 1 + raise HTTPClient::KeepAliveDisconnected, "Connection error" + end + + [OpenStruct.new(phone_numbers: ['+372.555666777'])] + end + object + end + + CompanyRegister::Client.class_variable_set(:@@test_call_count, 0) + + @contact.disclosed_attributes = [] + @contact.save! + OrgRegistrantPhoneCheckerJob.perform_now(type: 'single', registrant_user_code: @contact.code) + @contact.reload + + assert @contact.disclosed_attributes.include?('phone') + assert_operator CompanyRegister::Client.class_variable_get(:@@test_call_count), :>=, 2 + + CompanyRegister::Client.remove_class_variable(:@@test_call_count) + CompanyRegister::Client.define_singleton_method(:new, original_new_method) + end + + def test_returns_empty_array_when_max_retries_exceeded + original_new_method = CompanyRegister::Client.method(:new) + CompanyRegister::Client.define_singleton_method(:new) do + object = original_new_method.call + def object.company_details(registration_number:) + raise HTTPClient::KeepAliveDisconnected, "Persistent connection error" + end + object + end + + @contact.disclosed_attributes = ['phone'] + @contact.save! + OrgRegistrantPhoneCheckerJob.perform_now(type: 'single', registrant_user_code: @contact.code) + @contact.reload + + assert_not @contact.disclosed_attributes.include?('phone') + + CompanyRegister::Client.define_singleton_method(:new, original_new_method) + end end \ No newline at end of file diff --git a/test/lib/retryable_test.rb b/test/lib/retryable_test.rb new file mode 100644 index 000000000..f4fd1acf7 --- /dev/null +++ b/test/lib/retryable_test.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'test_helper' + +class RetryableTest < ActiveSupport::TestCase + class TestClass + include Retryable + + attr_reader :call_count + + def initialize + @call_count = 0 + end + + def test_retriable_method(should_fail: false, fail_count: 2) + with_retry( + max_attempts: 3, + retry_delay: 0.1, + logger: Rails.logger + ) do + @call_count += 1 + + if should_fail && @call_count <= fail_count + raise StandardError, "Тестовая ошибка #{@call_count}" + end + + 'success' + end + end + + def test_retriable_with_fallback(should_fail: true) + with_retry( + max_attempts: 2, + retry_delay: 0.1, + logger: Rails.logger, + fallback: -> { 'fallback' } + ) do + @call_count += 1 + raise StandardError, 'Постоянная ошибка' if should_fail + 'success' + end + end + end + + test 'should retry specified number of times and succeed' do + test_object = TestClass.new + result = test_object.test_retriable_method(should_fail: true, fail_count: 2) + + assert_equal 3, test_object.call_count + assert_equal 'success', result + end + + test 'should succeed on first try if no errors' do + test_object = TestClass.new + result = test_object.test_retriable_method(should_fail: false) + + assert_equal 1, test_object.call_count + assert_equal 'success', result + end + + test 'should use fallback when all retries fail' do + test_object = TestClass.new + result = test_object.test_retriable_with_fallback(should_fail: true) + + assert_equal 2, test_object.call_count + assert_equal 'fallback', result + end +end \ No newline at end of file