From 0742419f346fd290255fe06ce2415b4515a854cf Mon Sep 17 00:00:00 2001 From: asatwal Date: Thu, 12 Nov 2026 13:43:30 +0000 Subject: [PATCH] Remove the airbyte mandatory config check from main flow and add dataset to err messages --- lib/dfe/analytics/big_query_api.rb | 6 ++---- spec/dfe/analytics/big_query_api_spec.rb | 16 ++-------------- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/lib/dfe/analytics/big_query_api.rb b/lib/dfe/analytics/big_query_api.rb index ec48b907..a6b1631f 100644 --- a/lib/dfe/analytics/big_query_api.rb +++ b/lib/dfe/analytics/big_query_api.rb @@ -11,12 +11,10 @@ class BigQueryApi RETRY_INTERVAL_MULTIPLIER = 2 BIGQUERY_MANDATORY_CONFIG = %i[bigquery_project_id bigquery_table_name bigquery_dataset azure_client_id azure_token_path azure_scope gcp_scope google_cloud_credentials].freeze - BIGQUERY_AIRBYTE_MANDATORY_CONFIG = %i[bigquery_airbyte_dataset].freeze def self.client @client ||= begin DfE::Analytics::Config.check_missing_config!(BIGQUERY_MANDATORY_CONFIG) - DfE::Analytics::Config.check_missing_config!(BIGQUERY_AIRBYTE_MANDATORY_CONFIG) if DfE::Analytics.airbyte_enabled? Google::Apis::BigqueryV2::BigqueryService.new end @@ -79,7 +77,7 @@ def self.apply_policy_tags(dataset, tables, policy_tags) table_name.to_s ) rescue Google::Apis::ClientError => e - error_message = "DfE::Analytics Failed to retrieve table: #{table_name}: #{e.message}" + error_message = "DfE::Analytics Failed to retrieve table: #{dataset}.#{table_name}: #{e.message}" Rails.logger.error(error_message) raise PolicyTagError, error_message end @@ -111,7 +109,7 @@ def self.apply_policy_tags(dataset, tables, policy_tags) fields: 'schema' ) rescue Google::Apis::ClientError => e - error_message = "DfE::Analytics Failed to update table: #{table_name}: #{e.message}" + error_message = "DfE::Analytics Failed to update table: #{dataset}.#{table_name}: #{e.message}" Rails.logger.error(error_message) raise PolicyTagError, error_message end diff --git a/spec/dfe/analytics/big_query_api_spec.rb b/spec/dfe/analytics/big_query_api_spec.rb index f512ae08..44bc97d0 100644 --- a/spec/dfe/analytics/big_query_api_spec.rb +++ b/spec/dfe/analytics/big_query_api_spec.rb @@ -32,20 +32,8 @@ end end - it 'raises a configuration error if airbyte is enabled and airbyte config is missing' do - with_analytics_config(test_dummy_config.merge(bigquery_airbyte_dataset: nil)) do - allow(DfE::Analytics::Config).to receive(:airbyte_enabled?).and_return(true) - - expect do - described_class.client - end.to raise_error(DfE::Analytics::Config::ConfigurationError) - end - end - it 'initializes and returns the BigQuery client with authorization' do - with_analytics_config(test_dummy_config.merge(bigquery_airbyte_dataset: 'airbyte_dataset')) do - allow(DfE::Analytics::Config).to receive(:airbyte_enabled?).and_return(true) - + with_analytics_config(test_dummy_config) do expect(described_class.client).to eq(client) end end @@ -53,7 +41,7 @@ describe '.insert' do subject(:insert) do - with_analytics_config(test_dummy_config.merge(bigquery_airbyte_dataset: 'airbyte_dataset')) do + with_analytics_config(test_dummy_config) do described_class.insert([event.as_json]) end end