diff --git a/docs/airbyte.md b/docs/airbyte.md index 825868dc..c25753a9 100644 --- a/docs/airbyte.md +++ b/docs/airbyte.md @@ -25,14 +25,35 @@ Main configuration options for Airbyte: - The service database is synchronised with BigQuery every 15 minutes -- The final tables are using incremental append mode, with no deduplication. This results in a new row in the final table for every change in the service table. This allows Dashboards and Reports to have access the latest and historical versions of the data. +- The final tables use incremental append mode by default, with no deduplication. This results in a new row in the final table for every change in the service table. This allows Dashboards and Reports to have access to the latest and historical versions of the data. + +- A different Airbyte sync mode can be configured for an individual table in `analytics.yml`. The following sync mode values are available: + + | VALUE | DESCRIPTION | + |---------------------------------|------------------------------------| + | `incremental_deduped_history` | Incremental \| Append + Deduped | + | `full_refresh_overwrite` | Full refresh \| Overwrite | + | `incremental_append` | Incremental \| Append | + | `full_refresh_append` | Full refresh \| Append | + | `full_refresh_overwrite_deduped` | Full refresh \| Overwrite + Deduped | + + The following is an example of how to configure a custom Airbyte sync mode for the `audits` table in `analytics.yml`: + + ```yaml + audits: + - sync_mode: incremental_deduped_history + - id + - auditable_type + - action + - created_at + ``` - Most of the configuration is maintained by SD DevOps using terraform. Any service specific configuration is delegated to the service. ### Airbyte DfE Analytics configuration and tasks -- Only tables and columns specified in the `analytics.yml` config file are synchronised +- Only tables and columns specified in the `analytics.yml` config file are synchronised - Any columns with sensitive data listed in the `analytics_blocklist.yml` config file will have the hidden policy tag applied. A rails rake task is available for this:
`rake dfe:analytics:big_query_apply_policy_tags` diff --git a/lib/dfe/analytics.rb b/lib/dfe/analytics.rb index ee7260ac..98c72f43 100644 --- a/lib/dfe/analytics.rb +++ b/lib/dfe/analytics.rb @@ -106,9 +106,28 @@ def self.enabled? end def self.allowlist + analytics_config.transform_values do |attributes| + attributes.reject { |attribute| airbyte_sync_mode?(attribute) } + end + end + + def self.airbyte_sync_modes + analytics_config.each_with_object({}) do |(entity, attributes), sync_modes| + config = attributes.find { |attribute| airbyte_sync_mode?(attribute) } + sync_modes[entity] = config[:sync_mode] || config['sync_mode'] if config + end + end + + def self.analytics_config Rails.application.config_for(:analytics) end + def self.airbyte_sync_mode?(attribute) + attribute.is_a?(Hash) && (attribute.key?(:sync_mode) || attribute.key?('sync_mode')) + end + + private_class_method :analytics_config, :airbyte_sync_mode? + def self.hidden_pii Rails.application.config_for(:analytics_hidden_pii) rescue RuntimeError diff --git a/lib/dfe/analytics/airbyte_stream_config.rb b/lib/dfe/analytics/airbyte_stream_config.rb index 49031215..31028d46 100644 --- a/lib/dfe/analytics/airbyte_stream_config.rb +++ b/lib/dfe/analytics/airbyte_stream_config.rb @@ -41,7 +41,7 @@ def self.entity_attributes private_class_method def self.table_for(entity, attributes) { name: entity.to_s, - syncMode: INCREMENTAL_APPEND_SYNC_MODE, + syncMode: DfE::Analytics.airbyte_sync_modes.fetch(entity, INCREMENTAL_APPEND_SYNC_MODE), cursorField: CURSOR_FIELD, primaryKey: [[primary_key_for(attributes)]], selectedFields: (CURSOR_FIELD + AIRBYTE_FIELDS + attributes).uniq.map { |attribute| { fieldPath: [attribute] } } diff --git a/spec/dfe/analytics/airbyte_stream_config_spec.rb b/spec/dfe/analytics/airbyte_stream_config_spec.rb index b4c0e47b..c4c5d09a 100644 --- a/spec/dfe/analytics/airbyte_stream_config_spec.rb +++ b/spec/dfe/analytics/airbyte_stream_config_spec.rb @@ -44,6 +44,30 @@ end end + context 'when a sync mode is configured for an entity' do + let(:entity_attributes) do + { + users: %w[id name email], + audits: %w[id auditable_type action created_at] + } + end + + before do + allow(DfE::Analytics).to receive(:airbyte_sync_modes).and_return( + audits: 'incremental_deduped_history' + ) + end + + it 'uses the configured sync mode only for that entity' do + streams = airbyte_stream_config[:configurations][:streams] + users_stream = streams.find { |stream| stream[:name] == 'users' } + audits_stream = streams.find { |stream| stream[:name] == 'audits' } + + expect(users_stream[:syncMode]).to eq('incremental_append') + expect(audits_stream[:syncMode]).to eq('incremental_deduped_history') + end + end + context 'when attributes do not include "id"' do let(:entity_attributes) { { users: %w[email name] } } diff --git a/spec/dfe/analytics_spec.rb b/spec/dfe/analytics_spec.rb index 7a11dd73..7e70d34b 100644 --- a/spec/dfe/analytics_spec.rb +++ b/spec/dfe/analytics_spec.rb @@ -1,6 +1,50 @@ # frozen_string_literal: true RSpec.describe DfE::Analytics do + describe '.allowlist' do + before do + allow(Rails.application).to receive(:config_for).with(:analytics).and_return( + users: %w[id name email], + audits: [ + { sync_mode: 'incremental_deduped_history' }, + 'id', + 'auditable_type', + 'auditable_id', + 'action', + 'audited_changes', + 'created_at' + ] + ) + end + + it 'excludes Airbyte sync mode configuration from the entity attributes' do + expect(described_class.allowlist).to eq( + users: %w[id name email], + audits: %w[id auditable_type auditable_id action audited_changes created_at] + ) + end + end + + describe '.airbyte_sync_modes' do + before do + allow(Rails.application).to receive(:config_for).with(:analytics).and_return( + users: %w[id name email], + audits: [ + { sync_mode: 'incremental_deduped_history' }, + 'id', + 'auditable_type', + 'created_at' + ] + ) + end + + it 'returns the configured Airbyte sync modes by entity' do + expect(described_class.airbyte_sync_modes).to eq( + audits: 'incremental_deduped_history' + ) + end + end + it 'has a version number' do expect(DfE::Analytics::VERSION).not_to be nil end