Merge pull request #31 from fivetran/add_event_extension_table_disable

add event_extension var
fivetran · Jul 25, 2024 · d0a989e · d0a989e
2 parents 1bda010 + 1497112
commit d0a989e
Show file tree

Hide file tree

Showing 12 changed files with 30 additions and 17 deletions.
diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh
@@ -19,7 +19,8 @@ dbt deps
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
-dbt run --vars '{iterable__using_campaign_label_history: false, iterable__using_user_unsubscribed_message_type_history: false, iterable__using_campaign_suppression_list_history: false}' --full-refresh --target "$db"
+dbt run --vars '{iterable_user_history_pass_through_columns: [{name: "phone_number_updated_at"}], iterable_event_extension_pass_through_columns: [{name: "web_push_message"}]}' --full-refresh --target "$db" 
+dbt run --vars '{iterable__using_campaign_label_history: false, iterable__using_user_unsubscribed_message_type_history: false, iterable__using_campaign_suppression_list_history: false, iterable__using_event_extension: false}' --full-refresh --target "$db"
 dbt run --vars '{ does_table_exist('user_unsubscribed_message_type'): false, does_table_exist('user_unsubscribed_channel'): false}' --full-refresh --target "$db"
 dbt test --target "$db"
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,9 @@
+# dbt_iterable_source v0.9.0
+[PR #31](https://github.com/fivetran/dbt_iterable_source/pull/31) contains the following updates:
+
+## 🚨 Breaking Changes 🚨
+- Introduces variable `iterable__using_event_extension` to disable the `stg_iterable__event_extension` model. This allows the downstream models to run even if the source `event_extension` table does not exist. For more information on how to configure refer to the [README](https://github.com/fivetran/dbt_iterable_source/blob/main/README.md#step-4-enablingdisabling-models).
+
 # dbt_iterable_source v0.8.1
 [PR #30](https://github.com/fivetran/dbt_iterable_source/pull/30) introduces the following updates: 
 

diff --git a/README.md b/README.md
@@ -42,7 +42,7 @@ Include the following Iterable package version in your `packages.yml` file.
 ```yaml
 packages:
  - package: fivetran/iterable_source
- version: [">=0.8.0", "<0.9.0"]
+ version: [">=0.9.0", "<0.10.0"]
 ```
 
 ## Step 3: Define database and schema variables
@@ -65,6 +65,7 @@ vars:
  iterable__using_campaign_label_history: false # default is true
  iterable__using_user_unsubscribed_message_type: false # default is true
  iterable__using_campaign_suppression_list_history: false # default is true 
+ iterable__using_event_extension: false # default is true 
 
 ```
 
@@ -75,7 +76,7 @@ vars:
 
 ### Passing Through Additional Fields
 
-This package includes fields we judged were standard across Iterable users. However, the Fivetran connector allows for additional columns to be brought through in the `event_extension` and `user_history` objects. Therefore, if you wish to bring them through, leverage our passthrough column variables.
+This package includes fields we judged were standard across Iterable users. However, the Fivetran connector allows for additional columns to be brought through in the `event_extension` and `user_history` objects. Therefore, if you wish to bring them through, leverage our passthrough column variables. For `event_extension` columns, ensure that `iterable__using_event_extension` is set to True, which is the default.
 
 **Notice**: A `dbt run --full-refresh` is required each time these variables are edited.
 

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,7 +1,7 @@
 config-version: 2
 name: 'iterable_source'
 
-version: '0.8.1'
+version: '0.9.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 models:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml
@@ -16,13 +16,13 @@ integration_tests:
  pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
  dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
  port: 5439
- schema: iterable_source_integration_tests_4
+ schema: iterable_source_integration_tests_8
  threads: 8
  bigquery:
  type: bigquery
  method: service-account-json
  project: 'dbt-package-testing'
- schema: iterable_source_integration_tests_4
+ schema: iterable_source_integration_tests_8
  threads: 8
  keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
  snowflake:
@@ -33,7 +33,7 @@ integration_tests:
  role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
  database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
  warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
- schema: iterable_source_integration_tests_4
+ schema: iterable_source_integration_tests_8
  threads: 8
  postgres:
  type: postgres
@@ -42,13 +42,13 @@ integration_tests:
  pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
  dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
  port: 5432
- schema: iterable_source_integration_tests_4
+ schema: iterable_source_integration_tests_8
  threads: 8
  databricks:
  catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
  host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
  http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
- schema: iterable_source_integration_tests_4
+ schema: iterable_source_integration_tests_8
  threads: 8
  token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
  type: databricks
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,12 +1,12 @@
 config-version: 2
 name: 'iterable_source_integration_tests'
 
-version: '0.8.1'
+version: '0.9.0'
 profile: 'integration_tests'
 
 vars:
  iterable_source:
- iterable_schema: iterable_source_integration_tests_4
+ iterable_schema: iterable_source_integration_tests_8
  iterable_campaign_history_identifier: "campaign_history_data"
  iterable_campaign_label_history_identifier: "campaign_label_history_data"
  iterable_campaign_list_history_identifier: "campaign_list_history_data"

diff --git a/models/src_iterable.yml b/models/src_iterable.yml
@@ -138,6 +138,8 @@ sources:
  - name: event_extension
  identifier: "{{ var('iterable_event_extension_identifier', 'event_extension')}}"
  description: Table containing additional properties from the event table.
+ config:
+ enabled: "{{ var('iterable__using_event_extension', true) }}"
  columns:
  - name: _fivetran_id
  description: A Fivetran-created unique identifier derived from hashing campaign_id, created_at, and event_name. 

diff --git a/models/stg_iterable__event_extension.sql b/models/stg_iterable__event_extension.sql
@@ -1,6 +1,7 @@
-{{
- config( materialized='view' )
-}}
+{{ config(
+ enabled=var('iterable__using_event_extension', True),
+ materialized='view'
+) }}
 
 with base as (
 

diff --git a/models/tmp/stg_iterable__event_extension_tmp.sql b/models/tmp/stg_iterable__event_extension_tmp.sql
@@ -1,2 +1,4 @@
+{{ config(enabled=var('iterable__using_event_extension', True)) }}
+
 select * 
 from {{ var('event_extension') }}