Merge pull request #143 from dbt-labs/removing_dbt_utils

Removing dbt utils dependency

.changes/unreleased/Under the Hood-20221012-114117.yaml

@@ -0,0 +1,7 @@
+kind: Under the Hood
+body: Removing dbt_utils
+time: 2022-10-12T11:41:17.757328-05:00
+custom:
+  Author: callum-mcdata
+  Issue: "143"
+  PR: "143"

integration_tests/models/custom_calendar.sql

@@ -1,7 +1,7 @@
 {{ config(materialized='table') }}
 
 with days as (
-    {{ dbt_utils.date_spine(
+    {{ metrics.metric_date_spine(
     datepart="day",
     start_date="cast('2010-01-01' as date)",
     end_date="cast('2030-01-01' as date)"

integration_tests/packages.yml

@@ -1,4 +1,4 @@
 packages:
   - local: ../
   - package: calogica/dbt_expectations
-    version: [">=0.5.0", "<0.6.0"]
+    version: [">=0.6.0", "<0.7.0"]

macros/misc/metrics__date_spine.sql

@@ -0,0 +1,128 @@
+{% macro metric_get_intervals_between(start_date, end_date, datepart) -%}
+    {{ return(adapter.dispatch('metric_get_intervals_between', 'metrics')(start_date, end_date, datepart)) }}
+{%- endmacro %}
+
+{% macro default__metric_get_intervals_between(start_date, end_date, datepart) -%}
+    {%- call statement('metric_get_intervals_between', fetch_result=True) %}
+
+        select {{ datediff(start_date, end_date, datepart) }}
+
+    {%- endcall -%}
+
+    {%- set value_list = load_result('metric_get_intervals_between') -%}
+
+    {%- if value_list and value_list['data'] -%}
+        {%- set values = value_list['data'] | map(attribute=0) | list %}
+        {{ return(values[0]) }}
+    {%- else -%}
+        {{ return(1) }}
+    {%- endif -%}
+
+{%- endmacro %}
+
+
+{% macro metric_date_spine(datepart, start_date, end_date) %}
+    {{ return(adapter.dispatch('metric_date_spine', 'metrics')(datepart, start_date, end_date)) }}
+{%- endmacro %}
+
+{% macro default__metric_date_spine(datepart, start_date, end_date) %}
+
+
+{# call as follows:
+
+metric_date_spine(
+    "day",
+    "to_date('01/01/2016', 'mm/dd/yyyy')",
+    "dateadd(week, 1, current_date)"
+) #}
+
+
+with rawdata as (
+
+    {{metrics.metric_generate_series(
+        metrics.metric_get_intervals_between(start_date, end_date, datepart)
+    )}}
+
+),
+
+all_periods as (
+
+    select (
+        {{
+            dateadd(
+                datepart,
+                "row_number() over (order by 1) - 1",
+                start_date
+            )
+        }}
+    ) as date_{{datepart}}
+    from rawdata
+
+),
+
+filtered as (
+
+    select *
+    from all_periods
+    where date_{{datepart}} <= {{ end_date }}
+
+)
+
+select * from filtered
+
+{% endmacro %}
+
+
+{% macro metric_get_powers_of_two(upper_bound) %}
+    {{ return(adapter.dispatch('metric_get_powers_of_two', 'metrics')(upper_bound)) }}
+{% endmacro %}
+
+{% macro default__metric_get_powers_of_two(upper_bound) %}
+
+    {% if upper_bound <= 0 %}
+    {{ exceptions.raise_compiler_error("upper bound must be positive") }}
+    {% endif %}
+
+    {% for _ in range(1, 100) %}
+       {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %}
+    {% endfor %}
+
+{% endmacro %}
+
+
+{% macro metric_generate_series(upper_bound) %}
+    {{ return(adapter.dispatch('metric_generate_series', 'metrics')(upper_bound)) }}
+{% endmacro %}
+
+{% macro default__metric_generate_series(upper_bound) %}
+
+    {% set n = metrics.metric_get_powers_of_two(upper_bound) %}
+
+    with p as (
+        select 0 as generated_number union all select 1
+    ), unioned as (
+
+    select
+
+    {% for i in range(n) %}
+    p{{i}}.generated_number * power(2, {{i}})
+    {% if not loop.last %} + {% endif %}
+    {% endfor %}
+    + 1
+    as generated_number
+
+    from
+
+    {% for i in range(n) %}
+    p as p{{i}}
+    {% if not loop.last %} cross join {% endif %}
+    {% endfor %}
+
+    )
+
+    select *
+    from unioned
+    where generated_number <= {{upper_bound}}
+    order by generated_number
+
+{% endmacro %}

macros/misc/metrics__equality.sql

@@ -0,0 +1,99 @@
+{% test metric_equality(model, compare_model, compare_columns=None) %}
+  {{ return(adapter.dispatch('test_metric_equality', 'metrics')(model, compare_model, compare_columns)) }}
+{% endtest %}
+
+{% macro default__test_metric_equality(model, compare_model, compare_columns=None) %}
+
+{% set set_diff %}
+    count(*) + coalesce(abs(
+        sum(case when which_diff = 'a_minus_b' then 1 else 0 end) -
+        sum(case when which_diff = 'b_minus_a' then 1 else 0 end)
+    ), 0)
+{% endset %}
+
+{#-- Needs to be set at parse time, before we return '' below --#}
+{{ config(fail_calc = set_diff) }}
+
+{#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #}
+{%- if not execute -%}
+    {{ return('') }}
+{% endif %}
+
+-- setup
+{%- do metrics._metric_is_relation(model, 'test_metric_equality') -%}
+
+{#-
+If the compare_cols arg is provided, we can run this test without querying the
+information schema — this allows the model to be an ephemeral model
+-#}
+
+{%- if not compare_columns -%}
+    {%- do metrics._metric_is_ephemeral(model, 'test_metric_equality') -%}
+    {%- set compare_columns = adapter.get_columns_in_relation(model) | map(attribute='quoted') -%}
+{%- endif -%}
+
+{% set compare_cols_csv = compare_columns | join(', ') %}
+
+with a as (
+
+    select * from {{ model }}
+
+),
+
+b as (
+
+    select * from {{ compare_model }}
+
+),
+
+a_minus_b as (
+
+    select {{compare_cols_csv}} from a
+    {{ except() }}
+    select {{compare_cols_csv}} from b
+
+),
+
+b_minus_a as (
+
+    select {{compare_cols_csv}} from b
+    {{ except() }}
+    select {{compare_cols_csv}} from a
+
+),
+
+unioned as (
+
+    select 'a_minus_b' as which_diff, a_minus_b.* from a_minus_b
+    union all
+    select 'b_minus_a' as which_diff, b_minus_a.* from b_minus_a
+
+)
+
+select * from unioned
+
+{% endmacro %}
+
+
+{% macro _metric_is_relation(obj, macro) %}
+    {%- if not (obj is mapping and obj.get('metadata', {}).get('type', '').endswith('Relation')) -%}
+        {%- do exceptions.raise_compiler_error("Macro " ~ macro ~ " expected a Relation but received the value: " ~ obj) -%}
+    {%- endif -%}
+{% endmacro %}
+
+{% macro _metric_is_ephemeral(obj, macro) %}
+    {%- if obj.is_cte -%}
+        {% set ephemeral_prefix = api.Relation.add_ephemeral_prefix('') %}
+        {% if obj.name.startswith(ephemeral_prefix) %}
+            {% set model_name = obj.name[(ephemeral_prefix|length):] %}
+        {% else %}
+            {% set model_name = obj.name %}
+        {%- endif -%}
+        {% set error_message %}
+The `{{ macro }}` macro cannot be used with ephemeral models, as it relies on the information schema.
+
+`{{ model_name }}` is an ephemeral model. Consider making it a view or table instead.
+        {% endset %}
+        {%- do exceptions.raise_compiler_error(error_message) -%}
+    {%- endif -%}
+{% endmacro %}

models/dbt_metrics_default_calendar.sql

@@ -1,7 +1,7 @@
 {{ config(materialized='table') }}
 
 with days as (
-    {{ dbt_utils.date_spine(
+    {{ metrics.metric_date_spine(
     datepart="day",
     start_date="cast('1990-01-01' as date)",
     end_date="cast('2030-01-01' as date)"

tests/functional/base_metric_types/test_base_average_metric.py

@@ -8,6 +8,7 @@
     fact_orders_source_csv,
     fact_orders_sql,
     fact_orders_yml,
+    packages_yml
 )
 
 # models/base_average_metric.sql
@@ -26,7 +27,7 @@
 models:
   - name: base_average_metric
     tests: 
-      - dbt_utils.equality:
+      - metrics.metric_equality:
           compare_model: ref('base_average_metric__expected')
 metrics:
   - name: base_average_metric

tests/functional/base_metric_types/test_base_count_distinct_metric.py

@@ -26,7 +26,7 @@
 models:
   - name: base_count_distinct_metric
     tests: 
-      - dbt_utils.equality:
+      - metrics.metric_equality:
           compare_model: ref('base_count_distinct_metric__expected')
 
 metrics:

tests/functional/base_metric_types/test_base_count_metric.py

@@ -26,7 +26,7 @@
 models:
   - name: base_count_metric
     tests: 
-      - dbt_utils.equality:
+      - metrics.metric_equality:
           compare_model: ref('base_count_metric__expected')
 metrics:
   - name: base_count_metric

tests/functional/base_metric_types/test_base_max_metric.py

@@ -26,7 +26,7 @@
 models:
   - name: base_max_metric
     tests: 
-      - dbt_utils.equality:
+      - metrics.metric_equality:
           compare_model: ref('base_max_metric__expected')
 metrics:
   - name: base_max_metric

tests/functional/base_metric_types/test_base_min_metric.py

@@ -26,7 +26,7 @@
 models:
   - name: base_min_metric
     tests: 
-      - dbt_utils.equality:
+      - metrics.metric_equality:
           compare_model: ref('base_min_metric__expected')
 metrics:
   - name: base_min_metric

tests/functional/base_metric_types/test_base_sum_metric.py

@@ -26,7 +26,7 @@
 models:
   - name: base_sum_metric
     tests: 
-      - dbt_utils.equality:
+      - metrics.metric_equality:
           compare_model: ref('base_sum_metric__expected')
 metrics:
   - name: base_sum_metric

tests/functional/derived_metric_types/test_derived_metric.py

@@ -42,7 +42,7 @@
 models:
   - name: derived_metric
     tests: 
-      - dbt_utils.equality:
+      - metrics.metric_equality:
           compare_model: ref('derived_metric__expected')
 metrics:
   - name: derived_metric

tests/functional/derived_metric_types/test_divide_by_zero_derived_metric.py

@@ -28,7 +28,7 @@
 models:
   - name: divide_by_zero_expression_metric
     tests: 
-      - dbt_utils.equality:
+      - metrics.metric_equality:
           compare_model: ref('divide_by_zero_expression_metric__expected')
 metrics:
   - name: base_sum_metric

tests/functional/derived_metric_types/test_metric_on_derived_metric.py

@@ -42,7 +42,7 @@
 models:
   - name: metric_on_derived_metric
     tests: 
-      - dbt_utils.equality:
+      - metrics.metric_equality:
           compare_model: ref('metric_on_derived_metric__expected')
 metrics:
   - name: derived_metric

tests/functional/derived_metric_types/test_ratio_metric.py

@@ -58,7 +58,7 @@
 models:
   - name: ratio_metric
     tests: 
-      - dbt_utils.equality:
+      - metrics.metric_equality:
           compare_model: ref('ratio_metric__expected')
 metrics:
   - name: ratio_metric

tests/functional/develop/test_develop_matches_calculate.py

@@ -70,7 +70,7 @@
 models:
   - name: testing_metric_develop
     tests: 
-      - dbt_utils.equality:
+      - metrics.metric_equality:
           compare_model: ref('testing_metric_calculate')
 
 """

tests/functional/develop/test_develop_metric.py

@@ -45,7 +45,7 @@
 models:
   - name: develop_metric
     tests: 
-      - dbt_utils.equality:
+      - metrics.metric_equality:
           compare_model: ref('develop_metric__expected')
 
 """
@@ -209,7 +209,7 @@ def test_build_completion(self,project,):
 models:
   - name: develop_multiple_metrics
     tests: 
-      - dbt_utils.equality:
+      - metrics.metric_equality:
           compare_model: ref('develop_multiple_metrics__expected')
 
 """