Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Rework filter bots macro #86

Merged
merged 1 commit into from
Oct 18, 2024
Merged

Rework filter bots macro #86

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion integration_tests/models/source/bigquery/snowplow_unified_events_stg.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -494,7 +494,7 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
select as struct JSON_EXTRACT_scalar(json_array,'$.category') as category,
JSON_EXTRACT_scalar(json_array,'$.primary_impact') as primary_impact,
JSON_EXTRACT_scalar(json_array,'$.reason') as reason,
JSON_EXTRACT_scalar(json_array,'$.spider_or_robot') as spider_or_robot
cast(JSON_EXTRACT_scalar(json_array,'$.spider_or_robot') as boolean) as spider_or_robot

from unnest(contexts_com_iab_snowplow_spiders_and_robots_1_0_0) as json_array
) as contexts_com_iab_snowplow_spiders_and_robots_1_0_0
Expand Down
14 changes: 14 additions & 0 deletions macros/filter_bots.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,31 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
{%- endmacro -%}

{% macro default__filter_bots(table_alias = none) %}
{% if var('snowplow__enable_iab', false) %}
{# additional logic in case the result is null due to server anonymization #}
and coalesce(iab__spider_or_robot, False ) = False
{% endif %}
and {% if table_alias %}{{table_alias~'.'}}{% endif %}useragent not similar to '%(bot|crawl|slurp|spider|archiv|spinn|sniff|seo|audit|survey|pingdom|worm|capture|(browser|screen)shots|analyz|index|thumb|check|facebook|PingdomBot|PhantomJS|YandexBot|Twitterbot|a_archiver|facebookexternalhit|Bingbot|BingPreview|Googlebot|Baiduspider|360(Spider|User-agent)|semalt)%'
{% endmacro %}

{% macro bigquery__filter_bots(table_alias = none) %}
{% if var('snowplow__enable_iab', false) %}
and coalesce(iab__spider_or_robot, False ) = False
{% endif %}
and not regexp_contains({% if table_alias %}{{table_alias~'.'}}{% endif %}useragent, '(bot|crawl|slurp|spider|archiv|spinn|sniff|seo|audit|survey|pingdom|worm|capture|(browser|screen)shots|analyz|index|thumb|check|facebook|PingdomBot|PhantomJS|YandexBot|Twitterbot|a_archiver|facebookexternalhit|Bingbot|BingPreview|Googlebot|Baiduspider|360(Spider|User-agent)|semalt)')
{% endmacro %}

{% macro spark__filter_bots(table_alias = none) %}
{% if var('snowplow__enable_iab', false) %}
{# had to add different syntax as the coalesce based one resulted in a Spark error #}
and (not iab__spider_or_robot = True or iab__spider_or_robot is null)
{% endif %}
and not rlike({% if table_alias %}{{table_alias~'.'}}{% endif %}useragent, '.*(bot|crawl|slurp|spider|archiv|spinn|sniff|seo|audit|survey|pingdom|worm|capture|(browser|screen)shots|analyz|index|thumb|check|facebook|PingdomBot|PhantomJS|YandexBot|Twitterbot|a_archiver|facebookexternalhit|Bingbot|BingPreview|Googlebot|Baiduspider|360(Spider|User-agent)|semalt).*')
{% endmacro %}

{% macro snowflake__filter_bots(table_alias = none) %}
{% if var('snowplow__enable_iab', false) %}
and coalesce(iab__spider_or_robot, False ) = False
{% endif %}
and not rlike({% if table_alias %}{{table_alias~'.'}}{% endif %}useragent, '.*(bot|crawl|slurp|spider|archiv|spinn|sniff|seo|audit|survey|pingdom|worm|capture|(browser|screen)shots|analyz|index|thumb|check|facebook|PingdomBot|PhantomJS|YandexBot|Twitterbot|a_archiver|facebookexternalhit|Bingbot|BingPreview|Googlebot|Baiduspider|360(Spider|User-agent)|semalt).*')
{% endmacro %}
8 changes: 0 additions & 8 deletions ...l_modules/core_web_vitals/scratch/bigquery/snowplow_unified_web_vital_events_this_run.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,14 +70,6 @@ with prep as (

-- exclude bot traffic

{% if var('snowplow__enable_iab', false) %}
and not {{ snowplow_utils.get_field(column_name = 'contexts_com_iab_snowplow_spiders_and_robots_1_0_0',
field_name = 'spider_or_robot',
table_alias = 'e',
type = 'boolean',
array_index = 0)}} = True
{% endif %}

{{ snowplow_unified.filter_bots() }}

)
Expand Down
8 changes: 0 additions & 8 deletions ...modules/core_web_vitals/scratch/databricks/snowplow_unified_web_vital_events_this_run.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,14 +70,6 @@ with prep as (

-- exclude bot traffic

{% if var('snowplow__enable_iab', false) %}
and not {{ snowplow_utils.get_field(column_name = 'contexts_com_iab_snowplow_spiders_and_robots_1',
field_name = 'spider_or_robot',
table_alias = 'e',
type = 'boolean',
array_index = 0)}} = True
{% endif %}

{{ snowplow_unified.filter_bots() }}

)
Expand Down
4 changes: 0 additions & 4 deletions ...al_modules/core_web_vitals/scratch/default/snowplow_unified_web_vital_events_this_run.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,6 @@ with prep as (

-- exclude bot traffic

{% if var('snowplow__enable_iab', false) %}
and not e.iab__spider_or_robot = True
agnessnowplow marked this conversation as resolved.
Show resolved Hide resolved
{% endif %}

{{ snowplow_unified.filter_bots() }}

)
Expand Down
Loading