Setup packages with bash

docker-compose.yaml

@@ -16,6 +16,7 @@ x-airflow-common:
  - ./variables/docker-env-secrets # added to gitignore
  volumes:
  - ./dags:/opt/airflow/dags
+ - ./gcp_airflow_foundations_config:/opt/airflow/gcp_airflow_foundations_config
  - ./logs:/opt/airflow/logs
  - ./plugins:/opt/airflow/plugins
  - ./variables:/opt/airflow/variables

docker/Dockerfile

@@ -5,7 +5,7 @@ FROM "apache/airflow:${AIRFLOW_IMAGE_NAME}"
 # - Install GCP util
 RUN curl -sSL https://sdk.cloud.google.com | bash
 ENV PATH $PATH:/home/airflow/google-cloud-sdk/bin
-
+ENV PKG_NAME gcp_airflow_foundations
 # - Copy a custom airflow config file
 #COPY airflow.cfg ${AIRFLOW_HOME}/airflow.cfg
 
@@ -15,16 +15,16 @@ ENV PATH $PATH:/home/airflow/google-cloud-sdk/bin
 RUN mkdir /opt/airflow/gcp_airflow_foundations
 
 # Copying only files essential for installing gcp_airflow_foundations
-COPY setup.py setup.cfg README.md MANIFEST.in requirements.txt requirements-providers.txt requirements-ci.txt /opt/airflow/
+COPY setup.py setup.cfg README.md MANIFEST.in requirements.txt requirements-providers.txt requirements-test.txt /opt/airflow/
 #COPY tests /opt/airflow/tests
 COPY gcp_airflow_foundations/__init__.py /opt/airflow/gcp_airflow_foundations/__init__.py
 COPY gcp_airflow_foundations/version.py /opt/airflow/gcp_airflow_foundations/version.py
-
+COPY gcp_airflow_foundations_config/framework.yaml /opt/airflow/gcp_airflow_foundations_config/framework.yaml
 WORKDIR /opt/airflow
 
 # USER root
 RUN pip install --upgrade pip
-
+RUN pip install pyyaml
 # - Install dependencies
 RUN pip install -e .[providers,test]
 

docker/Dockerfile-ci

@@ -6,6 +6,7 @@ FROM "apache/airflow:${AIRFLOW_IMAGE_NAME}"
 # - Install GCP util
 RUN curl -sSL https://sdk.cloud.google.com | bash
 ENV PATH $PATH:/home/airflow/google-cloud-sdk/bin
+ENV PKG_NAME gcp_airflow_foundations
 
 # - Copy a custom airflow config file
 #COPY airflow.cfg ${AIRFLOW_HOME}/airflow.cfg
@@ -16,18 +17,19 @@ ENV PATH $PATH:/home/airflow/google-cloud-sdk/bin
 RUN mkdir /opt/airflow/gcp_airflow_foundations
 
 # Copying only files essential for installing gcp_airflow_foundations
-COPY setup.py setup.cfg README.md MANIFEST.in requirements.txt requirements-providers.txt requirements-ci.txt /opt/airflow/
+COPY setup.py setup.cfg README.md MANIFEST.in requirements.txt requirements-providers.txt requirements-test.txt /opt/airflow/
 COPY gcp_airflow_foundations/__init__.py /opt/airflow/gcp_airflow_foundations/__init__.py
 COPY gcp_airflow_foundations/version.py /opt/airflow/gcp_airflow_foundations/version.py
+COPY gcp_airflow_foundations_config/framework.yaml /opt/airflow/gcp_airflow_foundations_config/framework.yaml
 
 WORKDIR /opt/airflow
 
 RUN pip install --upgrade pip
-
+RUN pip install -r requirements-test.txt
 
 # - Install dependencies
 RUN pip install -e .[providers,test]
-RUN pip install -r requirements.txt
 
+# init db airflow
 RUN airflow db init
 

gcp_airflow_foundations/version.py

@@ -1 +1 @@
-__version__ = "0.3.5"
+__version__ = "0.3.6"

gcp_airflow_foundations_config/framework.yaml

@@ -0,0 +1,44 @@
+package-name: gcp_airflow_foundations
+
+package-startwith: gcp_airflow_foundations.
+
+name: gcp-airflow-foundations
+
+description: |
+ `gcp-airflow-foundations`
+
+long-description: |
+ Please upload README from this metadata, with range dependencies and versions available
+
+versions:
+ - "0.3.5"
+ - "0.3.3"
+ - "0.3.2"
+ - "0.3.1"
+ - "0.3.0"
+ - "0.2.11"
+ - "0.2.10"
+ - "0.2.9"
+ - "0.2.8"
+ - "0.2.7"
+ - "0.2.6"
+ - "0.2.5"
+ - "0.2.4"
+ - "0.2.3"
+ - "0.2.2"
+ - "0.2.1"
+ - "0.2.0"
+
+
+dependencies:
+ - pydantic==1.8.2
+ - facebook_business>=10.0.0
+ - dacite>=1.5.0
+ - regex>=2021.11.1
+ - twilio
+ - pandas>=0.17.1
+ - pyarrow>=3.0.0
+
+extras:
+ - providers
+ - test

gcp_airflow_foundations_config/plugins_facebook.yaml

@@ -0,0 +1,20 @@
+package-name: gcp_airflow_foundations_facebook
+
+package-startwith: gcp_airflow_foundations_facebook
+
+name: gcp-airflow-foundations-facebook
+
+description: |
+ `A Facebook plug-in for gcp-airflow-foundations`
+
+long-description: |
+ Please upload README from this metadata, with range dependencies and versions available
+
+versions:
+ - "0.0.1"
+
+dependencies:
+ - pydantic==1.8.2
+ - facebook-business>=6.0.2
+
+extras: {}

gcp_airflow_foundations_facebook/base_class/facebook_config.py

@@ -0,0 +1,90 @@
+from typing import List, Optional
+
+
+from pydantic import validator
+from pydantic.dataclasses import dataclass
+
+from facebook_business.adobjects.adsinsights import AdsInsights
+
+from gcp_airflow_foundations_facebook.enums.facebook import (
+ Level,
+ DatePreset,
+ AccountLookupScope,
+ ApiObject,
+)
+
+valid_fields = {
+ "account_name": AdsInsights.Field.account_name,
+ "account_id": AdsInsights.Field.account_id,
+ "attribution_setting": AdsInsights.Field.attribution_setting,
+ "account_currency": AdsInsights.Field.account_currency,
+ "campaign_name": AdsInsights.Field.campaign_name,
+ "campaign_id": AdsInsights.Field.campaign_id,
+ "adset_name": AdsInsights.Field.adset_name,
+ "adset_id": AdsInsights.Field.adset_id,
+ "ad_name": AdsInsights.Field.ad_name,
+ "ad_id": AdsInsights.Field.ad_id,
+ "impressions": AdsInsights.Field.impressions,
+ "spend": AdsInsights.Field.spend,
+ "reach": AdsInsights.Field.reach,
+ "clicks": AdsInsights.Field.clicks,
+ "cpc": AdsInsights.Field.cpc,
+ "ctr": AdsInsights.Field.ctr,
+ "cpm": AdsInsights.Field.cpm,
+ "unique_clicks": AdsInsights.Field.unique_clicks,
+ "inline_link_clicks": AdsInsights.Field.inline_link_clicks,
+ "unique_inline_link_click_ctr": AdsInsights.Field.unique_inline_link_click_ctr,
+ "inline_link_click_ctr": AdsInsights.Field.inline_link_click_ctr,
+ "unique_inline_link_clicks": AdsInsights.Field.unique_inline_link_clicks,
+ "cost_per_unique_inline_link_click": AdsInsights.Field.cost_per_unique_inline_link_click,
+ "cost_per_unique_outbound_click": AdsInsights.Field.cost_per_unique_outbound_click,
+ "cost_per_unique_click": AdsInsights.Field.cost_per_unique_click,
+ "cost_per_thruplay": AdsInsights.Field.cost_per_thruplay,
+ "video_30_sec_watched_actions": AdsInsights.Field.video_30_sec_watched_actions,
+ "video_p25_watched_actions": AdsInsights.Field.video_p25_watched_actions,
+ "video_p50_watched_actions": AdsInsights.Field.video_p50_watched_actions,
+ "video_p75_watched_actions": AdsInsights.Field.video_p75_watched_actions,
+ "video_p100_watched_actions": AdsInsights.Field.video_p100_watched_actions,
+ "video_play_actions": AdsInsights.Field.video_play_actions,
+ "conversion_values": AdsInsights.Field.conversion_values,
+ "conversions": AdsInsights.Field.conversions,
+ "cost_per_conversion": AdsInsights.Field.cost_per_conversion,
+ "actions": AdsInsights.Field.actions,
+ "action_values": AdsInsights.Field.action_values,
+ "cost_per_action_type": AdsInsights.Field.cost_per_action_type,
+}
+
+
+@dataclass
+class FacebookConfig:
+ """
+ Attributes:
+ fields: A list of dimensions and metrics for the Facebook Graph API. For more information see: https://developers.facebook.com/docs/marketing-api/insights/parameters/v12.0
+ level: Represents the level of result {ad, adset, campaign, account}
+ account_lookup_scope: Whether to query all accounts managed by the user or only the active ones
+ account_bq_table: A BigQuery table with the account_id's
+ time_increment: The time dimension of the results
+ time_range: The time range used to query the Graph API
+ use_account_attribution_setting: When this parameter is set to true, your ads results will be shown using the attribution settings defined for the ad account.
+ use_unified_attribution_setting: When this parameter is set to true, your ads results will be shown using unified attribution settings defined at ad set level and parameter
+ """
+
+ fields: Optional[List[str]]
+ level: Optional[Level]
+ account_lookup_scope: AccountLookupScope
+ accounts_bq_table: Optional[str]
+ time_increment: Optional[str]
+ time_range: Optional[dict]
+ use_account_attribution_setting: Optional[bool] = False
+ use_unified_attribution_setting: Optional[bool] = False
+
+ @validator("fields")
+ def valid_fields(cls, v):
+ if v is not None:
+ for field in v:
+ assert (
+ field in valid_fields
+ ), f"`{field}` is not a valid field for the Facebook API"
+ return [valid_fields[field] for field in v]
+ else:
+ return []

gcp_airflow_foundations_facebook/base_class/facebook_table_config.py

@@ -0,0 +1,27 @@
+
+from dataclasses import dataclass
+from pydantic import validator
+from typing import List, Optional
+
+from gcp_airflow_foundations_facebook.enums.facebook import ApiObject
+
+
+@dataclass
+class FacebookTableConfig:
+ """
+ Attributes:
+ api_object: The API object to query {insights, campaign, adset}
+ breakdowns: How to break down the result. For more than one breakdown, only certain combinations are available.
+ action_breakdowns: How to break down action results. Supports more than one breakdowns. Default value is ["action_type"].
+ """
+
+ api_object: Optional[ApiObject]
+ breakdowns: Optional[List[str]]
+ action_breakdowns: Optional[List[str]]
+
+ @validator("api_object")
+ def valid_fields(cls, v):
+ if v is None:
+ return ApiObject.INSIGHTS
+ else:
+ return v

gcp_airflow_foundations_facebook/enums/facebook.py

@@ -0,0 +1,31 @@
+from enum import Enum, unique
+
+
+@unique
+class Level(Enum):
+ AD = "ad"
+ ADSET = "adset"
+ CAMPAIGN = "campaign"
+ ACCOUNT = "account"
+
+
+@unique
+class DatePreset(Enum):
+ TODAY = "today"
+ YESTERDAY = "yesterday"
+ THIS_MONTH = "this_month"
+ LAST_MONTH = "last_month"
+ MAXIMUM = "maximum"
+
+
+@unique
+class AccountLookupScope(Enum):
+ ALL = "all"
+ ACTIVE = "active"
+
+
+@unique
+class ApiObject(Enum):
+ INSIGHTS = "INSIGHTS"
+ CAMPAIGNS = "CAMPAIGNS"
+ ADSETS = "ADSETS"