dbt: access_url, fix dbt case

estuary · Sep 20, 2024 · 4ce7f9c · 4ce7f9c
1 parent 5985e03
commit 4ce7f9c
Show file tree

Hide file tree

Showing 15 changed files with 77 additions and 81 deletions.
diff --git a/go/dbt/trigger.go b/go/dbt/trigger.go
@@ -12,10 +12,11 @@ import (
 )
 
 type JobConfig struct {
-	JobID         string `json:"job_id" jsonschema:"title=Job ID,description=DBT job ID"`
-	AccountID     string `json:"account_id" jsonschema:"title=Account ID,description=DBT account ID"`
-	AccountPrefix string `json:"account_prefix" jsonschema:"title=Account Prefix,description=DBT account prefix"`
-	APIKey        string `json:"api_key" jsonschema:"title=API Key,description=DBT API Key" jsonschema_extras:"secret=true"`
+	JobID         string `json:"job_id" jsonschema:"title=Job ID,description=dbt job ID"`
+	AccountID     string `json:"account_id" jsonschema:"title=Account ID,description=dbt account ID"`
+	AccessURL     string `json:"access_url,omitempty" jsonschema:"title=Access URL,description=dbt access URL can be found in your Account Settings. See go.estuary.dev/dbt-cloud-trigger"`
+	AccountPrefix string `json:"account_prefix" jsonschema:"-"`
+	APIKey        string `json:"api_key" jsonschema:"title=API Key,description=dbt API Key" jsonschema_extras:"secret=true"`
 	Cause         string `json:"cause,omitempty" jsonschema:"title=Cause Message,description=You can set a custom 'cause' message for the job trigger. Defaults to 'Estuary Flow'."`
 	Mode          string `json:"mode,omitempty" jsonschema:"title=Job Trigger Mode,description=Specifies how should already-running jobs be treated. Defaults to 'skip' which skips the trigger if a job is already running; 'replace' cancels the running job and runs a new one; while 'ignore' triggers a new job regardless of existing jobs.,enum=skip,enum=replace,enum=ignore,default=skip"`
 }
@@ -28,7 +29,6 @@ func (c *JobConfig) Validate() error {
 	var requiredProperties = [][]string{
 		{"job_id", c.JobID},
 		{"account_id", c.AccountID},
-		{"account_prefix", c.AccountPrefix},
 		{"api_key", c.APIKey},
 	}
 	for _, req := range requiredProperties {
@@ -37,6 +37,10 @@ func (c *JobConfig) Validate() error {
 		}
 	}
 
+	if c.AccountPrefix == "" && c.AccessURL == "" {
+		return fmt.Errorf("dbt job trigger missing access_url")
+	}
+
 	return nil
 }
 
@@ -47,19 +51,18 @@ const (
 )
 
 func (c *JobConfig) Enabled() bool {
-	return c.JobID != "" && c.AccountID != "" && c.AccountPrefix != "" && c.APIKey != ""
+	return c.JobID != "" && c.AccountID != "" && (c.AccountPrefix != "" || c.AccessURL != "") && c.APIKey != ""
 }
 
 // See https://docs.getdbt.com/dbt-cloud/api-v2 for more details
 // on different hostnames
 func (config *JobConfig) host() string {
-	var host = fmt.Sprintf("%s.us1.dbt.com", config.AccountPrefix)
-	if config.AccountPrefix == "emea" {
-		host = "emea.dbt.com"
-	} else if config.AccountPrefix == "au" {
-		host = "au.dbt.com"
+	if config.AccessURL != "" {
+		return config.AccessURL
 	}
 
+	var host = fmt.Sprintf("%s.us1.dbt.com", config.AccountPrefix)
+
 	return host
 }
 

diff --git a/materialize-bigquery/.snapshots/TestSpecification b/materialize-bigquery/.snapshots/TestSpecification
@@ -109,22 +109,22 @@
           "job_id": {
             "type": "string",
             "title": "Job ID",
-            "description": "DBT job ID"
+            "description": "dbt job ID"
           },
           "account_id": {
             "type": "string",
             "title": "Account ID",
-            "description": "DBT account ID"
+            "description": "dbt account ID"
           },
-          "account_prefix": {
+          "access_url": {
             "type": "string",
-            "title": "Account Prefix",
-            "description": "DBT account prefix"
+            "title": "Access URL",
+            "description": "dbt access URL can be found in your Account Settings. See go.estuary.dev/dbt-cloud-trigger"
           },
           "api_key": {
             "type": "string",
             "title": "API Key",
-            "description": "DBT API Key",
+            "description": "dbt API Key",
             "secret": true
           },
           "cause": {
@@ -149,11 +149,10 @@
         "required": [
           "job_id",
           "account_id",
-          "account_prefix",
           "api_key"
         ],
-        "title": "DBT Job Trigger",
-        "description": "Trigger a DBT Job when new data is available"
+        "title": "dbt Cloud Job Trigger",
+        "description": "Trigger a dbt job when new data is available"
       }
     },
     "type": "object",

diff --git a/materialize-bigquery/bigquery.go b/materialize-bigquery/bigquery.go
@@ -27,7 +27,7 @@ type config struct {
 	BillingProjectID string                     `json:"billing_project_id,omitempty" jsonschema:"title=Billing Project ID,description=Billing Project ID connected to the BigQuery dataset. Defaults to Project ID if not specified." jsonschema_extras:"order=6"`
 	HardDelete       bool                       `json:"hardDelete,omitempty" jsonschema:"title=Hard Delete,description=If this option is enabled items deleted in the source will also be deleted from the destination. By default is disabled and _meta/op in the destination will signify whether rows have been deleted (soft-delete).,default=false" jsonschema_extras:"order=7"`
 	Schedule         boilerplate.ScheduleConfig `json:"syncSchedule,omitempty" jsonschema:"title=Sync Schedule,description=Configure schedule of transactions for the materialization."`
-	DBTJobTrigger    dbt.JobConfig              `json:"dbt_job_trigger,omitempty" jsonschema:"title=DBT Job Trigger,description=Trigger a DBT Job when new data is available"`
+	DBTJobTrigger    dbt.JobConfig              `json:"dbt_job_trigger,omitempty" jsonschema:"title=dbt Cloud Job Trigger,description=Trigger a dbt job when new data is available"`
 }
 
 func (c *config) Validate() error {

diff --git a/materialize-databricks/.snapshots/TestSpecification b/materialize-databricks/.snapshots/TestSpecification
@@ -123,22 +123,22 @@
           "job_id": {
             "type": "string",
             "title": "Job ID",
-            "description": "DBT job ID"
+            "description": "dbt job ID"
           },
           "account_id": {
             "type": "string",
             "title": "Account ID",
-            "description": "DBT account ID"
+            "description": "dbt account ID"
           },
-          "account_prefix": {
+          "access_url": {
             "type": "string",
-            "title": "Account Prefix",
-            "description": "DBT account prefix"
+            "title": "Access URL",
+            "description": "dbt access URL can be found in your Account Settings. See go.estuary.dev/dbt-cloud-trigger"
           },
           "api_key": {
             "type": "string",
             "title": "API Key",
-            "description": "DBT API Key",
+            "description": "dbt API Key",
             "secret": true
           },
           "cause": {
@@ -163,11 +163,10 @@
         "required": [
           "job_id",
           "account_id",
-          "account_prefix",
           "api_key"
         ],
-        "title": "DBT Job Trigger",
-        "description": "Trigger a DBT Job when new data is available"
+        "title": "dbt Cloud Job Trigger",
+        "description": "Trigger a dbt Job when new data is available"
       }
     },
     "type": "object",

diff --git a/materialize-databricks/config.go b/materialize-databricks/config.go
@@ -20,7 +20,7 @@ type config struct {
 	HardDelete    bool                       `json:"hardDelete,omitempty" jsonschema:"title=Hard Delete,description=If this option is enabled items deleted in the source will also be deleted from the destination. By default is disabled and _meta/op in the destination will signify whether rows have been deleted (soft-delete).,default=false" jsonschema_extras:"order=4"`
 	Credentials   credentialConfig           `json:"credentials" jsonschema:"title=Authentication" jsonschema_extras:"order=5"`
 	Schedule      boilerplate.ScheduleConfig `json:"syncSchedule,omitempty" jsonschema:"title=Sync Schedule,description=Configure schedule of transactions for the materialization."`
-	DBTJobTrigger dbt.JobConfig              `json:"dbt_job_trigger,omitempty" jsonschema:"title=DBT Job Trigger,description=Trigger a DBT Job when new data is available"`
+	DBTJobTrigger dbt.JobConfig              `json:"dbt_job_trigger,omitempty" jsonschema:"title=dbt Cloud Job Trigger,description=Trigger a dbt Job when new data is available"`
 }
 
 const (

diff --git a/materialize-mysql/.snapshots/TestSpecification b/materialize-mysql/.snapshots/TestSpecification
@@ -46,22 +46,22 @@
           "job_id": {
             "type": "string",
             "title": "Job ID",
-            "description": "DBT job ID"
+            "description": "dbt job ID"
           },
           "account_id": {
             "type": "string",
             "title": "Account ID",
-            "description": "DBT account ID"
+            "description": "dbt account ID"
           },
-          "account_prefix": {
+          "access_url": {
             "type": "string",
-            "title": "Account Prefix",
-            "description": "DBT account prefix"
+            "title": "Access URL",
+            "description": "dbt access URL can be found in your Account Settings. See go.estuary.dev/dbt-cloud-trigger"
           },
           "api_key": {
             "type": "string",
             "title": "API Key",
-            "description": "DBT API Key",
+            "description": "dbt API Key",
             "secret": true
           },
           "cause": {
@@ -86,11 +86,10 @@
         "required": [
           "job_id",
           "account_id",
-          "account_prefix",
           "api_key"
         ],
-        "title": "DBT Job Trigger",
-        "description": "Trigger a DBT Job when new data is available"
+        "title": "dbt Cloud Job Trigger",
+        "description": "Trigger a dbt Job when new data is available"
       },
       "advanced": {
         "properties": {

diff --git a/materialize-mysql/driver.go b/materialize-mysql/driver.go
@@ -45,7 +45,7 @@ type config struct {
 	Timezone   string `json:"timezone,omitempty" jsonschema:"title=Timezone,description=Timezone to use when materializing datetime columns. Should normally be left blank to use the database's 'time_zone' system variable. Only required if the 'time_zone' system variable cannot be read. Must be a valid IANA time zone name or +HH:MM offset. Takes precedence over the 'time_zone' system variable if both are set." jsonschema_extras:"order=4"`
 	HardDelete bool   `json:"hardDelete,omitempty" jsonschema:"title=Hard Delete,description=If this option is enabled items deleted in the source will also be deleted from the destination. By default is disabled and _meta/op in the destination will signify whether rows have been deleted (soft-delete).,default=false" jsonschema_extras:"order=5"`
 
-	DBTJobTrigger dbt.JobConfig `json:"dbt_job_trigger,omitempty" jsonschema:"title=DBT Job Trigger,description=Trigger a DBT Job when new data is available"`
+	DBTJobTrigger dbt.JobConfig `json:"dbt_job_trigger,omitempty" jsonschema:"title=dbt Cloud Job Trigger,description=Trigger a dbt Job when new data is available"`
 
 	Advanced advancedConfig `json:"advanced,omitempty" jsonschema:"title=Advanced Options,description=Options for advanced users. You should not typically need to modify these." jsonschema_extras:"advanced=true"`
 

diff --git a/materialize-postgres/.snapshots/TestSpecification b/materialize-postgres/.snapshots/TestSpecification
@@ -47,22 +47,22 @@
           "job_id": {
             "type": "string",
             "title": "Job ID",
-            "description": "DBT job ID"
+            "description": "dbt job ID"
           },
           "account_id": {
             "type": "string",
             "title": "Account ID",
-            "description": "DBT account ID"
+            "description": "dbt account ID"
           },
-          "account_prefix": {
+          "access_url": {
             "type": "string",
-            "title": "Account Prefix",
-            "description": "DBT account prefix"
+            "title": "Access URL",
+            "description": "dbt access URL can be found in your Account Settings. See go.estuary.dev/dbt-cloud-trigger"
           },
           "api_key": {
             "type": "string",
             "title": "API Key",
-            "description": "DBT API Key",
+            "description": "dbt API Key",
             "secret": true
           },
           "cause": {
@@ -87,11 +87,10 @@
         "required": [
           "job_id",
           "account_id",
-          "account_prefix",
           "api_key"
         ],
-        "title": "DBT Job Trigger",
-        "description": "Trigger a DBT Job when new data is available"
+        "title": "dbt Cloud Job Trigger",
+        "description": "Trigger a dbt Job when new data is available"
       },
       "advanced": {
         "properties": {

diff --git a/materialize-postgres/driver.go b/materialize-postgres/driver.go
@@ -51,7 +51,7 @@ type config struct {
 	Schema     string `json:"schema,omitempty" jsonschema:"title=Database Schema,default=public,description=Database schema for bound collection tables (unless overridden within the binding resource configuration) as well as associated materialization metadata tables" jsonschema_extras:"order=4"`
 	HardDelete bool   `json:"hardDelete,omitempty" jsonschema:"title=Hard Delete,description=If this option is enabled items deleted in the source will also be deleted from the destination. By default is disabled and _meta/op in the destination will signify whether rows have been deleted (soft-delete).,default=false" jsonschema_extras:"order=5"`
 
-	DBTJobTrigger dbt.JobConfig `json:"dbt_job_trigger,omitempty" jsonschema:"title=DBT Job Trigger,description=Trigger a DBT Job when new data is available"`
+	DBTJobTrigger dbt.JobConfig `json:"dbt_job_trigger,omitempty" jsonschema:"title=dbt Cloud Job Trigger,description=Trigger a dbt Job when new data is available"`
 
 	Advanced advancedConfig `json:"advanced,omitempty" jsonschema:"title=Advanced Options,description=Options for advanced users. You should not typically need to modify these." jsonschema_extras:"advanced=true"`
 

diff --git a/materialize-redshift/.snapshots/TestSpecification b/materialize-redshift/.snapshots/TestSpecification
@@ -128,22 +128,22 @@
           "job_id": {
             "type": "string",
             "title": "Job ID",
-            "description": "DBT job ID"
+            "description": "dbt job ID"
           },
           "account_id": {
             "type": "string",
             "title": "Account ID",
-            "description": "DBT account ID"
+            "description": "dbt account ID"
           },
-          "account_prefix": {
+          "access_url": {
             "type": "string",
-            "title": "Account Prefix",
-            "description": "DBT account prefix"
+            "title": "Access URL",
+            "description": "dbt access URL can be found in your Account Settings. See go.estuary.dev/dbt-cloud-trigger"
           },
           "api_key": {
             "type": "string",
             "title": "API Key",
-            "description": "DBT API Key",
+            "description": "dbt API Key",
             "secret": true
           },
           "cause": {
@@ -168,11 +168,10 @@
         "required": [
           "job_id",
           "account_id",
-          "account_prefix",
           "api_key"
         ],
-        "title": "DBT Job Trigger",
-        "description": "Trigger a DBT Job when new data is available"
+        "title": "dbt Cloud Job Trigger",
+        "description": "Trigger a dbt Job when new data is available"
       },
       "networkTunnel": {
         "properties": {

diff --git a/materialize-redshift/driver.go b/materialize-redshift/driver.go
@@ -75,7 +75,7 @@ type config struct {
 	BucketPath         string                     `json:"bucketPath,omitempty" jsonschema:"title=Bucket Path,description=A prefix that will be used to store objects in S3." jsonschema_extras:"order=9"`
 	HardDelete         bool                       `json:"hardDelete,omitempty" jsonschema:"title=Hard Delete,description=If this option is enabled items deleted in the source will also be deleted from the destination. By default is disabled and _meta/op in the destination will signify whether rows have been deleted (soft-delete).,default=false" jsonschema_extras:"order=10"`
 	Schedule           boilerplate.ScheduleConfig `json:"syncSchedule,omitempty" jsonschema:"title=Sync Schedule,description=Configure schedule of transactions for the materialization."`
-	DBTJobTrigger      dbt.JobConfig              `json:"dbt_job_trigger,omitempty" jsonschema:"title=DBT Job Trigger,description=Trigger a DBT Job when new data is available"`
+	DBTJobTrigger      dbt.JobConfig              `json:"dbt_job_trigger,omitempty" jsonschema:"title=dbt Cloud Job Trigger,description=Trigger a dbt Job when new data is available"`
 	NetworkTunnel      *tunnelConfig              `json:"networkTunnel,omitempty" jsonschema:"title=Network Tunnel,description=Connect to your Redshift cluster through an SSH server that acts as a bastion host for your network."`
 }
 

diff --git a/materialize-snowflake/.snapshots/TestSpecification b/materialize-snowflake/.snapshots/TestSpecification
@@ -170,22 +170,22 @@
           "job_id": {
             "type": "string",
             "title": "Job ID",
-            "description": "DBT job ID"
+            "description": "dbt job ID"
           },
           "account_id": {
             "type": "string",
             "title": "Account ID",
-            "description": "DBT account ID"
+            "description": "dbt account ID"
           },
-          "account_prefix": {
+          "access_url": {
             "type": "string",
-            "title": "Account Prefix",
-            "description": "DBT account prefix"
+            "title": "Access URL",
+            "description": "dbt access URL can be found in your Account Settings. See go.estuary.dev/dbt-cloud-trigger"
           },
           "api_key": {
             "type": "string",
             "title": "API Key",
-            "description": "DBT API Key",
+            "description": "dbt API Key",
             "secret": true
           },
           "cause": {
@@ -210,11 +210,10 @@
         "required": [
           "job_id",
           "account_id",
-          "account_prefix",
           "api_key"
         ],
-        "title": "DBT Job Trigger",
-        "description": "Trigger a DBT Job when new data is available"
+        "title": "dbt Cloud Job Trigger",
+        "description": "Trigger a dbt Job when new data is available"
       }
     },
     "type": "object",

diff --git a/materialize-snowflake/config.go b/materialize-snowflake/config.go
@@ -30,7 +30,7 @@ type config struct {
 	HardDelete    bool                       `json:"hardDelete,omitempty" jsonschema:"title=Hard Delete,description=If this option is enabled items deleted in the source will also be deleted from the destination. By default is disabled and _meta/op in the destination will signify whether rows have been deleted (soft-delete).,default=false" jsonschema_extras:"order=8"`
 	Credentials   credentialConfig           `json:"credentials" jsonschema:"title=Authentication"`
 	Schedule      boilerplate.ScheduleConfig `json:"syncSchedule,omitempty" jsonschema:"title=Sync Schedule,description=Configure schedule of transactions for the materialization."`
-	DBTJobTrigger dbt.JobConfig              `json:"dbt_job_trigger,omitempty" jsonschema:"title=DBT Job Trigger,description=Trigger a DBT Job when new data is available"`
+	DBTJobTrigger dbt.JobConfig              `json:"dbt_job_trigger,omitempty" jsonschema:"title=dbt Cloud Job Trigger,description=Trigger a dbt Job when new data is available"`
 }
 
 // toURI manually builds the DSN connection string.