diff --git a/backend/controllers/github.go b/backend/controllers/github.go index b9fa3f491..c4d0195a8 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -519,7 +519,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR return fmt.Errorf("error processing event") } - jobsForImpactedProjects, _, err := dg_github.ConvertGithubPullRequestEventToJobs(payload, impactedProjects, nil, *config) + jobsForImpactedProjects, _, err := dg_github.ConvertGithubPullRequestEventToJobs(payload, impactedProjects, nil, *config, false) if err != nil { log.Printf("Error converting event to jobsForImpactedProjects: %v", err) utils.InitCommentReporter(ghService, prNumber, fmt.Sprintf(":x: Error converting event to jobsForImpactedProjects: %v", err)) diff --git a/backend/services/spec.go b/backend/services/spec.go index cfeeee720..958b11890 100644 --- a/backend/services/spec.go +++ b/backend/services/spec.go @@ -7,9 +7,11 @@ import ( "github.com/diggerhq/digger/backend/utils" "github.com/diggerhq/digger/libs/scheduler" "github.com/diggerhq/digger/libs/spec" + "github.com/samber/lo" "log" "os" "strconv" + "strings" ) func GetVCSTokenFromJob(job models.DiggerJob, gh utils.GithubClientProvider) (*string, error) { @@ -57,6 +59,30 @@ func GetRunNameFromJob(job models.DiggerJob) (*string, error) { return &runName, nil } +func getVariablesSpecFromEnvMap(envVars map[string]string) []spec.VariableSpec { + variablesSpec := make([]spec.VariableSpec, 0) + for k, v := range envVars { + if strings.HasPrefix(v, "$DIGGER_") { + val := strings.ReplaceAll(v, "$DIGGER_", "") + variablesSpec = append(variablesSpec, spec.VariableSpec{ + Name: k, + Value: val, + IsSecret: false, + IsInterpolated: true, + }) + } else { + variablesSpec = append(variablesSpec, spec.VariableSpec{ + Name: k, + Value: v, + IsSecret: false, + IsInterpolated: false, + }) + + } + } + return variablesSpec +} + func GetSpecFromJob(job models.DiggerJob) (*spec.Spec, error) { var jobSpec scheduler.JobJson err := json.Unmarshal([]byte(job.SerializedJobSpec), &jobSpec) @@ -65,6 +91,23 @@ func GetSpecFromJob(job models.DiggerJob) (*spec.Spec, error) { return nil, fmt.Errorf("could not marshal json string: %v", err) } + variablesSpec := make([]spec.VariableSpec, 0) + stateVariables := getVariablesSpecFromEnvMap(jobSpec.StateEnvVars) + commandVariables := getVariablesSpecFromEnvMap(jobSpec.CommandEnvVars) + runVariables := getVariablesSpecFromEnvMap(jobSpec.RunEnvVars) + variablesSpec = append(variablesSpec, stateVariables...) + variablesSpec = append(variablesSpec, commandVariables...) + variablesSpec = append(variablesSpec, runVariables...) + + // check for duplicates in list of variablesSpec + justNames := lo.Map(variablesSpec, func(item spec.VariableSpec, i int) string { + return item.Name + }) + hasDuplicates := len(justNames) != len(lo.Uniq(justNames)) + if hasDuplicates { + return nil, fmt.Errorf("could not load variables due to duplicates: %v", err) + } + batch := job.Batch spec := spec.Spec{ @@ -93,6 +136,7 @@ func GetSpecFromJob(job models.DiggerJob) (*spec.Spec, error) { RepoName: batch.RepoName, WorkflowFile: job.WorkflowFile, }, + Variables: variablesSpec, Policy: spec.PolicySpec{ PolicyType: "http", }, diff --git a/cli/cmd/digger/default.go b/cli/cmd/digger/default.go index 61296a7aa..92a7665f4 100644 --- a/cli/cmd/digger/default.go +++ b/cli/cmd/digger/default.go @@ -54,6 +54,7 @@ var defaultCmd = &cobra.Command{ lib_spec.BackendApiProvider{}, lib_spec.BasicPolicyProvider{}, lib_spec.PlanStorageProvider{}, + lib_spec.VariablesProvider{}, comment_updater.CommentUpdaterProviderBasic{}, ) } diff --git a/cli/cmd/digger/main_test.go b/cli/cmd/digger/main_test.go index 8920ce810..6b3e5352f 100644 --- a/cli/cmd/digger/main_test.go +++ b/cli/cmd/digger/main_test.go @@ -900,7 +900,7 @@ func TestGitHubNewPullRequestContext(t *testing.T) { } event := context.Event.(github.PullRequestEvent) - jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig) + jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig, false) _, _, err = digger.RunJobs(jobs, &prManager, prManager, lock, reporter, &planStorage, policyChecker, comment_updater.NoopCommentUpdater{}, backendApi, "123", false, false, "1", "dir") assert.NoError(t, err) @@ -995,7 +995,7 @@ func TestGitHubNewPullRequestInMultiEnvProjectContext(t *testing.T) { impactedProjects, requestedProject, prNumber, err := dggithub.ProcessGitHubEvent(ghEvent, &diggerConfig, prManager) assert.NoError(t, err) event := context.Event.(github.PullRequestEvent) - jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig) + jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig, false) spew.Dump(lock.MapLock) assert.Equal(t, pullRequestNumber, prNumber) assert.Equal(t, 1, len(jobs)) diff --git a/cli/cmd/digger/run_spec.go b/cli/cmd/digger/run_spec.go index fb70897b8..f6988da36 100644 --- a/cli/cmd/digger/run_spec.go +++ b/cli/cmd/digger/run_spec.go @@ -40,6 +40,7 @@ var runSpecCmd = &cobra.Command{ lib_spec.BackendApiProvider{}, lib_spec.BasicPolicyProvider{}, lib_spec.PlanStorageProvider{}, + lib_spec.VariablesProvider{}, comment_summary.CommentUpdaterProviderBasic{}, ) if err != nil { diff --git a/cli/pkg/github/github.go b/cli/pkg/github/github.go index 9f90ce848..744cc6a0b 100644 --- a/cli/pkg/github/github.go +++ b/cli/pkg/github/github.go @@ -135,7 +135,7 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh } workflow := diggerConfig.Workflows[projectConfig.Workflow] - stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars) + stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, true) planStorage, err := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, nil) if err != nil { @@ -170,7 +170,7 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh } workflow := diggerConfig.Workflows[projectConfig.Workflow] - stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars) + stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, true) StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(projectConfig) @@ -234,7 +234,7 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh coversAllImpactedProjects := false err = nil if prEvent, ok := ghEvent.(github.PullRequestEvent); ok { - jobs, coversAllImpactedProjects, err = dg_github.ConvertGithubPullRequestEventToJobs(&prEvent, impactedProjects, requestedProject, *diggerConfig) + jobs, coversAllImpactedProjects, err = dg_github.ConvertGithubPullRequestEventToJobs(&prEvent, impactedProjects, requestedProject, *diggerConfig, true) } else if commentEvent, ok := ghEvent.(github.IssueCommentEvent); ok { prBranchName, _, err := githubPrService.GetBranchName(*commentEvent.Issue.Number) diff --git a/cli/pkg/integration/integration_test.go b/cli/pkg/integration/integration_test.go index 754f92777..9c45bf4a2 100644 --- a/cli/pkg/integration/integration_test.go +++ b/cli/pkg/integration/integration_test.go @@ -396,7 +396,7 @@ func TestHappyPath(t *testing.T) { impactedProjects, requestedProject, prNumber, err := dg_github.ProcessGitHubEvent(ghEvent, diggerConfig, &githubPrService) assert.NoError(t, err) event := ghEvent.(github.PullRequestEvent) - jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, *diggerConfig) + jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, *diggerConfig, false) assert.NoError(t, err) zipManager := storage.Zipper{} planStorage := &storage.GithubPlanStorage{ @@ -439,7 +439,7 @@ func TestHappyPath(t *testing.T) { impactedProjects, requestedProject, prNumber, err = dg_github.ProcessGitHubEvent(ghEvent, diggerConfig, &githubPrService) assert.NoError(t, err) prEvent := ghEvent.(github.PullRequestEvent) - jobs, _, err = dg_github.ConvertGithubPullRequestEventToJobs(&prEvent, impactedProjects, requestedProject, *diggerConfig) + jobs, _, err = dg_github.ConvertGithubPullRequestEventToJobs(&prEvent, impactedProjects, requestedProject, *diggerConfig, false) assert.NoError(t, err) _, _, err = digger.RunJobs(jobs, &githubPrService, &githubPrService, lock, reporter, planStorage, nil, comment_updater.NoopCommentUpdater{}, nil, "", false, false, "123", dir) assert.NoError(t, err) @@ -551,7 +551,7 @@ func TestMultiEnvHappyPath(t *testing.T) { impactedProjects, requestedProject, prNumber, err := dg_github.ProcessGitHubEvent(ghEvent, diggerConfig, &githubPrService) assert.NoError(t, err) pEvent := ghEvent.(github.PullRequestEvent) - jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&pEvent, impactedProjects, requestedProject, *diggerConfig) + jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&pEvent, impactedProjects, requestedProject, *diggerConfig, false) assert.NoError(t, err) zipManager := storage.Zipper{} @@ -769,7 +769,7 @@ workflows: impactedProjects, requestedProject, prNumber, err := dg_github.ProcessGitHubEvent(ghEvent, diggerConfig, &githubPrService) assert.NoError(t, err) pEvent := ghEvent.(github.PullRequestEvent) - jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&pEvent, impactedProjects, requestedProject, *diggerConfig) + jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&pEvent, impactedProjects, requestedProject, *diggerConfig, false) assert.NoError(t, err) zipManager := storage.Zipper{} diff --git a/cli/pkg/spec/spec.go b/cli/pkg/spec/spec.go index 5048b74e1..3e5e78618 100644 --- a/cli/pkg/spec/spec.go +++ b/cli/pkg/spec/spec.go @@ -33,6 +33,7 @@ func RunSpec( backedProvider spec.BackendApiProvider, policyProvider spec.SpecPolicyProvider, PlanStorageProvider spec.PlanStorageProvider, + variablesProvider spec.VariablesProvider, commentUpdaterProvider comment_summary.CommentUpdaterProvider, ) error { @@ -78,14 +79,6 @@ func RunSpec( reportError(spec, backendApi, message, err) } - // TODO: avoid calling GetChangedFilesHere, avoid loading digger config entirely - // also see below TODO to leverage variables provider and avoid passing it to commentUpdaterProvider - diggerConfig, _, _, err := digger_config.LoadDiggerConfig("./", false, []string{}) - if err != nil { - usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("Failed to read Digger digger_config. %s", err), 4) - } - log.Printf("Digger digger_config read successfully\n") - // TODO: render mode being passable from the string commentUpdater, err := commentUpdaterProvider.Get(digger_config.CommentRenderModeBasic) if err != nil { @@ -100,10 +93,17 @@ func RunSpec( } // TODO: make this part purely based on variables providers - workflow := diggerConfig.Workflows[job.ProjectWorkflow] - stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars) - job.StateEnvVars = lo.Assign(job.StateEnvVars, stateEnvVars) - job.CommandEnvVars = lo.Assign(job.CommandEnvVars, commandEnvVars) + + // get variables from the variables spec + variablesMap, err := variablesProvider.GetVariables(spec.Variables) + if err != nil { + log.Printf("could not get variables from provider: %v", err) + reporterError(spec, backendApi, err) + usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get variables from provider: %v", err), 1) + } + job.StateEnvVars = lo.Assign(job.StateEnvVars, variablesMap) + job.CommandEnvVars = lo.Assign(job.CommandEnvVars, variablesMap) + job.RunEnvVars = lo.Assign(job.RunEnvVars, variablesMap) jobs := []scheduler.Job{job} diff --git a/ee/cli/cmd/digger/default.go b/ee/cli/cmd/digger/default.go index 32af0cb86..88dc9b4c9 100644 --- a/ee/cli/cmd/digger/default.go +++ b/ee/cli/cmd/digger/default.go @@ -53,6 +53,7 @@ var defaultCmd = &cobra.Command{ lib_spec.BackendApiProvider{}, policy.AdvancedPolicyProvider{}, lib_spec.PlanStorageProvider{}, + lib_spec.VariablesProvider{}, comment_summary.CommentUpdaterProviderBasic{}, ) } diff --git a/ee/cli/cmd/digger/main_test.go b/ee/cli/cmd/digger/main_test.go index 724621ea3..2e8cd98f2 100644 --- a/ee/cli/cmd/digger/main_test.go +++ b/ee/cli/cmd/digger/main_test.go @@ -900,7 +900,7 @@ func TestGitHubNewPullRequestContext(t *testing.T) { } event := context.Event.(github.PullRequestEvent) - jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig) + jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig, false) if err != nil { assert.NoError(t, err) log.Println(err) @@ -1001,7 +1001,7 @@ func TestGitHubNewPullRequestInMultiEnvProjectContext(t *testing.T) { impactedProjects, requestedProject, prNumber, err := dggithub.ProcessGitHubEvent(ghEvent, &diggerConfig, &prManager) assert.NoError(t, err) event := context.Event.(github.PullRequestEvent) - jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig) + jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig, false) assert.Equal(t, pullRequestNumber, prNumber) assert.Equal(t, 1, len(jobs)) diff --git a/ee/cli/cmd/digger/run_spec.go b/ee/cli/cmd/digger/run_spec.go index 3cdd6f5a6..8cc624305 100644 --- a/ee/cli/cmd/digger/run_spec.go +++ b/ee/cli/cmd/digger/run_spec.go @@ -41,6 +41,7 @@ var runSpecCmd = &cobra.Command{ lib_spec.BackendApiProvider{}, policy.AdvancedPolicyProvider{}, lib_spec.PlanStorageProvider{}, + lib_spec.VariablesProvider{}, comment_summary.CommentUpdaterProviderBasic{}, ) if err != nil { diff --git a/libs/ci/azure/azure.go b/libs/ci/azure/azure.go index 8912a04fe..9a64a6442 100644 --- a/libs/ci/azure/azure.go +++ b/libs/ci/azure/azure.go @@ -442,7 +442,7 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []dig } prNumber := parseAzureContext.Event.(AzurePrEvent).Resource.PullRequestId - stateEnvVars, commandEnvVars := digger_config2.CollectTerraformEnvConfig(workflow.EnvVars) + stateEnvVars, commandEnvVars := digger_config2.CollectTerraformEnvConfig(workflow.EnvVars, true) StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(project) jobs = append(jobs, scheduler.Job{ ProjectName: project.Name, @@ -472,7 +472,7 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []dig } prNumber := parseAzureContext.Event.(AzurePrEvent).Resource.PullRequestId - stateEnvVars, commandEnvVars := digger_config2.CollectTerraformEnvConfig(workflow.EnvVars) + stateEnvVars, commandEnvVars := digger_config2.CollectTerraformEnvConfig(workflow.EnvVars, true) StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(project) jobs = append(jobs, scheduler.Job{ ProjectName: project.Name, @@ -502,7 +502,7 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []dig if !ok { return nil, false, fmt.Errorf("failed to find workflow digger_config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := digger_config2.CollectTerraformEnvConfig(workflow.EnvVars) + stateEnvVars, commandEnvVars := digger_config2.CollectTerraformEnvConfig(workflow.EnvVars, true) StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(project) jobs = append(jobs, scheduler.Job{ ProjectName: project.Name, @@ -557,7 +557,7 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []dig if !ok { return nil, false, fmt.Errorf("failed to find workflow digger_config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := digger_config2.CollectTerraformEnvConfig(workflow.EnvVars) + stateEnvVars, commandEnvVars := digger_config2.CollectTerraformEnvConfig(workflow.EnvVars, true) StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(project) jobs = append(jobs, scheduler.Job{ ProjectName: project.Name, diff --git a/libs/ci/generic/events.go b/libs/ci/generic/events.go index c13f1cb8a..a657c1745 100644 --- a/libs/ci/generic/events.go +++ b/libs/ci/generic/events.go @@ -148,7 +148,7 @@ func CreateJobsForProjects(projects []digger_config.Project, command string, eve } runEnvVars := GetRunEnvVars(defaultBranch, prBranch, project.Name, project.Dir) - stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars) + stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, false) StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(project) workspace := project.Workspace jobs = append(jobs, scheduler.Job{ diff --git a/libs/ci/github/github.go b/libs/ci/github/github.go index 9830c404b..1bdf3ca35 100644 --- a/libs/ci/github/github.go +++ b/libs/ci/github/github.go @@ -353,7 +353,7 @@ func (svc GithubService) CheckBranchExists(branchName string) (bool, error) { return true, nil } -func ConvertGithubPullRequestEventToJobs(payload *github.PullRequestEvent, impactedProjects []digger_config.Project, requestedProject *digger_config.Project, config digger_config.DiggerConfig) ([]scheduler.Job, bool, error) { +func ConvertGithubPullRequestEventToJobs(payload *github.PullRequestEvent, impactedProjects []digger_config.Project, requestedProject *digger_config.Project, config digger_config.DiggerConfig, performEnvVarInterpolation bool) ([]scheduler.Job, bool, error) { workflows := config.Workflows jobs := make([]scheduler.Job, 0) @@ -368,7 +368,7 @@ func ConvertGithubPullRequestEventToJobs(payload *github.PullRequestEvent, impac runEnvVars := generic.GetRunEnvVars(defaultBranch, prBranch, project.Name, project.Dir) - stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars) + stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, performEnvVarInterpolation) pullRequestNumber := payload.PullRequest.Number StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(project) diff --git a/libs/ci/gitlab/gitlab.go b/libs/ci/gitlab/gitlab.go index 025e0b850..041386a57 100644 --- a/libs/ci/gitlab/gitlab.go +++ b/libs/ci/gitlab/gitlab.go @@ -357,7 +357,7 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex return nil, true, fmt.Errorf("failed to find workflow digger_config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars) + stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, true) StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(project) jobs = append(jobs, scheduler.Job{ ProjectName: project.Name, @@ -385,7 +385,7 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex if !ok { return nil, true, fmt.Errorf("failed to find workflow digger_config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars) + stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, true) var StateEnvProvider *stscreds.WebIdentityRoleProvider var CommandEnvProvider *stscreds.WebIdentityRoleProvider if project.AwsRoleToAssume != nil { @@ -455,7 +455,7 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex if workspaceOverride != "" { workspace = workspaceOverride } - stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars) + stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, true) StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(project) jobs = append(jobs, scheduler.Job{ ProjectName: project.Name, diff --git a/libs/ci/gitlab/webhooks.go b/libs/ci/gitlab/webhooks.go index 525a2a3d6..6218f4ed3 100644 --- a/libs/ci/gitlab/webhooks.go +++ b/libs/ci/gitlab/webhooks.go @@ -46,7 +46,7 @@ func ConvertGithubPullRequestEventToJobs(payload *gitlab.MergeEvent, impactedPro runEnvVars := generic.GetRunEnvVars(defaultBranch, prBranch, project.Name, project.Dir) - stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars) + stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, false) pullRequestNumber := payload.ObjectAttributes.IID namespace := payload.Project.PathWithNamespace sender := payload.User.Username diff --git a/libs/digger_config/digger_config.go b/libs/digger_config/digger_config.go index 2617477df..ef63ec387 100644 --- a/libs/digger_config/digger_config.go +++ b/libs/digger_config/digger_config.go @@ -688,7 +688,7 @@ func retrieveConfigFile(workingDir string) (string, error) { return "", nil } -func CollectTerraformEnvConfig(envs *TerraformEnvConfig) (map[string]string, map[string]string) { +func CollectTerraformEnvConfig(envs *TerraformEnvConfig, performInterpolation bool) (map[string]string, map[string]string) { stateEnvVars := map[string]string{} commandEnvVars := map[string]string{} @@ -697,7 +697,11 @@ func CollectTerraformEnvConfig(envs *TerraformEnvConfig) (map[string]string, map if envvar.Value != "" { stateEnvVars[envvar.Name] = envvar.Value } else if envvar.ValueFrom != "" { - stateEnvVars[envvar.Name] = os.Getenv(envvar.ValueFrom) + if performInterpolation { + stateEnvVars[envvar.Name] = os.Getenv(envvar.ValueFrom) + } else { + stateEnvVars[envvar.Name] = fmt.Sprintf("$DIGGER_%v", envvar.ValueFrom) + } } } @@ -705,7 +709,11 @@ func CollectTerraformEnvConfig(envs *TerraformEnvConfig) (map[string]string, map if envvar.Value != "" { commandEnvVars[envvar.Name] = envvar.Value } else if envvar.ValueFrom != "" { - commandEnvVars[envvar.Name] = os.Getenv(envvar.ValueFrom) + if performInterpolation { + commandEnvVars[envvar.Name] = os.Getenv(envvar.ValueFrom) + } else { + commandEnvVars[envvar.Name] = fmt.Sprintf("$DIGGER_%v", envvar.ValueFrom) + } } } } diff --git a/libs/scheduler/convert.go b/libs/scheduler/convert.go index 1b2e8c7da..77de4e7c0 100644 --- a/libs/scheduler/convert.go +++ b/libs/scheduler/convert.go @@ -16,7 +16,7 @@ func ConvertProjectsToJobs(actor string, repoNamespace string, command string, p return nil, true, fmt.Errorf("failed to find workflow digger_config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars) + stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, false) StateEnvProvider, CommandEnvProvider := GetStateAndCommandProviders(project) jobs = append(jobs, Job{ ProjectName: project.Name, diff --git a/libs/spec/models.go b/libs/spec/models.go index 10db7aa1a..1c2dfb107 100644 --- a/libs/spec/models.go +++ b/libs/spec/models.go @@ -37,9 +37,10 @@ type PolicySpec struct { } type VariableSpec struct { - Name string `json:"name"` - Value string `json:"value"` - IsSecret bool `json:"is_secret"` + Name string `json:"name"` + Value string `json:"value"` + IsSecret bool `json:"is_secret"` + IsInterpolated bool `json:"is_interpolated"` } type SpecType string diff --git a/libs/spec/variables_provider.go b/libs/spec/variables_provider.go index 3d4275348..8742d0024 100644 --- a/libs/spec/variables_provider.go +++ b/libs/spec/variables_provider.go @@ -27,6 +27,9 @@ func (p VariablesProvider) GetVariables(variables []VariableSpec) (map[string]st return nil, fmt.Errorf("could not decrypt value using private key: %v", err) } res[v.Name] = string(value) + } else if v.IsInterpolated { + // if it is an interpolated value we get it form env variable of the variable + res[v.Name] = os.Getenv(v.Value) } else { res[v.Name] = v.Value } diff --git a/next/controllers/github.go b/next/controllers/github.go index 2e650b1ec..5d00b19c4 100644 --- a/next/controllers/github.go +++ b/next/controllers/github.go @@ -387,7 +387,7 @@ func handlePullRequestEvent(gh next_utils.GithubClientProvider, payload *github. return fmt.Errorf("error processing event") } - jobsForImpactedProjects, _, err := dg_github.ConvertGithubPullRequestEventToJobs(payload, impactedProjects, nil, *config) + jobsForImpactedProjects, _, err := dg_github.ConvertGithubPullRequestEventToJobs(payload, impactedProjects, nil, *config, false) if err != nil { log.Printf("Error converting event to jobsForImpactedProjects: %v", err) backend_utils.InitCommentReporter(ghService, prNumber, fmt.Sprintf(":x: Error converting event to jobsForImpactedProjects: %v", err))