rely on variables spec within orchestrator, not on workflow config (#…

…1681)

* rely purely on env spec

* fix json for spec

backend/controllers/github.go

@@ -519,7 +519,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
  return fmt.Errorf("error processing event")
  }
 
- jobsForImpactedProjects, _, err := dg_github.ConvertGithubPullRequestEventToJobs(payload, impactedProjects, nil, *config)
+ jobsForImpactedProjects, _, err := dg_github.ConvertGithubPullRequestEventToJobs(payload, impactedProjects, nil, *config, false)
  if err != nil {
  log.Printf("Error converting event to jobsForImpactedProjects: %v", err)
  utils.InitCommentReporter(ghService, prNumber, fmt.Sprintf(":x: Error converting event to jobsForImpactedProjects: %v", err))

backend/services/spec.go

@@ -7,9 +7,11 @@ import (
  "github.com/diggerhq/digger/backend/utils"
  "github.com/diggerhq/digger/libs/scheduler"
  "github.com/diggerhq/digger/libs/spec"
+ "github.com/samber/lo"
  "log"
  "os"
  "strconv"
+ "strings"
 )
 
 func GetVCSTokenFromJob(job models.DiggerJob, gh utils.GithubClientProvider) (*string, error) {
@@ -57,6 +59,30 @@ func GetRunNameFromJob(job models.DiggerJob) (*string, error) {
  return &runName, nil
 }
 
+func getVariablesSpecFromEnvMap(envVars map[string]string) []spec.VariableSpec {
+ variablesSpec := make([]spec.VariableSpec, 0)
+ for k, v := range envVars {
+ if strings.HasPrefix(v, "$DIGGER_") {
+ val := strings.ReplaceAll(v, "$DIGGER_", "")
+ variablesSpec = append(variablesSpec, spec.VariableSpec{
+ Name: k,
+ Value: val,
+ IsSecret: false,
+ IsInterpolated: true,
+ })
+ } else {
+ variablesSpec = append(variablesSpec, spec.VariableSpec{
+ Name: k,
+ Value: v,
+ IsSecret: false,
+ IsInterpolated: false,
+ })
+
+ }
+ }
+ return variablesSpec
+}
+
 func GetSpecFromJob(job models.DiggerJob) (*spec.Spec, error) {
  var jobSpec scheduler.JobJson
  err := json.Unmarshal([]byte(job.SerializedJobSpec), &jobSpec)
@@ -65,6 +91,23 @@ func GetSpecFromJob(job models.DiggerJob) (*spec.Spec, error) {
  return nil, fmt.Errorf("could not marshal json string: %v", err)
  }
 
+ variablesSpec := make([]spec.VariableSpec, 0)
+ stateVariables := getVariablesSpecFromEnvMap(jobSpec.StateEnvVars)
+ commandVariables := getVariablesSpecFromEnvMap(jobSpec.CommandEnvVars)
+ runVariables := getVariablesSpecFromEnvMap(jobSpec.RunEnvVars)
+ variablesSpec = append(variablesSpec, stateVariables...)
+ variablesSpec = append(variablesSpec, commandVariables...)
+ variablesSpec = append(variablesSpec, runVariables...)
+
+ // check for duplicates in list of variablesSpec
+ justNames := lo.Map(variablesSpec, func(item spec.VariableSpec, i int) string {
+ return item.Name
+ })
+ hasDuplicates := len(justNames) != len(lo.Uniq(justNames))
+ if hasDuplicates {
+ return nil, fmt.Errorf("could not load variables due to duplicates: %v", err)
+ }
+
  batch := job.Batch
 
  spec := spec.Spec{
@@ -93,6 +136,7 @@ func GetSpecFromJob(job models.DiggerJob) (*spec.Spec, error) {
  RepoName: batch.RepoName,
  WorkflowFile: job.WorkflowFile,
  },
+ Variables: variablesSpec,
  Policy: spec.PolicySpec{
  PolicyType: "http",
  },

cli/cmd/digger/default.go

@@ -54,6 +54,7 @@ var defaultCmd = &cobra.Command{
  lib_spec.BackendApiProvider{},
  lib_spec.BasicPolicyProvider{},
  lib_spec.PlanStorageProvider{},
+ lib_spec.VariablesProvider{},
  comment_updater.CommentUpdaterProviderBasic{},
  )
  }

cli/cmd/digger/main_test.go

@@ -900,7 +900,7 @@ func TestGitHubNewPullRequestContext(t *testing.T) {
  }
 
  event := context.Event.(github.PullRequestEvent)
- jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig)
+ jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig, false)
  _, _, err = digger.RunJobs(jobs, &prManager, prManager, lock, reporter, &planStorage, policyChecker, comment_updater.NoopCommentUpdater{}, backendApi, "123", false, false, "1", "dir")
 
  assert.NoError(t, err)
@@ -995,7 +995,7 @@ func TestGitHubNewPullRequestInMultiEnvProjectContext(t *testing.T) {
  impactedProjects, requestedProject, prNumber, err := dggithub.ProcessGitHubEvent(ghEvent, &diggerConfig, prManager)
  assert.NoError(t, err)
  event := context.Event.(github.PullRequestEvent)
- jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig)
+ jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig, false)
  spew.Dump(lock.MapLock)
  assert.Equal(t, pullRequestNumber, prNumber)
  assert.Equal(t, 1, len(jobs))

cli/cmd/digger/run_spec.go

@@ -40,6 +40,7 @@ var runSpecCmd = &cobra.Command{
  lib_spec.BackendApiProvider{},
  lib_spec.BasicPolicyProvider{},
  lib_spec.PlanStorageProvider{},
+ lib_spec.VariablesProvider{},
  comment_summary.CommentUpdaterProviderBasic{},
  )
  if err != nil {

cli/pkg/github/github.go

@@ -135,7 +135,7 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
  }
  workflow := diggerConfig.Workflows[projectConfig.Workflow]
 
- stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars)
+ stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, true)
 
  planStorage, err := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, nil)
  if err != nil {
@@ -170,7 +170,7 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
  }
  workflow := diggerConfig.Workflows[projectConfig.Workflow]
 
- stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars)
+ stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, true)
 
  StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(projectConfig)
 
@@ -234,7 +234,7 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
  coversAllImpactedProjects := false
  err = nil
  if prEvent, ok := ghEvent.(github.PullRequestEvent); ok {
- jobs, coversAllImpactedProjects, err = dg_github.ConvertGithubPullRequestEventToJobs(&prEvent, impactedProjects, requestedProject, *diggerConfig)
+ jobs, coversAllImpactedProjects, err = dg_github.ConvertGithubPullRequestEventToJobs(&prEvent, impactedProjects, requestedProject, *diggerConfig, true)
  } else if commentEvent, ok := ghEvent.(github.IssueCommentEvent); ok {
  prBranchName, _, err := githubPrService.GetBranchName(*commentEvent.Issue.Number)
 

cli/pkg/integration/integration_test.go

@@ -396,7 +396,7 @@ func TestHappyPath(t *testing.T) {
  impactedProjects, requestedProject, prNumber, err := dg_github.ProcessGitHubEvent(ghEvent, diggerConfig, &githubPrService)
  assert.NoError(t, err)
  event := ghEvent.(github.PullRequestEvent)
- jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, *diggerConfig)
+ jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, *diggerConfig, false)
  assert.NoError(t, err)
  zipManager := storage.Zipper{}
  planStorage := &storage.GithubPlanStorage{
@@ -439,7 +439,7 @@ func TestHappyPath(t *testing.T) {
  impactedProjects, requestedProject, prNumber, err = dg_github.ProcessGitHubEvent(ghEvent, diggerConfig, &githubPrService)
  assert.NoError(t, err)
  prEvent := ghEvent.(github.PullRequestEvent)
- jobs, _, err = dg_github.ConvertGithubPullRequestEventToJobs(&prEvent, impactedProjects, requestedProject, *diggerConfig)
+ jobs, _, err = dg_github.ConvertGithubPullRequestEventToJobs(&prEvent, impactedProjects, requestedProject, *diggerConfig, false)
  assert.NoError(t, err)
  _, _, err = digger.RunJobs(jobs, &githubPrService, &githubPrService, lock, reporter, planStorage, nil, comment_updater.NoopCommentUpdater{}, nil, "", false, false, "123", dir)
  assert.NoError(t, err)
@@ -551,7 +551,7 @@ func TestMultiEnvHappyPath(t *testing.T) {
  impactedProjects, requestedProject, prNumber, err := dg_github.ProcessGitHubEvent(ghEvent, diggerConfig, &githubPrService)
  assert.NoError(t, err)
  pEvent := ghEvent.(github.PullRequestEvent)
- jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&pEvent, impactedProjects, requestedProject, *diggerConfig)
+ jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&pEvent, impactedProjects, requestedProject, *diggerConfig, false)
  assert.NoError(t, err)
 
  zipManager := storage.Zipper{}
@@ -769,7 +769,7 @@ workflows:
  impactedProjects, requestedProject, prNumber, err := dg_github.ProcessGitHubEvent(ghEvent, diggerConfig, &githubPrService)
  assert.NoError(t, err)
  pEvent := ghEvent.(github.PullRequestEvent)
- jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&pEvent, impactedProjects, requestedProject, *diggerConfig)
+ jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&pEvent, impactedProjects, requestedProject, *diggerConfig, false)
  assert.NoError(t, err)
 
  zipManager := storage.Zipper{}

cli/pkg/spec/spec.go

@@ -33,6 +33,7 @@ func RunSpec(
  backedProvider spec.BackendApiProvider,
  policyProvider spec.SpecPolicyProvider,
  PlanStorageProvider spec.PlanStorageProvider,
+ variablesProvider spec.VariablesProvider,
  commentUpdaterProvider comment_summary.CommentUpdaterProvider,
 ) error {
 
@@ -78,14 +79,6 @@ func RunSpec(
  reportError(spec, backendApi, message, err)
  }
 
- // TODO: avoid calling GetChangedFilesHere, avoid loading digger config entirely
- // also see below TODO to leverage variables provider and avoid passing it to commentUpdaterProvider
- diggerConfig, _, _, err := digger_config.LoadDiggerConfig("./", false, []string{})
- if err != nil {
- usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("Failed to read Digger digger_config. %s", err), 4)
- }
- log.Printf("Digger digger_config read successfully\n")
-
  // TODO: render mode being passable from the string
  commentUpdater, err := commentUpdaterProvider.Get(digger_config.CommentRenderModeBasic)
  if err != nil {
@@ -100,10 +93,17 @@ func RunSpec(
  }
 
  // TODO: make this part purely based on variables providers
- workflow := diggerConfig.Workflows[job.ProjectWorkflow]
- stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars)
- job.StateEnvVars = lo.Assign(job.StateEnvVars, stateEnvVars)
- job.CommandEnvVars = lo.Assign(job.CommandEnvVars, commandEnvVars)
+
+ // get variables from the variables spec
+ variablesMap, err := variablesProvider.GetVariables(spec.Variables)
+ if err != nil {
+ log.Printf("could not get variables from provider: %v", err)
+ reporterError(spec, backendApi, err)
+ usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get variables from provider: %v", err), 1)
+ }
+ job.StateEnvVars = lo.Assign(job.StateEnvVars, variablesMap)
+ job.CommandEnvVars = lo.Assign(job.CommandEnvVars, variablesMap)
+ job.RunEnvVars = lo.Assign(job.RunEnvVars, variablesMap)
 
  jobs := []scheduler.Job{job}
 

ee/cli/cmd/digger/default.go

@@ -53,6 +53,7 @@ var defaultCmd = &cobra.Command{
  lib_spec.BackendApiProvider{},
  policy.AdvancedPolicyProvider{},
  lib_spec.PlanStorageProvider{},
+ lib_spec.VariablesProvider{},
  comment_summary.CommentUpdaterProviderBasic{},
  )
  }

ee/cli/cmd/digger/main_test.go

@@ -900,7 +900,7 @@ func TestGitHubNewPullRequestContext(t *testing.T) {
  }
 
  event := context.Event.(github.PullRequestEvent)
- jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig)
+ jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig, false)
  if err != nil {
  assert.NoError(t, err)
  log.Println(err)
@@ -1001,7 +1001,7 @@ func TestGitHubNewPullRequestInMultiEnvProjectContext(t *testing.T) {
  impactedProjects, requestedProject, prNumber, err := dggithub.ProcessGitHubEvent(ghEvent, &diggerConfig, &prManager)
  assert.NoError(t, err)
  event := context.Event.(github.PullRequestEvent)
- jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig)
+ jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig, false)
 
  assert.Equal(t, pullRequestNumber, prNumber)
  assert.Equal(t, 1, len(jobs))

ee/cli/cmd/digger/run_spec.go

@@ -41,6 +41,7 @@ var runSpecCmd = &cobra.Command{
  lib_spec.BackendApiProvider{},
  policy.AdvancedPolicyProvider{},
  lib_spec.PlanStorageProvider{},
+ lib_spec.VariablesProvider{},
  comment_summary.CommentUpdaterProviderBasic{},
  )
  if err != nil {

libs/ci/azure/azure.go

@@ -442,7 +442,7 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []dig
  }
 
  prNumber := parseAzureContext.Event.(AzurePrEvent).Resource.PullRequestId
- stateEnvVars, commandEnvVars := digger_config2.CollectTerraformEnvConfig(workflow.EnvVars)
+ stateEnvVars, commandEnvVars := digger_config2.CollectTerraformEnvConfig(workflow.EnvVars, true)
  StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(project)
  jobs = append(jobs, scheduler.Job{
  ProjectName: project.Name,
@@ -472,7 +472,7 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []dig
  }
 
  prNumber := parseAzureContext.Event.(AzurePrEvent).Resource.PullRequestId
- stateEnvVars, commandEnvVars := digger_config2.CollectTerraformEnvConfig(workflow.EnvVars)
+ stateEnvVars, commandEnvVars := digger_config2.CollectTerraformEnvConfig(workflow.EnvVars, true)
  StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(project)
  jobs = append(jobs, scheduler.Job{
  ProjectName: project.Name,
@@ -502,7 +502,7 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []dig
  if !ok {
  return nil, false, fmt.Errorf("failed to find workflow digger_config '%s' for project '%s'", project.Workflow, project.Name)
  }
- stateEnvVars, commandEnvVars := digger_config2.CollectTerraformEnvConfig(workflow.EnvVars)
+ stateEnvVars, commandEnvVars := digger_config2.CollectTerraformEnvConfig(workflow.EnvVars, true)
  StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(project)
  jobs = append(jobs, scheduler.Job{
  ProjectName: project.Name,
@@ -557,7 +557,7 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []dig
  if !ok {
  return nil, false, fmt.Errorf("failed to find workflow digger_config '%s' for project '%s'", project.Workflow, project.Name)
  }
- stateEnvVars, commandEnvVars := digger_config2.CollectTerraformEnvConfig(workflow.EnvVars)
+ stateEnvVars, commandEnvVars := digger_config2.CollectTerraformEnvConfig(workflow.EnvVars, true)
  StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(project)
  jobs = append(jobs, scheduler.Job{
  ProjectName: project.Name,

libs/ci/generic/events.go

@@ -148,7 +148,7 @@ func CreateJobsForProjects(projects []digger_config.Project, command string, eve
  }
 
  runEnvVars := GetRunEnvVars(defaultBranch, prBranch, project.Name, project.Dir)
- stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars)
+ stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, false)
  StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(project)
  workspace := project.Workspace
  jobs = append(jobs, scheduler.Job{

libs/ci/github/github.go

@@ -353,7 +353,7 @@ func (svc GithubService) CheckBranchExists(branchName string) (bool, error) {
  return true, nil
 }
 
-func ConvertGithubPullRequestEventToJobs(payload *github.PullRequestEvent, impactedProjects []digger_config.Project, requestedProject *digger_config.Project, config digger_config.DiggerConfig) ([]scheduler.Job, bool, error) {
+func ConvertGithubPullRequestEventToJobs(payload *github.PullRequestEvent, impactedProjects []digger_config.Project, requestedProject *digger_config.Project, config digger_config.DiggerConfig, performEnvVarInterpolation bool) ([]scheduler.Job, bool, error) {
  workflows := config.Workflows
  jobs := make([]scheduler.Job, 0)
 
@@ -368,7 +368,7 @@ func ConvertGithubPullRequestEventToJobs(payload *github.PullRequestEvent, impac
 
  runEnvVars := generic.GetRunEnvVars(defaultBranch, prBranch, project.Name, project.Dir)
 
- stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars)
+ stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, performEnvVarInterpolation)
  pullRequestNumber := payload.PullRequest.Number
 
  StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(project)

libs/ci/gitlab/gitlab.go

@@ -357,7 +357,7 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex
  return nil, true, fmt.Errorf("failed to find workflow digger_config '%s' for project '%s'", project.Workflow, project.Name)
  }
 
- stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars)
+ stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, true)
  StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(project)
  jobs = append(jobs, scheduler.Job{
  ProjectName: project.Name,
@@ -385,7 +385,7 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex
  if !ok {
  return nil, true, fmt.Errorf("failed to find workflow digger_config '%s' for project '%s'", project.Workflow, project.Name)
  }
- stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars)
+ stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, true)
  var StateEnvProvider *stscreds.WebIdentityRoleProvider
  var CommandEnvProvider *stscreds.WebIdentityRoleProvider
  if project.AwsRoleToAssume != nil {
@@ -455,7 +455,7 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex
  if workspaceOverride != "" {
  workspace = workspaceOverride
  }
- stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars)
+ stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, true)
  StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(project)
  jobs = append(jobs, scheduler.Job{
  ProjectName: project.Name,

libs/ci/gitlab/webhooks.go

@@ -46,7 +46,7 @@ func ConvertGithubPullRequestEventToJobs(payload *gitlab.MergeEvent, impactedPro
 
  runEnvVars := generic.GetRunEnvVars(defaultBranch, prBranch, project.Name, project.Dir)
 
- stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars)
+ stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, false)
  pullRequestNumber := payload.ObjectAttributes.IID
  namespace := payload.Project.PathWithNamespace
  sender := payload.User.Username

libs/digger_config/digger_config.go

@@ -688,7 +688,7 @@ func retrieveConfigFile(workingDir string) (string, error) {
  return "", nil
 }
 
-func CollectTerraformEnvConfig(envs *TerraformEnvConfig) (map[string]string, map[string]string) {
+func CollectTerraformEnvConfig(envs *TerraformEnvConfig, performInterpolation bool) (map[string]string, map[string]string) {
  stateEnvVars := map[string]string{}
  commandEnvVars := map[string]string{}
 
@@ -697,15 +697,23 @@ func CollectTerraformEnvConfig(envs *TerraformEnvConfig) (map[string]string, map
  if envvar.Value != "" {
  stateEnvVars[envvar.Name] = envvar.Value
  } else if envvar.ValueFrom != "" {
- stateEnvVars[envvar.Name] = os.Getenv(envvar.ValueFrom)
+ if performInterpolation {
+ stateEnvVars[envvar.Name] = os.Getenv(envvar.ValueFrom)
+ } else {
+ stateEnvVars[envvar.Name] = fmt.Sprintf("$DIGGER_%v", envvar.ValueFrom)
+ }
  }
  }
 
  for _, envvar := range envs.Commands {
  if envvar.Value != "" {
  commandEnvVars[envvar.Name] = envvar.Value
  } else if envvar.ValueFrom != "" {
- commandEnvVars[envvar.Name] = os.Getenv(envvar.ValueFrom)
+ if performInterpolation {
+ commandEnvVars[envvar.Name] = os.Getenv(envvar.ValueFrom)
+ } else {
+ commandEnvVars[envvar.Name] = fmt.Sprintf("$DIGGER_%v", envvar.ValueFrom)
+ }
  }
  }
  }