feat: add discover_projects_with_missing_config feature

runatlantis.io/docs/repo-level-atlantis-yaml.md

@@ -34,12 +34,18 @@ By default, this is not allowed.
 :::
 
 ::: warning
-Once an `atlantis.yaml` file exists in a repo, Atlantis won't try to determine
-where to run plan automatically. Instead it will just follow the project configuration.
-This means that you'll need to define each project in your repo.
+Once an `atlantis.yaml` file exists in a repo and one or more `projects` are configured,
+Atlantis won't try to determine where to run plan automatically. Instead it will just
+follow the project configuration. This means that you'll need to define each project
+in your repo.
 
 If you have many directories with Terraform configuration, each directory will
 need to be defined.
+
+This behavior can be overriden by setting `discover_projects_with_missing_config` to
+`true` in which case Atlantis will still try to discover projects which were not
+explicitly configured. If the directory of any discovered project conflicts with a
+manually configured project, the manually configured project will take precedence.
 :::
 
 ## Example Using All Keys
@@ -48,6 +54,7 @@ need to be defined.
 version: 3
 automerge: true
 delete_source_branch_on_merge: true
+discover_projects_with_missing_config: true
 parallel_plan: true
 parallel_apply: true
 abort_on_execution_order_fail: true
@@ -281,6 +288,21 @@ in each group one by one.
 If any plan/apply fails and `abort_on_execution_order_fail` is set to true on a repo level, all the 
 following groups will be aborted. For this example, if project2 fails then project1 will not run.
 
+### Discovering projects without explicit config
+```yaml
+version: 3
+discover_projects_with_missing_config: true
+projects:
+- dir: project1
+```
+With the config above, Atlantis will unconditionally try to discover projects based on modified_files,
+even when the directory of the project is missing from the configured `projects` in the repo configuration.
+If a discovered project has the same directory as a project which was manually configured in `projects`,
+the manual configuration will take precedence.
+
+Use this feature when some projects require specific configuration in a repo with many projects yet
+it's still desirable for Atlantis to plan/apply for projects not enumerated in the config.
+
 ### Custom Backend Config
 See [Custom Workflow Use Cases: Custom Backend Config](custom-workflows.html#custom-backend-config)
 

server/core/config/raw/repo_cfg.go

@@ -13,19 +13,23 @@ const DefaultEmojiReaction = ""
 // DefaultAbortOnExcecutionOrderFail being false is the default setting for abort on execution group failiures
 const DefaultAbortOnExcecutionOrderFail = false
 
+// DefaultDiscoverProjectsWithMissingConfig being false is the default setting for discovering projects with missing config
+const DefaultDiscoverProjectsWithMissingConfig = false
+
 // RepoCfg is the raw schema for repo-level atlantis.yaml config.
 type RepoCfg struct {
-	Version                    *int                `yaml:"version,omitempty"`
-	Projects                   []Project           `yaml:"projects,omitempty"`
-	Workflows                  map[string]Workflow `yaml:"workflows,omitempty"`
-	PolicySets                 PolicySets          `yaml:"policies,omitempty"`
-	Automerge                  *bool               `yaml:"automerge,omitempty"`
-	ParallelApply              *bool               `yaml:"parallel_apply,omitempty"`
-	ParallelPlan               *bool               `yaml:"parallel_plan,omitempty"`
-	DeleteSourceBranchOnMerge  *bool               `yaml:"delete_source_branch_on_merge,omitempty"`
-	EmojiReaction              *string             `yaml:"emoji_reaction,omitempty"`
-	AllowedRegexpPrefixes      []string            `yaml:"allowed_regexp_prefixes,omitempty"`
-	AbortOnExcecutionOrderFail *bool               `yaml:"abort_on_execution_order_fail,omitempty"`
+	Version                           *int                `yaml:"version,omitempty"`
+	Projects                          []Project           `yaml:"projects,omitempty"`
+	Workflows                         map[string]Workflow `yaml:"workflows,omitempty"`
+	PolicySets                        PolicySets          `yaml:"policies,omitempty"`
+	Automerge                         *bool               `yaml:"automerge,omitempty"`
+	ParallelApply                     *bool               `yaml:"parallel_apply,omitempty"`
+	ParallelPlan                      *bool               `yaml:"parallel_plan,omitempty"`
+	DeleteSourceBranchOnMerge         *bool               `yaml:"delete_source_branch_on_merge,omitempty"`
+	EmojiReaction                     *string             `yaml:"emoji_reaction,omitempty"`
+	AllowedRegexpPrefixes             []string            `yaml:"allowed_regexp_prefixes,omitempty"`
+	AbortOnExcecutionOrderFail        *bool               `yaml:"abort_on_execution_order_fail,omitempty"`
+	DiscoverProjectsWithMissingConfig *bool               `yaml:"discover_projects_with_missing_config,omitempty"`
 }
 
 func (r RepoCfg) Validate() error {
@@ -71,17 +75,23 @@ func (r RepoCfg) ToValid() valid.RepoCfg {
 		abortOnExcecutionOrderFail = *r.AbortOnExcecutionOrderFail
 	}
 
+	discoverProjectsWithMissingConfig := DefaultDiscoverProjectsWithMissingConfig
+	if r.DiscoverProjectsWithMissingConfig != nil {
+		discoverProjectsWithMissingConfig = *r.DiscoverProjectsWithMissingConfig
+	}
+
 	return valid.RepoCfg{
-		Version:                    *r.Version,
-		Projects:                   validProjects,
-		Workflows:                  validWorkflows,
-		Automerge:                  automerge,
-		ParallelApply:              parallelApply,
-		ParallelPlan:               parallelPlan,
-		ParallelPolicyCheck:        parallelPlan,
-		DeleteSourceBranchOnMerge:  r.DeleteSourceBranchOnMerge,
-		AllowedRegexpPrefixes:      r.AllowedRegexpPrefixes,
-		EmojiReaction:              emojiReaction,
-		AbortOnExcecutionOrderFail: abortOnExcecutionOrderFail,
+		Version:                           *r.Version,
+		Projects:                          validProjects,
+		Workflows:                         validWorkflows,
+		Automerge:                         automerge,
+		ParallelApply:                     parallelApply,
+		ParallelPlan:                      parallelPlan,
+		ParallelPolicyCheck:               parallelPlan,
+		DeleteSourceBranchOnMerge:         r.DeleteSourceBranchOnMerge,
+		AllowedRegexpPrefixes:             r.AllowedRegexpPrefixes,
+		EmojiReaction:                     emojiReaction,
+		AbortOnExcecutionOrderFail:        abortOnExcecutionOrderFail,
+		DiscoverProjectsWithMissingConfig: discoverProjectsWithMissingConfig,
 	}
 }

server/core/config/raw/repo_cfg_test.go

@@ -259,48 +259,53 @@ func TestConfig_ToValid(t *testing.T) {
 			},
 		},
 		{
-			description: "automerge, parallel_apply and abort_on_execution_order_fail omitted",
+			description: "automerge, parallel_apply, abort_on_execution_order_fail, and discover_projects_with_missing_config omitted",
 			input: raw.RepoCfg{
 				Version: Int(2),
 			},
 			exp: valid.RepoCfg{
-				Version:                    2,
-				Automerge:                  nil,
-				ParallelApply:              nil,
-				AbortOnExcecutionOrderFail: false,
-				Workflows:                  map[string]valid.Workflow{},
+				Version:                           2,
+				Automerge:                         nil,
+				ParallelApply:                     nil,
+				AbortOnExcecutionOrderFail:        false,
+				DiscoverProjectsWithMissingConfig: false,
+				Workflows:                         map[string]valid.Workflow{},
 			},
 		},
 		{
-			description: "automerge, parallel_apply and abort_on_execution_order_fail true",
+			description: "automerge, parallel_apply, abort_on_execution_order_fail, and discover_projects_with_missing_config true",
 			input: raw.RepoCfg{
-				Version:                    Int(2),
-				Automerge:                  Bool(true),
-				ParallelApply:              Bool(true),
-				AbortOnExcecutionOrderFail: Bool(true),
+				Version:                           Int(2),
+				Automerge:                         Bool(true),
+				ParallelApply:                     Bool(true),
+				AbortOnExcecutionOrderFail:        Bool(true),
+				DiscoverProjectsWithMissingConfig: Bool(true),
 			},
 			exp: valid.RepoCfg{
-				Version:                    2,
-				Automerge:                  Bool(true),
-				ParallelApply:              Bool(true),
-				AbortOnExcecutionOrderFail: true,
-				Workflows:                  map[string]valid.Workflow{},
+				Version:                           2,
+				Automerge:                         Bool(true),
+				ParallelApply:                     Bool(true),
+				AbortOnExcecutionOrderFail:        true,
+				DiscoverProjectsWithMissingConfig: true,
+				Workflows:                         map[string]valid.Workflow{},
 			},
 		},
 		{
-			description: "automerge, parallel_apply and abort_on_execution_order_fail false",
+			description: "automerge, parallel_apply, abort_on_execution_order_fail, and discover_projects_with_missing_config false",
 			input: raw.RepoCfg{
-				Version:                    Int(2),
-				Automerge:                  Bool(false),
-				ParallelApply:              Bool(false),
-				AbortOnExcecutionOrderFail: Bool(false),
+				Version:                           Int(2),
+				Automerge:                         Bool(false),
+				ParallelApply:                     Bool(false),
+				AbortOnExcecutionOrderFail:        Bool(false),
+				DiscoverProjectsWithMissingConfig: Bool(false),
 			},
 			exp: valid.RepoCfg{
-				Version:                    2,
-				Automerge:                  Bool(false),
-				ParallelApply:              Bool(false),
-				AbortOnExcecutionOrderFail: false,
-				Workflows:                  map[string]valid.Workflow{},
+				Version:                           2,
+				Automerge:                         Bool(false),
+				ParallelApply:                     Bool(false),
+				AbortOnExcecutionOrderFail:        false,
+				Workflows:                         map[string]valid.Workflow{},
+				DiscoverProjectsWithMissingConfig: false,
 			},
 		},
 		{

server/core/config/valid/repo_cfg.go

@@ -14,20 +14,21 @@ import (
 // RepoCfg is the atlantis.yaml config after it's been parsed and validated.
 type RepoCfg struct {
 	// Version is the version of the atlantis YAML file.
-	Version                    int
-	Projects                   []Project
-	Workflows                  map[string]Workflow
-	PolicySets                 PolicySets
-	Automerge                  *bool
-	ParallelApply              *bool
-	ParallelPlan               *bool
-	ParallelPolicyCheck        *bool
-	DeleteSourceBranchOnMerge  *bool
-	RepoLocking                *bool
-	CustomPolicyCheck          *bool
-	EmojiReaction              string
-	AllowedRegexpPrefixes      []string
-	AbortOnExcecutionOrderFail bool
+	Version                           int
+	Projects                          []Project
+	Workflows                         map[string]Workflow
+	PolicySets                        PolicySets
+	Automerge                         *bool
+	ParallelApply                     *bool
+	ParallelPlan                      *bool
+	ParallelPolicyCheck               *bool
+	DeleteSourceBranchOnMerge         *bool
+	RepoLocking                       *bool
+	CustomPolicyCheck                 *bool
+	EmojiReaction                     string
+	AllowedRegexpPrefixes             []string
+	AbortOnExcecutionOrderFail        bool
+	DiscoverProjectsWithMissingConfig bool
 }
 
 func (r RepoCfg) FindProjectsByDirWorkspace(repoRelDir string, workspace string) []Project {

server/events/project_command_builder.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/runatlantis/atlantis/server/core/config"
 	"github.com/runatlantis/atlantis/server/events/command"
+	"github.com/runatlantis/atlantis/server/events/models"
 	"github.com/runatlantis/atlantis/server/events/vcs"
 )
 
@@ -349,18 +350,23 @@ func (p *DefaultProjectCommandBuilder) buildAllCommandsByCfg(ctx *command.Contex
 				return nil, errors.Wrapf(err, "parsing %s", repoCfgFile)
 			}
 			ctx.Log.Info("successfully parsed remote %s file", repoCfgFile)
-			if len(repoCfg.Projects) > 0 {
-				matchingProjects, err := p.ProjectFinder.DetermineProjectsViaConfig(ctx.Log, modifiedFiles, repoCfg, "", nil)
-				if err != nil {
-					return nil, err
-				}
-				ctx.Log.Info("%d projects are changed on MR %q based on their when_modified config", len(matchingProjects), ctx.Pull.Num)
-				if len(matchingProjects) == 0 {
-					ctx.Log.Info("skipping repo clone since no project was modified")
-					return []command.ProjectContext{}, nil
+			// If discover_projects_with_missing_config is set, we never want to skip cloning
+			if !repoCfg.DiscoverProjectsWithMissingConfig {
+				if len(repoCfg.Projects) > 0 {
+					matchingProjects, err := p.ProjectFinder.DetermineProjectsViaConfig(ctx.Log, modifiedFiles, repoCfg, "", nil)
+					if err != nil {
+						return nil, err
+					}
+					ctx.Log.Info("%d projects are changed on MR %q based on their when_modified config", len(matchingProjects), ctx.Pull.Num)
+					if len(matchingProjects) == 0 {
+						ctx.Log.Info("skipping repo clone since no project was modified")
+						return []command.ProjectContext{}, nil
+					}
+				} else {
+					ctx.Log.Info("no projects are defined in %s. Will resume automatic detection", repoCfgFile)
 				}
 			} else {
-				ctx.Log.Info("No projects are defined in %s. Will resume automatic detection", repoCfgFile)
+				ctx.Log.Info("discover_projects_with_missing_config enabled. Will resume automatic detection")
 			}
 			// NOTE: We discard this work here and end up doing it again after
 			// cloning to ensure all the return values are set properly with
@@ -454,17 +460,39 @@ func (p *DefaultProjectCommandBuilder) buildAllCommandsByCfg(ctx *command.Contex
 					p.TerraformExecutor,
 				)...)
 		}
-	} else {
+	}
+
+	if len(repoCfg.Projects) == 0 || repoCfg.DiscoverProjectsWithMissingConfig {
 		// If there is no config file or it specified no projects, then we'll plan each project that
 		// our algorithm determines was modified.
 		if hasRepoCfg {
-			ctx.Log.Info("No projects are defined in %s. Will resume automatic detection", repoCfgFile)
+			if len(repoCfg.Projects) != 0 {
+				ctx.Log.Info("discover_projects_with_missing_config enabled. Will resume automatic detection")
+			} else {
+				ctx.Log.Info("no projects are defined in %s. Will resume automatic detection", repoCfgFile)
+			}
 		} else {
 			ctx.Log.Info("found no %s file", repoCfgFile)
 		}
 		// build a module index for projects that are explicitly included
-		modifiedProjects := p.ProjectFinder.DetermineProjects(ctx.Log, modifiedFiles, ctx.Pull.BaseRepo.FullName, repoDir, p.AutoplanFileList, moduleInfo)
-		ctx.Log.Info("automatically determined that there were %d projects modified in this pull request: %s", len(modifiedProjects), modifiedProjects)
+		allModifiedProjects := p.ProjectFinder.DetermineProjects(ctx.Log, modifiedFiles, ctx.Pull.BaseRepo.FullName, repoDir, p.AutoplanFileList, moduleInfo)
+		// If a project is already manually configured with the same dir as a discovered project, the manually configured project should take precendence
+		modifiedProjects := make([]models.Project, 0)
+		configuredProjDirs := make(map[string]bool)
+		// We compare against all configured projects instead of projects which match the modified files in case a
+		// project is being specifically excluded (ex: when_modified doesn't match). We don't want to accidently
+		// "discover" it again.
+		for _, configProj := range repoCfg.Projects {
+			// Clean the path to make sure ./rel_path is equivalent to rel_path, etc
+			configuredProjDirs[filepath.Clean(configProj.Dir)] = true
+		}
+		for _, mp := range allModifiedProjects {
+			_, dirExists := configuredProjDirs[filepath.Clean(mp.Path)]
+			if !dirExists {
+				modifiedProjects = append(modifiedProjects, mp)
+			}
+		}
+		ctx.Log.Info("automatically determined that there were %d additional projects modified in this pull request: %s", len(modifiedProjects), modifiedProjects)
 		for _, mp := range modifiedProjects {
 			ctx.Log.Debug("determining config for project at dir: %q", mp.Path)
 			pWorkspace, err := p.ProjectFinder.DetermineWorkspaceFromHCL(ctx.Log, repoDir)