feat: add autodiscover enabled feature (runatlantis#3895)

* add flag to allow the user disable the autodiscover * add global config and doc * feat: Implement autodiscover.mode * fix: Minor doc fixes * fix: Small fixes to docs/indent/tests * fix: Line length, quoting, function comments * fix: Add a few more tests and remove newlines * fix: Always camel case never snake --------- Co-authored-by: Marcelo Medeiros <[email protected]> Co-authored-by: nitrocode <[email protected]> Co-authored-by: PePe Amengual <[email protected]>
gocardless · Feb 13, 2024 · b5d85a5 · b5d85a5
1 parent 3cc8204
commit b5d85a5
Show file tree

Hide file tree

Showing 22 changed files with 822 additions and 35 deletions.
diff --git a/cmd/server.go b/cmd/server.go
@@ -54,6 +54,7 @@ const (
 	AllowForkPRsFlag                 = "allow-fork-prs"
 	AllowRepoConfigFlag              = "allow-repo-config"
 	AtlantisURLFlag                  = "atlantis-url"
+	AutoDiscoverModeFlag             = "autodiscover-mode"
 	AutomergeFlag                    = "automerge"
 	ParallelPlanFlag                 = "parallel-plan"
 	ParallelApplyFlag                = "parallel-apply"
@@ -148,6 +149,7 @@ const (
 	DefaultADBasicUser                  = ""
 	DefaultADBasicPassword              = ""
 	DefaultADHostname                   = "dev.azure.com"
+	DefaultAutoDiscoverMode             = "auto"
 	DefaultAutoplanFileList             = "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl"
 	DefaultAllowCommands                = "version,plan,apply,unlock,approve_policies"
 	DefaultCheckoutStrategy             = CheckoutStrategyBranch
@@ -207,6 +209,12 @@ var stringFlags = map[string]stringFlag{
 	AtlantisURLFlag: {
 		description: "URL that Atlantis can be reached at. Defaults to http://$(hostname):$port where $port is from --" + PortFlag + ". Supports a base path ex. https://example.com/basepath.",
 	},
+	AutoDiscoverModeFlag: {
+		description: "Auto discover mode controls whether projects in a repo are discovered by Atlantis. Defaults to 'auto' which " +
+			"means projects will be discovered when no explicit projects are defined in repo config. Also supports 'enabled' (always " +
+			"discover projects) and 'disabled' (never discover projects).",
+		defaultValue: DefaultAutoDiscoverMode,
+	},
 	AutoplanModulesFromProjects: {
 		description: "Comma separated list of file patterns to select projects Atlantis will index for module dependencies." +
 			" Indexed projects will automatically be planned if a module they depend on is modified." +
@@ -871,6 +879,9 @@ func (s *ServerCmd) setDefaults(c *server.UserConfig) {
 	if c.WebPassword == "" {
 		c.WebPassword = DefaultWebPassword
 	}
+	if c.AutoDiscoverModeFlag == "" {
+		c.AutoDiscoverModeFlag = DefaultAutoDiscoverMode
+	}
 }
 
 func (s *ServerCmd) validate(userConfig server.UserConfig) error {

diff --git a/cmd/server_test.go b/cmd/server_test.go
@@ -60,6 +60,7 @@ var testFlags = map[string]interface{}{
 	AllowCommandsFlag:                "version,plan,unlock,import,approve_policies", // apply is disabled by DisableApply
 	AllowForkPRsFlag:                 true,
 	AllowRepoConfigFlag:              true,
+	AutoDiscoverModeFlag:             "auto",
 	AutomergeFlag:                    true,
 	AutoplanFileListFlag:             "**/*.tf,**/*.yml",
 	BitbucketBaseURLFlag:             "https://bitbucket-base-url.com",

diff --git a/runatlantis.io/docs/repo-level-atlantis-yaml.md b/runatlantis.io/docs/repo-level-atlantis-yaml.md
@@ -34,19 +34,27 @@ By default, this is not allowed.
 :::
 
 ::: warning
-Once an `atlantis.yaml` file exists in a repo, Atlantis won't try to determine
-where to run plan automatically. Instead it will just follow the project configuration.
-This means that you'll need to define each project in your repo.
+Once an `atlantis.yaml` file exists in a repo and one or more `projects` are configured,
+Atlantis won't try to determine where to run plan automatically. Instead it will just
+follow the project configuration. This means that you'll need to define each project
+in your repo.
 
 If you have many directories with Terraform configuration, each directory will
 need to be defined.
+
+This behavior can be overriden by setting `autodiscover.mode` to
+`enabled` in which case Atlantis will still try to discover projects which were not
+explicitly configured. If the directory of any discovered project conflicts with a
+manually configured project, the manually configured project will take precedence.
 :::
 
 ## Example Using All Keys
 
 ```yaml
 version: 3
 automerge: true
+autodiscover:
+  mode: auto
 delete_source_branch_on_merge: true
 parallel_plan: true
 parallel_apply: true
@@ -281,6 +289,34 @@ in each group one by one.
 If any plan/apply fails and `abort_on_execution_order_fail` is set to true on a repo level, all the 
 following groups will be aborted. For this example, if project2 fails then project1 will not run.
 
+### Autodiscovery Config
+```yaml
+autodiscover:
+  mode: "auto"
+```
+The above is the default configuration for `autodiscover.mode`. When `autodiscover.mode` is auto,
+projects will be discovered only if the repo has no `projects` configured.
+
+```yaml
+autodiscover:
+  mode: "disabled"
+```
+With the config above, Atlantis will never try to discover projects, even when there are no
+`projects` configured. This is useful if dynamically generating Atlantis config in pre_workflow hooks.
+See [Dynamic Repo Config Generation](pre-workflow-hooks.html#dynamic-repo-config-generation).
+
+```yaml
+autodiscover:
+  mode: "enabled"
+```
+With the config above, Atlantis will unconditionally try to discover projects based on modified_files,
+even when the directory of the project is missing from the configured `projects` in the repo configuration.
+If a discovered project has the same directory as a project which was manually configured in `projects`,
+the manual configuration will take precedence.
+
+Use this feature when some projects require specific configuration in a repo with many projects yet
+it's still desirable for Atlantis to plan/apply for projects not enumerated in the config.
+
 ### Custom Backend Config
 See [Custom Workflow Use Cases: Custom Backend Config](custom-workflows.html#custom-backend-config)
 

diff --git a/runatlantis.io/docs/server-configuration.md b/runatlantis.io/docs/server-configuration.md
@@ -106,6 +106,22 @@ Values are chosen in this order:
   * If a load balancer with a non http/https port (not the one defined in the `--port` flag) is used, update the URL to include the port like in the example above.
    * This URL is used as the `details` link next to each atlantis job to view the job's logs.
 
+### `--autodiscover-mode`
+  ```bash
+  atlantis server --autodiscover-mode="<auto|enabled|disabled>"
+  # or
+  ATLANTIS_AUTODISCOVER_MODE="<auto|enabled|disabled>"
+  ```
+  Sets auto discover mode, default is `auto`. When set to `auto`, projects in a repo will be discovered by
+  Atlantis when there are no projects configured in the repo config. If one or more projects are defined
+  in the repo config then auto discovery will be completely disabled.
+
+  When set to `enabled` projects will be discovered unconditionally. If an auto discovered project is already
+  defined in the projects section of the repo config, the project from the repo config will take precedence over
+  the auto discovered project.
+
+  When set to `disabled` projects will never be discovered, even if there are no projects configured in the repo config.
+
 ### `--automerge`
   ```bash
   atlantis server --automerge

diff --git a/runatlantis.io/docs/server-side-repo-config.md b/runatlantis.io/docs/server-side-repo-config.md
@@ -88,6 +88,10 @@ repos:
   # policy_check defines if policy checking should be enable on this repository.
   policy_check: false
 
+  # autodiscover defines how atlantis should automatically discover projects in this repository.
+  autodiscover:
+    mode: auto
+
   # id can also be an exact match.
 - id: github.com/myorg/specific-repo
 
@@ -496,6 +500,7 @@ If you set a workflow with the key `default`, it will override this.
 | repo_locking                  | bool     | false   | no       | Whether or not to get a lock.                                                                                                                                                                                                                                                                             |
 | policy_check                  | bool     | false   | no       | Whether or not to run policy checks on this repository.                                                                                                                                                                                                                                                   |
 | custom_policy_check                  | bool     | false   | no       | Whether or not to enable custom policy check tools outside of Conftest on this repository.                                                                                                                                                                                                       |
+| autodiscover                  | AutoDiscover     | none   | no       | Auto discover settings for this repo
 
 
 :::tip Notes

diff --git a/server/controllers/events/events_controller_e2e_test.go b/server/controllers/events/events_controller_e2e_test.go
@@ -1352,6 +1352,7 @@ func setupE2E(t *testing.T, repoDir string, opt setupOption) (events_controllers
 		false,
 		false,
 		false,
+		"auto",
 		statsScope,
 		logger,
 		terraformClient,

diff --git a/server/core/config/parser_validator_test.go b/server/core/config/parser_validator_test.go
@@ -1329,6 +1329,22 @@ func TestParseGlobalCfg(t *testing.T) {
   import_requirements: [invalid]`,
 			expErr: "repos: (0: (import_requirements: \"invalid\" is not a valid import_requirement, only \"approved\", \"mergeable\" and \"undiverged\" are supported.).).",
 		},
+		"disable autodiscover": {
+			input: `repos: 
+- id: /.*/
+  autodiscover:
+    mode: disabled`,
+			exp: valid.GlobalCfg{
+				Repos: []valid.Repo{
+					defaultCfg.Repos[0],
+					{
+						IDRegex:      regexp.MustCompile(".*"),
+						AutoDiscover: &valid.AutoDiscover{Mode: valid.AutoDiscoverDisabledMode},
+					},
+				},
+				Workflows: defaultCfg.Workflows,
+			},
+		},
 		"no workflows key": {
 			input: `repos: []`,
 			exp:   defaultCfg,
@@ -1404,13 +1420,17 @@ repos:
   allowed_overrides: [plan_requirements, apply_requirements, import_requirements, workflow, delete_source_branch_on_merge]
   allow_custom_workflows: true
   policy_check: true
+  autodiscover:
+    mode: enabled
 - id: /.*/
   branch: /(master|main)/
   pre_workflow_hooks:
     - run: custom workflow command
   post_workflow_hooks:
     - run: custom workflow command
   policy_check: false
+  autodiscover:
+    mode: disabled
 workflows:
   custom1:
     plan:
@@ -1457,13 +1477,15 @@ policies:
 						AllowedOverrides:     []string{"plan_requirements", "apply_requirements", "import_requirements", "workflow", "delete_source_branch_on_merge"},
 						AllowCustomWorkflows: Bool(true),
 						PolicyCheck:          Bool(true),
+						AutoDiscover:         &valid.AutoDiscover{Mode: valid.AutoDiscoverEnabledMode},
 					},
 					{
 						IDRegex:           regexp.MustCompile(".*"),
 						BranchRegex:       regexp.MustCompile("(master|main)"),
 						PreWorkflowHooks:  preWorkflowHooks,
 						PostWorkflowHooks: postWorkflowHooks,
 						PolicyCheck:       Bool(false),
+						AutoDiscover:      &valid.AutoDiscover{Mode: valid.AutoDiscoverDisabledMode},
 					},
 				},
 				Workflows: map[string]valid.Workflow{
@@ -1574,6 +1596,7 @@ workflows:
 						RepoLocking:               Bool(true),
 						PolicyCheck:               Bool(false),
 						CustomPolicyCheck:         Bool(false),
+						AutoDiscover:              raw.DefaultAutoDiscover(),
 					},
 				},
 				Workflows: map[string]valid.Workflow{
@@ -1727,7 +1750,10 @@ func TestParserValidator_ParseGlobalCfgJSON(t *testing.T) {
       "allowed_workflows": ["custom"],
       "apply_requirements": ["mergeable", "approved"],
       "allowed_overrides": ["workflow", "apply_requirements"],
-      "allow_custom_workflows": true
+      "allow_custom_workflows": true,
+      "autodiscover": {
+        "mode": "enabled"
+      }
     },
     {
       "id": "github.com/owner/repo"
@@ -1792,13 +1818,15 @@ func TestParserValidator_ParseGlobalCfgJSON(t *testing.T) {
 						AllowedWorkflows:     []string{"custom"},
 						AllowedOverrides:     []string{"workflow", "apply_requirements"},
 						AllowCustomWorkflows: Bool(true),
+						AutoDiscover:         &valid.AutoDiscover{Mode: valid.AutoDiscoverEnabledMode},
 					},
 					{
 						ID:                   "github.com/owner/repo",
 						IDRegex:              nil,
 						ApplyRequirements:    nil,
 						AllowedOverrides:     nil,
 						AllowCustomWorkflows: nil,
+						AutoDiscover:         nil,
 					},
 				},
 				Workflows: map[string]valid.Workflow{

diff --git a/server/core/config/raw/autodiscover.go b/server/core/config/raw/autodiscover.go
@@ -0,0 +1,38 @@
+package raw
+
+import (
+	validation "github.com/go-ozzo/ozzo-validation"
+	"github.com/runatlantis/atlantis/server/core/config/valid"
+)
+
+var DefaultAutoDiscoverMode = valid.AutoDiscoverAutoMode
+
+type AutoDiscover struct {
+	Mode *valid.AutoDiscoverMode `yaml:"mode,omitempty"`
+}
+
+func (a AutoDiscover) ToValid() *valid.AutoDiscover {
+	var v valid.AutoDiscover
+
+	if a.Mode != nil {
+		v.Mode = *a.Mode
+	} else {
+		v.Mode = DefaultAutoDiscoverMode
+	}
+
+	return &v
+}
+
+func (a AutoDiscover) Validate() error {
+	res := validation.ValidateStruct(&a,
+		// If a.Mode is nil, this should still pass validation.
+		validation.Field(&a.Mode, validation.In(valid.AutoDiscoverAutoMode, valid.AutoDiscoverDisabledMode, valid.AutoDiscoverEnabledMode)),
+	)
+	return res
+}
+
+func DefaultAutoDiscover() *valid.AutoDiscover {
+	return &valid.AutoDiscover{
+		Mode: DefaultAutoDiscoverMode,
+	}
+}