feat: add lock_all_projects_before_exec

runatlantis.io/docs/repo-level-atlantis-yaml.md

@@ -61,6 +61,7 @@ delete_source_branch_on_merge: true
 parallel_plan: true
 parallel_apply: true
 abort_on_execution_order_fail: true
+lock_all_projects_before_exec: true
 projects:
 - name: my-project-name
   branch: /main/
@@ -394,6 +395,25 @@ it's still desirable for Atlantis to plan/apply for projects not enumerated in t
 
 See [Custom Workflow Use Cases: Custom Backend Config](custom-workflows.md#custom-backend-config)
 
+### Lock all changed projects before plan/apply
+
+```yaml
+lock_all_projects_before_exec: true
+```
+
+By default, Atlantis acquires a lock for project right before running `plan` on it 
+(in this context, and later, we use `plan`, but this option works similarly with `apply` locking).
+For example, if you have a pull request with changes in projects `project1` and `project2`, 
+scheme of locking will be the following (parallel planning is disabled for easier understanding):
+```
+lock project1 -> plan project1 -> lock project2 (this lock may fail) -> plan project2
+```
+With this option enabled, Atlantis will lock each changed project before running `plan` on any of the projects.
+So, in the same example, the scheme will be as follows:
+```
+lock project1 -> lock project2 (locks for all changed projects are acquired, or not) -> plan project1 -> plan project2
+```
+
 ## Reference
 
 ### Top-Level Keys
@@ -405,6 +425,8 @@ delete_source_branch_on_merge: false
 projects:
 workflows:
 allowed_regexp_prefixes:
+abort_on_execution_order_fail: false
+lock_all_projects_before_exec: false
 ```
 
 | Key                           | Type                                                   | Default | Required | Description                                                                                                                        |
@@ -415,6 +437,8 @@ allowed_regexp_prefixes:
 | projects                      | array[[Project](repo-level-atlantis-yaml.md#project)]  | `[]`    | no       | Lists the projects in this repo.                                                                                                   |
 | workflows<br />*(restricted)* | map[string: [Workflow](custom-workflows.md#reference)] | `{}`    | no       | Custom workflows.                                                                                                                  |
 | allowed_regexp_prefixes       | array\[string\]                                          | `[]`    | no       | Lists the allowed regexp prefixes to use when the [`--enable-regexp-cmd`](server-configuration.md#enable-regexp-cmd) flag is used. |
+| abort_on_execution_order_fail | bool                                                   | `false` | no       | Stops all following execution groups when failed in some one.                                                                      |
+| lock_all_projects_before_exec | bool                                                   | `false` | no       | Acquires locks on each projects before planning/applying on any of project.                                                        |
 
 ### Project
 

server/core/config/raw/repo_cfg.go

@@ -13,6 +13,9 @@ const DefaultEmojiReaction = ""
 // DefaultAbortOnExcecutionOrderFail being false is the default setting for abort on execution group failiures
 const DefaultAbortOnExcecutionOrderFail = false
 
+// DefaultLockAllProjectsBeforeExec being false is the default setting for locking all projects before plan/apply
+const DefaultLockAllProjectsBeforeExec = false
+
 // RepoCfg is the raw schema for repo-level atlantis.yaml config.
 type RepoCfg struct {
 	Version                    *int                `yaml:"version,omitempty"`
@@ -29,6 +32,7 @@ type RepoCfg struct {
 	AbortOnExcecutionOrderFail *bool               `yaml:"abort_on_execution_order_fail,omitempty"`
 	RepoLocks                  *RepoLocks          `yaml:"repo_locks,omitempty"`
 	SilencePRComments          []string            `yaml:"silence_pr_comments,omitempty"`
+	LockAllProjectsBeforeExec  *bool               `yaml:"lock_all_projects_before_exec,omitempty"`
 }
 
 func (r RepoCfg) Validate() error {
@@ -83,6 +87,12 @@ func (r RepoCfg) ToValid() valid.RepoCfg {
 	if r.RepoLocks != nil {
 		repoLocks = r.RepoLocks.ToValid()
 	}
+
+	lockAllProjectsBeforeExec := DefaultLockAllProjectsBeforeExec
+	if r.LockAllProjectsBeforeExec != nil {
+		lockAllProjectsBeforeExec = *r.LockAllProjectsBeforeExec
+	}
+
 	return valid.RepoCfg{
 		Version:                    *r.Version,
 		Projects:                   validProjects,
@@ -98,5 +108,6 @@ func (r RepoCfg) ToValid() valid.RepoCfg {
 		AbortOnExcecutionOrderFail: abortOnExcecutionOrderFail,
 		RepoLocks:                  repoLocks,
 		SilencePRComments:          r.SilencePRComments,
+		LockAllProjectsBeforeExec:  lockAllProjectsBeforeExec,
 	}
 }

server/core/config/raw/repo_cfg_test.go

@@ -134,6 +134,7 @@ parallel_apply: true
 parallel_plan: false
 repo_locks:
   mode: on_apply
+lock_all_projects_before_exec: true
 projects:
 - dir: mydir
   workspace: myworkspace
@@ -157,12 +158,13 @@ allowed_regexp_prefixes:
 - dev/
 - staging/`,
 			exp: raw.RepoCfg{
-				Version:       Int(3),
-				AutoDiscover:  &raw.AutoDiscover{Mode: &autoDiscoverEnabled},
-				Automerge:     Bool(true),
-				ParallelApply: Bool(true),
-				ParallelPlan:  Bool(false),
-				RepoLocks:     &raw.RepoLocks{Mode: &repoLocksOnApply},
+				Version:                   Int(3),
+				AutoDiscover:              &raw.AutoDiscover{Mode: &autoDiscoverEnabled},
+				Automerge:                 Bool(true),
+				ParallelApply:             Bool(true),
+				ParallelPlan:              Bool(false),
+				RepoLocks:                 &raw.RepoLocks{Mode: &repoLocksOnApply},
+				LockAllProjectsBeforeExec: Bool(true),
 				Projects: []raw.Project{
 					{
 						Dir:              String("mydir"),

server/core/config/valid/repo_cfg.go

@@ -30,6 +30,7 @@ type RepoCfg struct {
 	AllowedRegexpPrefixes      []string
 	AbortOnExcecutionOrderFail bool
 	SilencePRComments          []string
+	LockAllProjectsBeforeExec  bool
 }
 
 func (r RepoCfg) FindProjectsByDirWorkspace(repoRelDir string, workspace string) []Project {

server/events/apply_command_runner.go

@@ -7,13 +7,18 @@ import (
 	"github.com/runatlantis/atlantis/server/events/vcs"
 )
 
+type ProjectCommandRunnerForApply interface {
+	ProjectApplyCommandRunner
+	ProjectLockCommandRunner
+}
+
 func NewApplyCommandRunner(
 	vcsClient vcs.Client,
 	disableApplyAll bool,
 	applyCommandLocker locking.ApplyLockChecker,
 	commitStatusUpdater CommitStatusUpdater,
 	prjCommandBuilder ProjectApplyCommandBuilder,
-	prjCmdRunner ProjectApplyCommandRunner,
+	prjCmdRunner ProjectCommandRunnerForApply,
 	autoMerger *AutoMerger,
 	pullUpdater *PullUpdater,
 	dbUpdater *DBUpdater,
@@ -48,7 +53,7 @@ type ApplyCommandRunner struct {
 	vcsClient            vcs.Client
 	commitStatusUpdater  CommitStatusUpdater
 	prjCmdBuilder        ProjectApplyCommandBuilder
-	prjCmdRunner         ProjectApplyCommandRunner
+	prjCmdRunner         ProjectCommandRunnerForApply
 	autoMerger           *AutoMerger
 	pullUpdater          *PullUpdater
 	dbUpdater            *DBUpdater
@@ -158,6 +163,13 @@ func (a *ApplyCommandRunner) Run(ctx *command.Context, cmd *CommentCommand) {
 		return
 	}
 
+	if len(projectCmds) > 0 && projectCmds[0].LockAllProjectsBeforeExec {
+		ctx.Log.Debug("locking all projects before running apply")
+		if !a.lockAllProjects(ctx, cmd, projectCmds) {
+			return
+		}
+	}
+
 	// Only run commands in parallel if enabled
 	var result command.Result
 	if a.isParallelEnabled(projectCmds) {
@@ -224,6 +236,30 @@ func (a *ApplyCommandRunner) updateCommitStatus(ctx *command.Context, pullStatus
 	}
 }
 
+func (a *ApplyCommandRunner) lockAllProjects(ctx *command.Context, cmd PullCommand, projectCmds []command.ProjectContext) bool {
+	result := runProjectCmds(projectCmds, a.prjCmdRunner.Lock)
+	if result.HasErrors() {
+		ctx.Log.Err("failed to lock all projects before running apply")
+
+		a.pullUpdater.updatePull(ctx, cmd, result)
+
+		if err := a.commitStatusUpdater.UpdateCombinedCount(
+			ctx.Log,
+			ctx.Pull.BaseRepo,
+			ctx.Pull,
+			models.FailedCommitStatus,
+			command.Apply,
+			0,
+			0,
+		); err != nil {
+			ctx.Log.Warn("unable to update commit status: %s", err)
+		}
+
+		return false
+	}
+	return true
+}
+
 // applyAllDisabledComment is posted when apply all commands (i.e. "atlantis apply")
 // are disabled and an apply all command is issued.
 var applyAllDisabledComment = "**Error:** Running `atlantis apply` without flags is disabled." +

server/events/command/name.go

@@ -30,6 +30,8 @@ const (
 	Import
 	// State is a command to run terraform state rm
 	State
+	// LockCmd is command to acquire lock
+	LockCmd
 	// Adding more? Don't forget to update String() below
 )
 
@@ -74,6 +76,8 @@ func (c Name) String() string {
 		return "import"
 	case State:
 		return "state"
+	case LockCmd:
+		return "lock"
 	}
 	return ""
 }
@@ -149,6 +153,8 @@ func ParseCommandName(name string) (Name, error) {
 		return Import, nil
 	case "state":
 		return State, nil
+	case "lock":
+		return LockCmd, nil
 	}
 	return -1, fmt.Errorf("unknown command name: %s", name)
 }

server/events/command/project_context.go

@@ -126,6 +126,8 @@ type ProjectContext struct {
 	// Allows custom policy check tools outside of Conftest to run in checks
 	CustomPolicyCheck bool
 	SilencePRComments []string
+	// LockAllProjectsBeforeExec controls how to lock projects, if true all projects will be locked before plan/apply
+	LockAllProjectsBeforeExec bool
 }
 
 // SetProjectScopeTags adds ProjectContext tags to a new returned scope.

server/events/command/project_result.go

@@ -18,6 +18,7 @@ type ProjectResult struct {
 	VersionSuccess     string
 	ImportSuccess      *models.ImportSuccess
 	StateRmSuccess     *models.StateRmSuccess
+	LockSuccess        *bool
 	ProjectName        string
 	SilencePRComments  []string
 }
@@ -83,5 +84,8 @@ func (p ProjectResult) PlanStatus() models.ProjectPlanStatus {
 
 // IsSuccessful returns true if this project result had no errors.
 func (p ProjectResult) IsSuccessful() bool {
-	return p.PlanSuccess != nil || (p.PolicyCheckResults != nil && p.Error == nil && p.Failure == "") || p.ApplySuccess != ""
+	return p.PlanSuccess != nil ||
+		(p.PolicyCheckResults != nil && p.Error == nil && p.Failure == "") ||
+		p.ApplySuccess != "" ||
+		(p.LockSuccess != nil && *p.LockSuccess)
 }

server/events/command/project_result_test.go

@@ -10,6 +10,7 @@ import (
 )
 
 func TestProjectResult_IsSuccessful(t *testing.T) {
+	trueVal := true
 	cases := map[string]struct {
 		pr  command.ProjectResult
 		exp bool
@@ -32,6 +33,12 @@ func TestProjectResult_IsSuccessful(t *testing.T) {
 			},
 			true,
 		},
+		"lock success": {
+			command.ProjectResult{
+				LockSuccess: &trueVal,
+			},
+			true,
+		},
 		"failure": {
 			command.ProjectResult{
 				Failure: "failure",

server/events/instrumented_project_command_runner.go

@@ -13,6 +13,7 @@ type IntrumentedCommandRunner interface {
 	ApprovePolicies(ctx command.ProjectContext) command.ProjectResult
 	Import(ctx command.ProjectContext) command.ProjectResult
 	StateRm(ctx command.ProjectContext) command.ProjectResult
+	Lock(ctx command.ProjectContext) command.ProjectResult
 }
 
 type InstrumentedProjectCommandRunner struct {
@@ -58,6 +59,10 @@ func (p *InstrumentedProjectCommandRunner) StateRm(ctx command.ProjectContext) c
 	return RunAndEmitStats(ctx, p.projectCommandRunner.StateRm, p.scope)
 }
 
+func (p *InstrumentedProjectCommandRunner) Lock(ctx command.ProjectContext) command.ProjectResult {
+	return RunAndEmitStats(ctx, p.projectCommandRunner.Lock, p.scope)
+}
+
 func RunAndEmitStats(ctx command.ProjectContext, execute func(ctx command.ProjectContext) command.ProjectResult, scope tally.Scope) command.ProjectResult {
 	commandName := ctx.CommandName.String()
 	// ensures we are differentiating between project level command and overall command

server/events/markdown_renderer.go

@@ -315,6 +315,10 @@ func (m *MarkdownRenderer) renderProjectResults(ctx *command.Context, results []
 			} else {
 				resultData.Rendered = m.renderTemplateTrimSpace(templates.Lookup("stateRmSuccessUnwrapped"), result.StateRmSuccess)
 			}
+		} else if result.LockSuccess != nil {
+			if !*result.LockSuccess {
+				continue // ignore successful locks
+			}
 			// Error out if no template was found, only if there are no errors or failures.
 			// This is because some errors and failures rely on additional context rendered by templtes, but not all errors or failures.
 		} else if !(result.Error != nil || result.Failure != "") {

server/events/markdown_renderer_test.go

@@ -58,6 +58,12 @@ func TestRenderErr(t *testing.T) {
 			fmt.Errorf("some conftest error"),
 			"**Policy Check Error**\n```\nsome conftest error\n```",
 		},
+		{
+			"lock error",
+			command.LockCmd,
+			err,
+			"**Lock Error**\n```\nerr\n```",
+		},
 	}
 
 	r := events.NewMarkdownRenderer(false, false, false, false, false, false, "", "atlantis", false)
@@ -122,6 +128,12 @@ func TestRenderFailure(t *testing.T) {
 			"failure",
 			"**Policy Check Failed**: failure",
 		},
+		{
+			"lock failure",
+			command.LockCmd,
+			"failure",
+			"**Lock Failed**: failure",
+		},
 	}
 
 	r := events.NewMarkdownRenderer(false, false, false, false, false, false, "", "atlantis", false)