From bac0f1384bc6252f075ae76342071233e381409b Mon Sep 17 00:00:00 2001
From: Nick Burgan <nick.burgan@exygy.com>
Date: Mon, 21 Aug 2023 14:16:19 -0700
Subject: [PATCH 01/13] Add ability to force unlock TF state

Sometimes, if a failure occurs or you've Ctrl+C'ed out of the middle of
an apply, the Terraform state may be left in a locked state. When this
happens, you must run 'terraform force-unlock' with the lock ID to
unlock it.

This change detects when an apply results in a lock error, and prompts
the user to rerun the command with the --force-unlock flag. It does this
by capturing the stderr stream of the 'terraform apply' command to
inspect the message if an error occurs. An added bonus is that since we
are using Popen to execute the command, we can handle things more
gracefully if a user Ctrl+Cs out of the apply and allow Terraform to
release the lock, reducing the amount of ways we can get into this
failure state.

In order to support the --force-unlock flag, there will be a change to
the civiform-deploy repo as well.
---
 cloud/shared/bin/lib/terraform.py | 91 +++++++++++++++++++++++++------
 cloud/shared/bin/run              | 11 +++-
 cloud/shared/bin/run.py           | 10 ++++
 3 files changed, 93 insertions(+), 19 deletions(-)

diff --git a/cloud/shared/bin/lib/terraform.py b/cloud/shared/bin/lib/terraform.py
index 56554f87..17ee7818 100644
--- a/cloud/shared/bin/lib/terraform.py
+++ b/cloud/shared/bin/lib/terraform.py
@@ -1,12 +1,64 @@
 import subprocess
 import os
+import re
 import shutil
 import shlex
+import inspect
 from typing import Optional
 
 from cloud.shared.bin.lib.config_loader import ConfigLoader
 from cloud.shared.bin.lib.print import print
 
+def force_unlock(config_loader: ConfigLoader, 
+                 lock_id: str,
+                 terraform_template_dir: Optional[str] = None):
+    if not terraform_template_dir:
+        terraform_template_dir = config_loader.get_template_dir()
+    
+    perform_init(config_loader, terraform_template_dir, False)
+
+    terraform_cmd = f'terraform -chdir={terraform_template_dir} force-unlock -force {lock_id}'
+    print(f" - Run {terraform_cmd}")
+    subprocess.check_call(shlex.split(terraform_cmd))
+    return True
+
+def perform_init(config_loader: ConfigLoader,
+                 terraform_template_dir: Optional[str] = None,
+                 upgrade: bool = True):
+    if not terraform_template_dir:
+        terraform_template_dir = config_loader.get_template_dir()
+    
+    init_cmd = f'terraform -chdir={terraform_template_dir} init'
+    if upgrade:
+        init_cmd += ' -upgrade'
+
+    if config_loader.use_local_backend:
+        init_cmd += ' -reconfigure'
+    else:
+        init_cmd += ' -input=false'
+        # backend vars file can be absent when pre-terraform setup is running
+        if os.path.exists(os.path.join(terraform_template_dir,
+                                       config_loader.backend_vars_filename)):
+            init_cmd += f' -backend-config={config_loader.backend_vars_filename}'
+    print(f" - Run {init_cmd}")
+    subprocess.check_call(shlex.split(init_cmd))
+    
+# We specifically don't want to capture stdout here. When running in interactive mode,
+# we'd miss the prompt to enter "yes" to continue on a terraform apply, even if we're
+# printing each line as it comes in, since the line the prompt is on does not contain
+# a new line character.
+def capture_stderr(cmd):
+    popen = subprocess.Popen(shlex.split(cmd), stderr=subprocess.PIPE, bufsize=1, universal_newlines=True)
+    try:
+        exit_code = popen.wait()
+        _, stderr = popen.communicate()
+        if stderr:
+            print(stderr)
+        return stderr, exit_code
+    except KeyboardInterrupt:
+        # Allow terraform to gracefully exit if a user Ctrl+C's out of the command
+        #popen.terminate()
+        popen.kill()
 
 # TODO(#2741): When using this for Azure make sure to setup backend bucket prior to calling these functions.
 def perform_apply(
@@ -18,20 +70,7 @@ def perform_apply(
         terraform_template_dir = config_loader.get_template_dir()
     tf_vars_filename = config_loader.tfvars_filename
 
-    terraform_cmd = f'terraform -chdir={terraform_template_dir}'
-
-    if config_loader.use_local_backend:
-        print(' - Run terraform init -upgrade -reconfigure')
-        subprocess.check_call(
-            shlex.split(f'{terraform_cmd} init -upgrade -reconfigure'))
-    else:
-        print(' - Run terraform init -upgrade')
-        init_cmd = f'{terraform_cmd} init -input=false -upgrade'
-        # backend vars file can be absent when pre-terraform setup is running
-        if os.path.exists(os.path.join(terraform_template_dir,
-                                       config_loader.backend_vars_filename)):
-            init_cmd += f' -backend-config={config_loader.backend_vars_filename}'
-        subprocess.check_call(shlex.split(init_cmd))
+    perform_init(config_loader, terraform_template_dir)
 
     if os.path.exists(os.path.join(terraform_template_dir, tf_vars_filename)):
         print(
@@ -45,16 +84,34 @@ def perform_apply(
         print(" - Test. Not applying terraform.")
         return True
 
-    print(" - Run terraform apply")
     # Enable compact-warnings as we have a bunch of
     # "value of undeclared variables" warnings as some variables used in one
     # deployment (e.g. aws) but not the other.
-    terraform_apply_cmd = f'{terraform_cmd} apply -input=false -var-file={tf_vars_filename} -compact-warnings'
+    terraform_apply_cmd = f'terraform -chdir={terraform_template_dir} apply -input=false -var-file={tf_vars_filename} -compact-warnings'
     if config_loader.skip_confirmations:
         terraform_apply_cmd += ' -auto-approve'
     if is_destroy:
         terraform_apply_cmd += ' -destroy'
-    subprocess.check_call(shlex.split(terraform_apply_cmd))
+
+    print(f" - Run {terraform_apply_cmd}")
+
+    output, exit_code = capture_stderr(terraform_apply_cmd)
+    if exit_code:
+        if "Error acquiring the state lock" in output:
+            # Lock ID is a standard UUID v4 in the form 00000000-0000-0000-0000-000000000000
+            match = re.search(r'ID:\s+([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})',output)
+            error_text = inspect.cleandoc("""
+                                     The Terraform state lock can not be acquired.
+                                     This can happen if you are running a command in another process, or if another Terraform process exited prematurely. 
+                                     """)
+            if match:
+                print(error_text + f"\nIf you are sure there are no other Terraform processes running, this can be fixed by rerunning the same command with \"--force-unlock={match.group(match.lastindex)}\"")
+            else:
+                print(error_text + "\nWe were unable to extract the lock ID from the error text. Inspect the error message above."
+                      "\nIf you are sure there are no other Terraform processes running, this error can be fixed by rerunning the same command with \"--force-unlock=<Lock ID>\""
+                )
+        exit(exit_code)
+
     return True
 
 
diff --git a/cloud/shared/bin/run b/cloud/shared/bin/run
index eaccba21..4e355f1b 100755
--- a/cloud/shared/bin/run
+++ b/cloud/shared/bin/run
@@ -19,7 +19,7 @@ set -o pipefail
 source cloud/shared/bin/python_env_setup
 
 # Get the arguments that we want to pass to run.py
-while getopts s:c:t: flag; do
+while getopts s:c:t:u: flag; do
   case "${flag}" in
     # The civiform_config file that contains the values to configure the deployment
     s) source_config=${OPTARG} ;;
@@ -27,6 +27,8 @@ while getopts s:c:t: flag; do
     c) command=${OPTARG} ;;
     # The tag of the image that should be used for this deployment (e.g. "latest")
     t) tag=${OPTARG} ;;
+    # The Terraform lock ID to force unlock
+    u) force_unlock_id=${OPTARG} ;;
   esac
 done
 
@@ -115,4 +117,9 @@ echo "env-var-docs @ git+https://github.com/civiform/civiform.git@${commit_sha}\
 
 initialize_python_env $dependencies_file_path
 
-cloud/shared/bin/run.py --command $command --tag $tag --config $source_config
+force_unlock_arg=""
+if [[ -n "${force_unlock_id}" ]]; then
+  force_unlock_arg="--force-unlock ${force_unlock_id}"
+fi
+
+cloud/shared/bin/run.py --command $command --tag $tag --config $source_config $force_unlock_arg
diff --git a/cloud/shared/bin/run.py b/cloud/shared/bin/run.py
index 63a47917..0aa4f958 100755
--- a/cloud/shared/bin/run.py
+++ b/cloud/shared/bin/run.py
@@ -14,6 +14,7 @@
 from cloud.shared.bin.lib.print import print
 from cloud.shared.bin.lib.write_tfvars import TfVarWriter
 from cloud.shared.bin.lib import backend_setup
+from cloud.shared.bin.lib import terraform
 
 _CIVIFORM_RELEASE_TAG_REGEX = re.compile(r'^v?[0-9]+\.[0-9]+\.[0-9]+$')
 
@@ -32,6 +33,9 @@ def main():
         '--config',
         default='civiform_config.sh',
         help='Path to CiviForm deployment config file.')
+    parser.add_argument(
+        '--force-unlock',
+        help='Lock ID to force unlock before performing the Terraform apply.')
 
     args = parser.parse_args()
     if args.tag:
@@ -55,6 +59,12 @@ def main():
     # Setup backend
     backend_setup.setup_backend(config)
 
+    # Run the command to force unlock the TF state lock
+    if args.force_unlock:
+        print("Force unlocking the Terraform state")
+        terraform.force_unlock(config, args.force_unlock)
+
+
     # Write the passthrough vars to a temporary file
     print("Writing TF Vars file")
     terraform_tfvars_path = os.path.join(

From 3eefba448bd9803bbfdebb899dd4558a14beb6de Mon Sep 17 00:00:00 2001
From: Nick Burgan <nick.burgan@exygy.com>
Date: Tue, 22 Aug 2023 12:29:25 -0700
Subject: [PATCH 02/13] Add ability to fix digest value of lock file in
 DynamoDB

If something goes wrong during deployment, especially when a user has force-unlocked due to a previous issue and then multiple apply actions are happening at once, the digest value for the Terraform lock file in S3 can be incorrect. This lets us set the digest value to the correct value, as given by the error message of a previous Terraform command, without having to go into the AWS console to set it manually.
---
 cloud/aws/templates/aws_oidc/bin/aws_cli.py | 28 ++++++++++++++++++---
 cloud/shared/bin/lib/terraform.py           | 11 +++++++-
 cloud/shared/bin/run                        | 15 ++++++++---
 cloud/shared/bin/run.py                     |  9 +++++++
 4 files changed, 55 insertions(+), 8 deletions(-)

diff --git a/cloud/aws/templates/aws_oidc/bin/aws_cli.py b/cloud/aws/templates/aws_oidc/bin/aws_cli.py
index 36b7d9a5..98dba844 100644
--- a/cloud/aws/templates/aws_oidc/bin/aws_cli.py
+++ b/cloud/aws/templates/aws_oidc/bin/aws_cli.py
@@ -109,6 +109,23 @@ def wait_for_ecs_service_healthy(self):
             )
             time.sleep(30)
 
+    def fix_digest_value(self, value):
+        """
+        Sets the lock file digest value in DynamoDB to the given value.
+
+        If something goes wrong during deployment, especially when a user has
+        force-unlocked due to a previous issue and then multiple apply actions
+        are happening at once, the digest value for the Terraform lock file in
+        S3 can be incorrect. This function lets us set the digest value to
+        the correct value, as given by the error message of a previous
+        Terraform command, without having to go into the AWS console to
+        set it manually.
+        """
+        table=f'{self.config.app_prefix}-{resources.S3_TERRAFORM_LOCK_TABLE}'
+        file=f'{self.config.app_prefix}-{resources.S3_TERRAFORM_STATE_BUCKET}'
+        command=f'dynamodb put-item --table-name={table} --item=\'{{"LockID":{{"S":"{file}/tfstate/terraform.tfstate-md5"}},"Digest":{{"S":"{value}"}}}}\''
+        self._call_cli(command, False)
+
     def _ecs_service_state(self) -> Dict:
         """
         Returns the ID and rolloutState of the PRIMARY ECS service deployment. If
@@ -169,7 +186,12 @@ def get_url_of_secret(self, secret_name: str) -> str:
     def get_url_of_s3_bucket(self, bucket_name: str) -> str:
         return f"https://{self.config.aws_region}.console.aws.amazon.com/s3/buckets/{bucket_name}"
 
-    def _call_cli(self, command: str) -> Dict:
-        command = f"aws --output=json --region={self.config.aws_region} " + command
+    def _call_cli(self, command: str, output: bool = True) -> Dict:
+        base = f"aws --region={self.config.aws_region} "
+        if output:
+            base += "--output=json "
+        command = base + command
         out = subprocess.check_output(shlex.split(command))
-        return json.loads(out.decode("ascii"))
+        if output:
+            return json.loads(out.decode("ascii"))
+        return
diff --git a/cloud/shared/bin/lib/terraform.py b/cloud/shared/bin/lib/terraform.py
index 17ee7818..d0f90240 100644
--- a/cloud/shared/bin/lib/terraform.py
+++ b/cloud/shared/bin/lib/terraform.py
@@ -41,7 +41,16 @@ def perform_init(config_loader: ConfigLoader,
                                        config_loader.backend_vars_filename)):
             init_cmd += f' -backend-config={config_loader.backend_vars_filename}'
     print(f" - Run {init_cmd}")
-    subprocess.check_call(shlex.split(init_cmd))
+    #subprocess.check_call(shlex.split(init_cmd))
+    output, exit_code = capture_stderr(init_cmd)
+    if exit_code:
+        # This is AWS-specific, and should be modified when we have actual
+        # Azure deployments
+        if 'state data in S3 does not have the expected content' in output:
+            match = re.search(r'value: ([0-9a-f]{32})', output)
+            if match:
+                print(f"To fix the above error, rerun this command with \"--fix-digest={match.group(match.lastindex)}\"")
+            exit(exit_code)
     
 # We specifically don't want to capture stdout here. When running in interactive mode,
 # we'd miss the prompt to enter "yes" to continue on a terraform apply, even if we're
diff --git a/cloud/shared/bin/run b/cloud/shared/bin/run
index 4e355f1b..f8d4a3b0 100755
--- a/cloud/shared/bin/run
+++ b/cloud/shared/bin/run
@@ -19,7 +19,7 @@ set -o pipefail
 source cloud/shared/bin/python_env_setup
 
 # Get the arguments that we want to pass to run.py
-while getopts s:c:t:u: flag; do
+while getopts s:c:t:u:d: flag; do
   case "${flag}" in
     # The civiform_config file that contains the values to configure the deployment
     s) source_config=${OPTARG} ;;
@@ -29,6 +29,8 @@ while getopts s:c:t:u: flag; do
     t) tag=${OPTARG} ;;
     # The Terraform lock ID to force unlock
     u) force_unlock_id=${OPTARG} ;;
+    # The Digest value for the DynamoDB table to set
+    d) digest_value=${OPTARG} ;;
   esac
 done
 
@@ -117,9 +119,14 @@ echo "env-var-docs @ git+https://github.com/civiform/civiform.git@${commit_sha}\
 
 initialize_python_env $dependencies_file_path
 
-force_unlock_arg=""
+args=("--command" "${command}" "--tag" "${tag}" "--config" "${source_config}")
+
 if [[ -n "${force_unlock_id}" ]]; then
-  force_unlock_arg="--force-unlock ${force_unlock_id}"
+  args=("${args[@]}" "--force-unlock" "${force_unlock_id}")
+fi
+
+if [[ -n "${digest_value}" ]]; then
+  args=("${args[@]}" "--fix-digest" "${digest_value}")
 fi
 
-cloud/shared/bin/run.py --command $command --tag $tag --config $source_config $force_unlock_arg
+cloud/shared/bin/run.py "${args[@]}"
diff --git a/cloud/shared/bin/run.py b/cloud/shared/bin/run.py
index 0aa4f958..5ab81d6c 100755
--- a/cloud/shared/bin/run.py
+++ b/cloud/shared/bin/run.py
@@ -15,6 +15,7 @@
 from cloud.shared.bin.lib.write_tfvars import TfVarWriter
 from cloud.shared.bin.lib import backend_setup
 from cloud.shared.bin.lib import terraform
+from cloud.aws.templates.aws_oidc.bin.aws_cli import AwsCli
 
 _CIVIFORM_RELEASE_TAG_REGEX = re.compile(r'^v?[0-9]+\.[0-9]+\.[0-9]+$')
 
@@ -36,6 +37,9 @@ def main():
     parser.add_argument(
         '--force-unlock',
         help='Lock ID to force unlock before performing the Terraform apply.')
+    parser.add_argument(
+        '--fix-digest',
+        help='Digest value to set in the DynamoDB table to fix when an error occured and this value was not updated on a previous deploy. Only works on AWS deployments.')
 
     args = parser.parse_args()
     if args.tag:
@@ -64,6 +68,11 @@ def main():
         print("Force unlocking the Terraform state")
         terraform.force_unlock(config, args.force_unlock)
 
+    if args.fix_digest:
+        print(f"Fixing the lock file digest value in DynamoDB, setting it to {args.fix_digest}")
+        aws = AwsCli(config)
+        aws.fix_digest_value(args.fix_digest)
+
 
     # Write the passthrough vars to a temporary file
     print("Writing TF Vars file")

From b444d9e4ad3aa771533f2f6d44c0ab1c6ce177ba Mon Sep 17 00:00:00 2001
From: Nick Burgan <nick.burgan@exygy.com>
Date: Tue, 22 Aug 2023 15:36:25 -0700
Subject: [PATCH 03/13] Add bin/fmt script

This currently relies on the binaries being installed locally. We may want to move to using the civiform formatter container instead, or something like it, in the future.
---
 bin/fmt | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100755 bin/fmt

diff --git a/bin/fmt b/bin/fmt
new file mode 100755
index 00000000..22b349db
--- /dev/null
+++ b/bin/fmt
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# DOC: Format terraform, python, and shell script files.
+
+if which -s terraform; then
+  echo "Formatting Terraform files"
+  terraform fmt -recursive -write
+else
+  echo "Terraform not found. Skipping."
+fi
+
+if which -s shfmt; then
+  echo "Formatting shell scripts"
+  shfmt -bn -ci -i 2 -w -l $(shfmt -f .)
+else
+  echo "Shfmt not found. Skipping."
+fi
+
+if which -s yapf; then
+  echo "Formatting python files"
+  yapf \
+    --verbose \
+    --style='{based_on_style: google, SPLIT_BEFORE_FIRST_ARGUMENT:true}' \
+    --in-place \
+    --recursive \
+    .
+else
+  echo "Yapf not found. Skipping."
+fi
\ No newline at end of file

From 8defbc24f0071413ec2b61754e4ff99cbc1ffca8 Mon Sep 17 00:00:00 2001
From: Nick Burgan <nick.burgan@exygy.com>
Date: Wed, 23 Aug 2023 09:54:51 -0700
Subject: [PATCH 04/13] Formatting fixes

---
 bin/fmt                                     |  2 +-
 cloud/aws/templates/aws_oidc/bin/aws_cli.py |  6 +--
 cloud/shared/bin/lib/terraform.py           | 52 ++++++++++++++-------
 cloud/shared/bin/run.py                     |  9 ++--
 4 files changed, 46 insertions(+), 23 deletions(-)

diff --git a/bin/fmt b/bin/fmt
index 22b349db..4587ede0 100755
--- a/bin/fmt
+++ b/bin/fmt
@@ -26,4 +26,4 @@ if which -s yapf; then
     .
 else
   echo "Yapf not found. Skipping."
-fi
\ No newline at end of file
+fi
diff --git a/cloud/aws/templates/aws_oidc/bin/aws_cli.py b/cloud/aws/templates/aws_oidc/bin/aws_cli.py
index 98dba844..e2c746ef 100644
--- a/cloud/aws/templates/aws_oidc/bin/aws_cli.py
+++ b/cloud/aws/templates/aws_oidc/bin/aws_cli.py
@@ -121,9 +121,9 @@ def fix_digest_value(self, value):
         Terraform command, without having to go into the AWS console to
         set it manually.
         """
-        table=f'{self.config.app_prefix}-{resources.S3_TERRAFORM_LOCK_TABLE}'
-        file=f'{self.config.app_prefix}-{resources.S3_TERRAFORM_STATE_BUCKET}'
-        command=f'dynamodb put-item --table-name={table} --item=\'{{"LockID":{{"S":"{file}/tfstate/terraform.tfstate-md5"}},"Digest":{{"S":"{value}"}}}}\''
+        table = f'{self.config.app_prefix}-{resources.S3_TERRAFORM_LOCK_TABLE}'
+        file = f'{self.config.app_prefix}-{resources.S3_TERRAFORM_STATE_BUCKET}'
+        command = f'dynamodb put-item --table-name={table} --item=\'{{"LockID":{{"S":"{file}/tfstate/terraform.tfstate-md5"}},"Digest":{{"S":"{value}"}}}}\''
         self._call_cli(command, False)
 
     def _ecs_service_state(self) -> Dict:
diff --git a/cloud/shared/bin/lib/terraform.py b/cloud/shared/bin/lib/terraform.py
index d0f90240..1cc8ed13 100644
--- a/cloud/shared/bin/lib/terraform.py
+++ b/cloud/shared/bin/lib/terraform.py
@@ -9,12 +9,14 @@
 from cloud.shared.bin.lib.config_loader import ConfigLoader
 from cloud.shared.bin.lib.print import print
 
-def force_unlock(config_loader: ConfigLoader, 
-                 lock_id: str,
-                 terraform_template_dir: Optional[str] = None):
+
+def force_unlock(
+        config_loader: ConfigLoader,
+        lock_id: str,
+        terraform_template_dir: Optional[str] = None):
     if not terraform_template_dir:
         terraform_template_dir = config_loader.get_template_dir()
-    
+
     perform_init(config_loader, terraform_template_dir, False)
 
     terraform_cmd = f'terraform -chdir={terraform_template_dir} force-unlock -force {lock_id}'
@@ -22,12 +24,14 @@ def force_unlock(config_loader: ConfigLoader,
     subprocess.check_call(shlex.split(terraform_cmd))
     return True
 
-def perform_init(config_loader: ConfigLoader,
-                 terraform_template_dir: Optional[str] = None,
-                 upgrade: bool = True):
+
+def perform_init(
+        config_loader: ConfigLoader,
+        terraform_template_dir: Optional[str] = None,
+        upgrade: bool = True):
     if not terraform_template_dir:
         terraform_template_dir = config_loader.get_template_dir()
-    
+
     init_cmd = f'terraform -chdir={terraform_template_dir} init'
     if upgrade:
         init_cmd += ' -upgrade'
@@ -49,15 +53,22 @@ def perform_init(config_loader: ConfigLoader,
         if 'state data in S3 does not have the expected content' in output:
             match = re.search(r'value: ([0-9a-f]{32})', output)
             if match:
-                print(f"To fix the above error, rerun this command with \"--fix-digest={match.group(match.lastindex)}\"")
+                print(
+                    f"To fix the above error, rerun this command with \"--fix-digest={match.group(match.lastindex)}\""
+                )
             exit(exit_code)
-    
+
+
 # We specifically don't want to capture stdout here. When running in interactive mode,
 # we'd miss the prompt to enter "yes" to continue on a terraform apply, even if we're
 # printing each line as it comes in, since the line the prompt is on does not contain
 # a new line character.
 def capture_stderr(cmd):
-    popen = subprocess.Popen(shlex.split(cmd), stderr=subprocess.PIPE, bufsize=1, universal_newlines=True)
+    popen = subprocess.Popen(
+        shlex.split(cmd),
+        stderr=subprocess.PIPE,
+        bufsize=1,
+        universal_newlines=True)
     try:
         exit_code = popen.wait()
         _, stderr = popen.communicate()
@@ -69,6 +80,7 @@ def capture_stderr(cmd):
         #popen.terminate()
         popen.kill()
 
+
 # TODO(#2741): When using this for Azure make sure to setup backend bucket prior to calling these functions.
 def perform_apply(
         config_loader: ConfigLoader,
@@ -108,16 +120,24 @@ def perform_apply(
     if exit_code:
         if "Error acquiring the state lock" in output:
             # Lock ID is a standard UUID v4 in the form 00000000-0000-0000-0000-000000000000
-            match = re.search(r'ID:\s+([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})',output)
-            error_text = inspect.cleandoc("""
+            match = re.search(
+                    r'ID:\s+([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})',
+                    output)
+            error_text = inspect.cleandoc(
+                """
                                      The Terraform state lock can not be acquired.
                                      This can happen if you are running a command in another process, or if another Terraform process exited prematurely. 
                                      """)
             if match:
-                print(error_text + f"\nIf you are sure there are no other Terraform processes running, this can be fixed by rerunning the same command with \"--force-unlock={match.group(match.lastindex)}\"")
+                print(
+                    error_text +
+                    f"\nIf you are sure there are no other Terraform processes running, this can be fixed by rerunning the same command with \"--force-unlock={match.group(match.lastindex)}\""
+                )
             else:
-                print(error_text + "\nWe were unable to extract the lock ID from the error text. Inspect the error message above."
-                      "\nIf you are sure there are no other Terraform processes running, this error can be fixed by rerunning the same command with \"--force-unlock=<Lock ID>\""
+                print(
+                    error_text +
+                    "\nWe were unable to extract the lock ID from the error text. Inspect the error message above."
+                    "\nIf you are sure there are no other Terraform processes running, this error can be fixed by rerunning the same command with \"--force-unlock=<Lock ID>\""
                 )
         exit(exit_code)
 
diff --git a/cloud/shared/bin/run.py b/cloud/shared/bin/run.py
index 5ab81d6c..dfcabf02 100755
--- a/cloud/shared/bin/run.py
+++ b/cloud/shared/bin/run.py
@@ -39,7 +39,9 @@ def main():
         help='Lock ID to force unlock before performing the Terraform apply.')
     parser.add_argument(
         '--fix-digest',
-        help='Digest value to set in the DynamoDB table to fix when an error occured and this value was not updated on a previous deploy. Only works on AWS deployments.')
+        help=
+        'Digest value to set in the DynamoDB table to fix when an error occured and this value was not updated on a previous deploy. Only works on AWS deployments.'
+    )
 
     args = parser.parse_args()
     if args.tag:
@@ -69,11 +71,12 @@ def main():
         terraform.force_unlock(config, args.force_unlock)
 
     if args.fix_digest:
-        print(f"Fixing the lock file digest value in DynamoDB, setting it to {args.fix_digest}")
+        print(
+            f"Fixing the lock file digest value in DynamoDB, setting it to {args.fix_digest}"
+        )
         aws = AwsCli(config)
         aws.fix_digest_value(args.fix_digest)
 
-
     # Write the passthrough vars to a temporary file
     print("Writing TF Vars file")
     terraform_tfvars_path = os.path.join(

From d48cbb972f2b5450951558ebb4c3bd1e6708ceb2 Mon Sep 17 00:00:00 2001
From: Nick Burgan <nick.burgan@exygy.com>
Date: Wed, 23 Aug 2023 10:49:22 -0700
Subject: [PATCH 05/13] Remove debug stuff

---
 cloud/shared/bin/lib/terraform.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/cloud/shared/bin/lib/terraform.py b/cloud/shared/bin/lib/terraform.py
index 1cc8ed13..183ad612 100644
--- a/cloud/shared/bin/lib/terraform.py
+++ b/cloud/shared/bin/lib/terraform.py
@@ -45,7 +45,6 @@ def perform_init(
                                        config_loader.backend_vars_filename)):
             init_cmd += f' -backend-config={config_loader.backend_vars_filename}'
     print(f" - Run {init_cmd}")
-    #subprocess.check_call(shlex.split(init_cmd))
     output, exit_code = capture_stderr(init_cmd)
     if exit_code:
         # This is AWS-specific, and should be modified when we have actual
@@ -77,8 +76,7 @@ def capture_stderr(cmd):
         return stderr, exit_code
     except KeyboardInterrupt:
         # Allow terraform to gracefully exit if a user Ctrl+C's out of the command
-        #popen.terminate()
-        popen.kill()
+        popen.terminate()
 
 
 # TODO(#2741): When using this for Azure make sure to setup backend bucket prior to calling these functions.

From cf00e0892fb0df60f26ac5d390753c0f4486322a Mon Sep 17 00:00:00 2001
From: Nick Burgan <nick.burgan@exygy.com>
Date: Tue, 29 Aug 2023 13:31:02 -0700
Subject: [PATCH 06/13] Perform fixes during the run if running interactively

If we hit a TF lock state issue, or the S3 digest issue, this allows the
user to fix the problem during this particular invocation of the
command, so they do not need to rerun the command. If we are not running
in a TTY, we'll fall back to prompting the user to rerun the command
with the appropriate flag.
---
 cloud/shared/bin/lib/terraform.py | 53 +++++++++++++++++++++++--------
 1 file changed, 40 insertions(+), 13 deletions(-)

diff --git a/cloud/shared/bin/lib/terraform.py b/cloud/shared/bin/lib/terraform.py
index 183ad612..25578073 100644
--- a/cloud/shared/bin/lib/terraform.py
+++ b/cloud/shared/bin/lib/terraform.py
@@ -1,5 +1,6 @@
 import subprocess
 import os
+import sys
 import re
 import shutil
 import shlex
@@ -8,21 +9,23 @@
 
 from cloud.shared.bin.lib.config_loader import ConfigLoader
 from cloud.shared.bin.lib.print import print
+from cloud.aws.templates.aws_oidc.bin.aws_cli import AwsCli
 
 
 def force_unlock(
         config_loader: ConfigLoader,
         lock_id: str,
-        terraform_template_dir: Optional[str] = None):
+        terraform_template_dir: Optional[str] = None,
+        initialize = True):
     if not terraform_template_dir:
         terraform_template_dir = config_loader.get_template_dir()
 
-    perform_init(config_loader, terraform_template_dir, False)
+    if initialize:
+        perform_init(config_loader, terraform_template_dir, False)
 
     terraform_cmd = f'terraform -chdir={terraform_template_dir} force-unlock -force {lock_id}'
     print(f" - Run {terraform_cmd}")
     subprocess.check_call(shlex.split(terraform_cmd))
-    return True
 
 
 def perform_init(
@@ -46,16 +49,28 @@ def perform_init(
             init_cmd += f' -backend-config={config_loader.backend_vars_filename}'
     print(f" - Run {init_cmd}")
     output, exit_code = capture_stderr(init_cmd)
-    if exit_code:
+    if exit_code > 0:
+        is_tty = sys.stdin.isatty()
         # This is AWS-specific, and should be modified when we have actual
         # Azure deployments
         if 'state data in S3 does not have the expected content' in output:
             match = re.search(r'value: ([0-9a-f]{32})', output)
             if match:
+                digest = match.group(match.lastindex)
+                if is_tty:
+                    answer = input("Would you like to fix this by setting the correct digest value? Ensure that no other deployment processes are in progress. [Y/n] >")
+                    if answer.lower() in ['y', 'yes', '']:
+                        aws = AwsCli(config_loader)
+                        aws.fix_digest_value(digest)
+                        perform_init(config_loader, terraform_template_dir, upgrade)
+                        return
                 print(
                     f"To fix the above error, rerun this command with \"--fix-digest={match.group(match.lastindex)}\""
                 )
+            # Since we've handled the error and printed a message, exit immediately
+            # rather than returning False and having it print a stack trace.
             exit(exit_code)
+        raise RuntimeError("Unhandled error during terraform init. See error message above for details.")
 
 
 # We specifically don't want to capture stdout here. When running in interactive mode,
@@ -82,14 +97,16 @@ def capture_stderr(cmd):
 # TODO(#2741): When using this for Azure make sure to setup backend bucket prior to calling these functions.
 def perform_apply(
         config_loader: ConfigLoader,
-        is_destroy=False,
-        terraform_template_dir: Optional[str] = None):
+        is_destroy = False,
+        terraform_template_dir: Optional[str] = None,
+        initialize = True):
     '''Generates terraform variable files and runs terraform init and apply.'''
     if not terraform_template_dir:
         terraform_template_dir = config_loader.get_template_dir()
     tf_vars_filename = config_loader.tfvars_filename
 
-    perform_init(config_loader, terraform_template_dir)
+    if initialize:
+        perform_init(config_loader, terraform_template_dir)
 
     if os.path.exists(os.path.join(terraform_template_dir, tf_vars_filename)):
         print(
@@ -115,7 +132,8 @@ def perform_apply(
     print(f" - Run {terraform_apply_cmd}")
 
     output, exit_code = capture_stderr(terraform_apply_cmd)
-    if exit_code:
+    if exit_code > 0:
+        is_tty = sys.stdin.isatty()
         if "Error acquiring the state lock" in output:
             # Lock ID is a standard UUID v4 in the form 00000000-0000-0000-0000-000000000000
             match = re.search(
@@ -123,13 +141,19 @@ def perform_apply(
                     output)
             error_text = inspect.cleandoc(
                 """
-                                     The Terraform state lock can not be acquired.
-                                     This can happen if you are running a command in another process, or if another Terraform process exited prematurely. 
-                                     """)
+                The Terraform state lock can not be acquired.
+                This can happen if you are running a command in another process, or if another Terraform process exited prematurely. 
+                """)
             if match:
+                lock_id = match.group(match.lastindex)
+                if is_tty:
+                    answer = input("Would you like to fix this by force-unlocking the Terraform state? Ensure that no other deployment processes are in progress. [Y/n] >")
+                    if answer.lower() in ['y', 'yes', '']:
+                        force_unlock(config_loader, lock_id, terraform_template_dir, False)
+                        return perform_apply(config_loader, is_destroy, terraform_template_dir, False)
                 print(
                     error_text +
-                    f"\nIf you are sure there are no other Terraform processes running, this can be fixed by rerunning the same command with \"--force-unlock={match.group(match.lastindex)}\""
+                    f"\nIf you are sure there are no other Terraform processes running, this can be fixed by rerunning the same command with \"--force-unlock={lock_id}\""
                 )
             else:
                 print(
@@ -137,7 +161,10 @@ def perform_apply(
                     "\nWe were unable to extract the lock ID from the error text. Inspect the error message above."
                     "\nIf you are sure there are no other Terraform processes running, this error can be fixed by rerunning the same command with \"--force-unlock=<Lock ID>\""
                 )
-        exit(exit_code)
+            # Since we've handled the error and printed a message, exit immediately
+            # rather than returning False and having it print a stack trace.
+            exit(exit_code)
+        return False
 
     return True
 

From acffcdcd834bfb0edb6e52ee275f4ac1ca5401c6 Mon Sep 17 00:00:00 2001
From: Nick Burgan <nick.burgan@exygy.com>
Date: Tue, 29 Aug 2023 15:37:39 -0700
Subject: [PATCH 07/13] bin/fmt fixes

---
 bin/fmt | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/bin/fmt b/bin/fmt
index 4587ede0..ae38776e 100755
--- a/bin/fmt
+++ b/bin/fmt
@@ -1,4 +1,4 @@
-#!/bin/bash
+#! /usr/bin/env bash
 
 # DOC: Format terraform, python, and shell script files.
 
@@ -6,14 +6,16 @@ if which -s terraform; then
   echo "Formatting Terraform files"
   terraform fmt -recursive -write
 else
-  echo "Terraform not found. Skipping."
+  echo "Can not find the terraform binary. Please install Terraform first."
+  exit 1
 fi
 
 if which -s shfmt; then
   echo "Formatting shell scripts"
   shfmt -bn -ci -i 2 -w -l $(shfmt -f .)
 else
-  echo "Shfmt not found. Skipping."
+  echo "Could not find the shfmt binary. Please install shfmt first."
+  exit 1
 fi
 
 if which -s yapf; then
@@ -25,5 +27,6 @@ if which -s yapf; then
     --recursive \
     .
 else
-  echo "Yapf not found. Skipping."
+  echo "Could not find yapf. Please install it first."
+  exit 1
 fi

From 42235a7cd0f2a67c0fcff576b041f6121d971381 Mon Sep 17 00:00:00 2001
From: Nick Burgan <nick.burgan@exygy.com>
Date: Tue, 29 Aug 2023 16:50:50 -0700
Subject: [PATCH 08/13] Change fix digest terminology to
 'lock-table-digest-value'

---
 cloud/aws/templates/aws_oidc/bin/aws_cli.py | 5 +++--
 cloud/shared/bin/lib/terraform.py           | 4 ++--
 cloud/shared/bin/run                        | 2 +-
 cloud/shared/bin/run.py                     | 8 ++++----
 4 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/cloud/aws/templates/aws_oidc/bin/aws_cli.py b/cloud/aws/templates/aws_oidc/bin/aws_cli.py
index e2c746ef..c960d44c 100644
--- a/cloud/aws/templates/aws_oidc/bin/aws_cli.py
+++ b/cloud/aws/templates/aws_oidc/bin/aws_cli.py
@@ -109,9 +109,10 @@ def wait_for_ecs_service_healthy(self):
             )
             time.sleep(30)
 
-    def fix_digest_value(self, value):
+    def set_lock_table_digest_value(self, value):
         """
-        Sets the lock file digest value in DynamoDB to the given value.
+        Sets the lock file digest value in DynamoDB to the given value. This
+        digest value is a checksum of the Terraform state file stored in S3.
 
         If something goes wrong during deployment, especially when a user has
         force-unlocked due to a previous issue and then multiple apply actions
diff --git a/cloud/shared/bin/lib/terraform.py b/cloud/shared/bin/lib/terraform.py
index 25578073..4cc90a10 100644
--- a/cloud/shared/bin/lib/terraform.py
+++ b/cloud/shared/bin/lib/terraform.py
@@ -61,11 +61,11 @@ def perform_init(
                     answer = input("Would you like to fix this by setting the correct digest value? Ensure that no other deployment processes are in progress. [Y/n] >")
                     if answer.lower() in ['y', 'yes', '']:
                         aws = AwsCli(config_loader)
-                        aws.fix_digest_value(digest)
+                        aws.set_lock_table_digest_value(digest)
                         perform_init(config_loader, terraform_template_dir, upgrade)
                         return
                 print(
-                    f"To fix the above error, rerun this command with \"--fix-digest={match.group(match.lastindex)}\""
+                    f"To fix the above error, rerun this command with \"--lock-table-digest-value={match.group(match.lastindex)}\""
                 )
             # Since we've handled the error and printed a message, exit immediately
             # rather than returning False and having it print a stack trace.
diff --git a/cloud/shared/bin/run b/cloud/shared/bin/run
index f8d4a3b0..33b525fb 100755
--- a/cloud/shared/bin/run
+++ b/cloud/shared/bin/run
@@ -126,7 +126,7 @@ if [[ -n "${force_unlock_id}" ]]; then
 fi
 
 if [[ -n "${digest_value}" ]]; then
-  args=("${args[@]}" "--fix-digest" "${digest_value}")
+  args=("${args[@]}" "--lock-table-digest-value" "${digest_value}")
 fi
 
 cloud/shared/bin/run.py "${args[@]}"
diff --git a/cloud/shared/bin/run.py b/cloud/shared/bin/run.py
index dfcabf02..182762e6 100755
--- a/cloud/shared/bin/run.py
+++ b/cloud/shared/bin/run.py
@@ -38,7 +38,7 @@ def main():
         '--force-unlock',
         help='Lock ID to force unlock before performing the Terraform apply.')
     parser.add_argument(
-        '--fix-digest',
+        '--lock-table-digest-value',
         help=
         'Digest value to set in the DynamoDB table to fix when an error occured and this value was not updated on a previous deploy. Only works on AWS deployments.'
     )
@@ -70,12 +70,12 @@ def main():
         print("Force unlocking the Terraform state")
         terraform.force_unlock(config, args.force_unlock)
 
-    if args.fix_digest:
+    if args.lock_table_digest_value:
         print(
-            f"Fixing the lock file digest value in DynamoDB, setting it to {args.fix_digest}"
+            f"Fixing the lock file digest value in DynamoDB, setting it to {args.lock_table_digest_value}"
         )
         aws = AwsCli(config)
-        aws.fix_digest_value(args.fix_digest)
+        aws.set_lock_table_digest_value(args.lock_table_digest_value)
 
     # Write the passthrough vars to a temporary file
     print("Writing TF Vars file")

From cadc1607fa02a4e5b4e39c0d5f2da9cf075457d6 Mon Sep 17 00:00:00 2001
From: Nick Burgan <nick.burgan@exygy.com>
Date: Tue, 29 Aug 2023 17:04:06 -0700
Subject: [PATCH 09/13] Inline comments

---
 cloud/aws/templates/aws_oidc/bin/aws_cli.py | 2 +-
 cloud/shared/bin/lib/terraform.py           | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cloud/aws/templates/aws_oidc/bin/aws_cli.py b/cloud/aws/templates/aws_oidc/bin/aws_cli.py
index c960d44c..948c2134 100644
--- a/cloud/aws/templates/aws_oidc/bin/aws_cli.py
+++ b/cloud/aws/templates/aws_oidc/bin/aws_cli.py
@@ -125,7 +125,7 @@ def set_lock_table_digest_value(self, value):
         table = f'{self.config.app_prefix}-{resources.S3_TERRAFORM_LOCK_TABLE}'
         file = f'{self.config.app_prefix}-{resources.S3_TERRAFORM_STATE_BUCKET}'
         command = f'dynamodb put-item --table-name={table} --item=\'{{"LockID":{{"S":"{file}/tfstate/terraform.tfstate-md5"}},"Digest":{{"S":"{value}"}}}}\''
-        self._call_cli(command, False)
+        self._call_cli(command, False) # output = False
 
     def _ecs_service_state(self) -> Dict:
         """
diff --git a/cloud/shared/bin/lib/terraform.py b/cloud/shared/bin/lib/terraform.py
index 4cc90a10..5b254161 100644
--- a/cloud/shared/bin/lib/terraform.py
+++ b/cloud/shared/bin/lib/terraform.py
@@ -21,7 +21,7 @@ def force_unlock(
         terraform_template_dir = config_loader.get_template_dir()
 
     if initialize:
-        perform_init(config_loader, terraform_template_dir, False)
+        perform_init(config_loader, terraform_template_dir, False) # initialize = False
 
     terraform_cmd = f'terraform -chdir={terraform_template_dir} force-unlock -force {lock_id}'
     print(f" - Run {terraform_cmd}")
@@ -149,8 +149,8 @@ def perform_apply(
                 if is_tty:
                     answer = input("Would you like to fix this by force-unlocking the Terraform state? Ensure that no other deployment processes are in progress. [Y/n] >")
                     if answer.lower() in ['y', 'yes', '']:
-                        force_unlock(config_loader, lock_id, terraform_template_dir, False)
-                        return perform_apply(config_loader, is_destroy, terraform_template_dir, False)
+                        force_unlock(config_loader, lock_id, terraform_template_dir, False) # initialize = False
+                        return perform_apply(config_loader, is_destroy, terraform_template_dir, False) # initialize = False
                 print(
                     error_text +
                     f"\nIf you are sure there are no other Terraform processes running, this can be fixed by rerunning the same command with \"--force-unlock={lock_id}\""

From ce693fb26ebc4151da7f9d4dca47ba356bd7cced Mon Sep 17 00:00:00 2001
From: Nick Burgan <nick.burgan@exygy.com>
Date: Tue, 29 Aug 2023 17:16:20 -0700
Subject: [PATCH 10/13] Change to using env vars instead of flags when
 rerunning a command to fix a problem

This allows us not to have to change a CE's fork of civiform-deploy.
---
 cloud/shared/bin/lib/terraform.py |  8 ++++----
 cloud/shared/bin/run              | 12 ++++--------
 2 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/cloud/shared/bin/lib/terraform.py b/cloud/shared/bin/lib/terraform.py
index 5b254161..856d5da2 100644
--- a/cloud/shared/bin/lib/terraform.py
+++ b/cloud/shared/bin/lib/terraform.py
@@ -21,7 +21,7 @@ def force_unlock(
         terraform_template_dir = config_loader.get_template_dir()
 
     if initialize:
-        perform_init(config_loader, terraform_template_dir, False) # initialize = False
+        perform_init(config_loader, terraform_template_dir, False) # upgrade = False
 
     terraform_cmd = f'terraform -chdir={terraform_template_dir} force-unlock -force {lock_id}'
     print(f" - Run {terraform_cmd}")
@@ -65,7 +65,7 @@ def perform_init(
                         perform_init(config_loader, terraform_template_dir, upgrade)
                         return
                 print(
-                    f"To fix the above error, rerun this command with \"--lock-table-digest-value={match.group(match.lastindex)}\""
+                    f"To fix the above error, set the LOCK_TABLE_DIGEST_VALUE environment variable to \"{digest}\" and rerun this command."
                 )
             # Since we've handled the error and printed a message, exit immediately
             # rather than returning False and having it print a stack trace.
@@ -153,13 +153,13 @@ def perform_apply(
                         return perform_apply(config_loader, is_destroy, terraform_template_dir, False) # initialize = False
                 print(
                     error_text +
-                    f"\nIf you are sure there are no other Terraform processes running, this can be fixed by rerunning the same command with \"--force-unlock={lock_id}\""
+                    f"\nIf you are sure there are no other Terraform processes running, this can be fixed by setting the FORCE_UNLOCK_ID environment variable to \"{lock_id}\" and rerunning the same command."
                 )
             else:
                 print(
                     error_text +
                     "\nWe were unable to extract the lock ID from the error text. Inspect the error message above."
-                    "\nIf you are sure there are no other Terraform processes running, this error can be fixed by rerunning the same command with \"--force-unlock=<Lock ID>\""
+                    "\nIf you are sure there are no other Terraform processes running, this error can be fixed by setting the FORCE_UNLOCK_ID environment variable to the lock ID value, and then rerunning the same command."
                 )
             # Since we've handled the error and printed a message, exit immediately
             # rather than returning False and having it print a stack trace.
diff --git a/cloud/shared/bin/run b/cloud/shared/bin/run
index 33b525fb..b2d2cec4 100755
--- a/cloud/shared/bin/run
+++ b/cloud/shared/bin/run
@@ -27,10 +27,6 @@ while getopts s:c:t:u:d: flag; do
     c) command=${OPTARG} ;;
     # The tag of the image that should be used for this deployment (e.g. "latest")
     t) tag=${OPTARG} ;;
-    # The Terraform lock ID to force unlock
-    u) force_unlock_id=${OPTARG} ;;
-    # The Digest value for the DynamoDB table to set
-    d) digest_value=${OPTARG} ;;
   esac
 done
 
@@ -121,12 +117,12 @@ initialize_python_env $dependencies_file_path
 
 args=("--command" "${command}" "--tag" "${tag}" "--config" "${source_config}")
 
-if [[ -n "${force_unlock_id}" ]]; then
-  args=("${args[@]}" "--force-unlock" "${force_unlock_id}")
+if [[ -n "${FORCE_UNLOCK_ID}" ]]; then
+  args=("${args[@]}" "--force-unlock" "${FORCE_UNLOCK_ID}")
 fi
 
-if [[ -n "${digest_value}" ]]; then
-  args=("${args[@]}" "--lock-table-digest-value" "${digest_value}")
+if [[ -n "${LOCK_TABLE_DIGEST_VALUE}" ]]; then
+  args=("${args[@]}" "--lock-table-digest-value" "${LOCK_TABLE_DIGEST_VALUE}")
 fi
 
 cloud/shared/bin/run.py "${args[@]}"

From 1d0f21c9a7af61c7a4d84610fee17bf64a8b4fcb Mon Sep 17 00:00:00 2001
From: Nick Burgan <nick.burgan@exygy.com>
Date: Tue, 29 Aug 2023 17:22:01 -0700
Subject: [PATCH 11/13] Formatting fixes

---
 cloud/aws/templates/aws_oidc/bin/aws_cli.py |  2 +-
 cloud/shared/bin/lib/terraform.py           | 32 ++++++++++++++-------
 2 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/cloud/aws/templates/aws_oidc/bin/aws_cli.py b/cloud/aws/templates/aws_oidc/bin/aws_cli.py
index 948c2134..cb12d48a 100644
--- a/cloud/aws/templates/aws_oidc/bin/aws_cli.py
+++ b/cloud/aws/templates/aws_oidc/bin/aws_cli.py
@@ -125,7 +125,7 @@ def set_lock_table_digest_value(self, value):
         table = f'{self.config.app_prefix}-{resources.S3_TERRAFORM_LOCK_TABLE}'
         file = f'{self.config.app_prefix}-{resources.S3_TERRAFORM_STATE_BUCKET}'
         command = f'dynamodb put-item --table-name={table} --item=\'{{"LockID":{{"S":"{file}/tfstate/terraform.tfstate-md5"}},"Digest":{{"S":"{value}"}}}}\''
-        self._call_cli(command, False) # output = False
+        self._call_cli(command, False)  # output = False
 
     def _ecs_service_state(self) -> Dict:
         """
diff --git a/cloud/shared/bin/lib/terraform.py b/cloud/shared/bin/lib/terraform.py
index 856d5da2..6771e410 100644
--- a/cloud/shared/bin/lib/terraform.py
+++ b/cloud/shared/bin/lib/terraform.py
@@ -16,12 +16,13 @@ def force_unlock(
         config_loader: ConfigLoader,
         lock_id: str,
         terraform_template_dir: Optional[str] = None,
-        initialize = True):
+        initialize=True):
     if not terraform_template_dir:
         terraform_template_dir = config_loader.get_template_dir()
 
     if initialize:
-        perform_init(config_loader, terraform_template_dir, False) # upgrade = False
+        perform_init(
+            config_loader, terraform_template_dir, False)  # upgrade = False
 
     terraform_cmd = f'terraform -chdir={terraform_template_dir} force-unlock -force {lock_id}'
     print(f" - Run {terraform_cmd}")
@@ -58,11 +59,14 @@ def perform_init(
             if match:
                 digest = match.group(match.lastindex)
                 if is_tty:
-                    answer = input("Would you like to fix this by setting the correct digest value? Ensure that no other deployment processes are in progress. [Y/n] >")
+                    answer = input(
+                        "Would you like to fix this by setting the correct digest value? Ensure that no other deployment processes are in progress. [Y/n] >"
+                    )
                     if answer.lower() in ['y', 'yes', '']:
                         aws = AwsCli(config_loader)
                         aws.set_lock_table_digest_value(digest)
-                        perform_init(config_loader, terraform_template_dir, upgrade)
+                        perform_init(
+                            config_loader, terraform_template_dir, upgrade)
                         return
                 print(
                     f"To fix the above error, set the LOCK_TABLE_DIGEST_VALUE environment variable to \"{digest}\" and rerun this command."
@@ -70,7 +74,9 @@ def perform_init(
             # Since we've handled the error and printed a message, exit immediately
             # rather than returning False and having it print a stack trace.
             exit(exit_code)
-        raise RuntimeError("Unhandled error during terraform init. See error message above for details.")
+        raise RuntimeError(
+            "Unhandled error during terraform init. See error message above for details."
+        )
 
 
 # We specifically don't want to capture stdout here. When running in interactive mode,
@@ -97,9 +103,9 @@ def capture_stderr(cmd):
 # TODO(#2741): When using this for Azure make sure to setup backend bucket prior to calling these functions.
 def perform_apply(
         config_loader: ConfigLoader,
-        is_destroy = False,
+        is_destroy=False,
         terraform_template_dir: Optional[str] = None,
-        initialize = True):
+        initialize=True):
     '''Generates terraform variable files and runs terraform init and apply.'''
     if not terraform_template_dir:
         terraform_template_dir = config_loader.get_template_dir()
@@ -147,10 +153,16 @@ def perform_apply(
             if match:
                 lock_id = match.group(match.lastindex)
                 if is_tty:
-                    answer = input("Would you like to fix this by force-unlocking the Terraform state? Ensure that no other deployment processes are in progress. [Y/n] >")
+                    answer = input(
+                        "Would you like to fix this by force-unlocking the Terraform state? Ensure that no other deployment processes are in progress. [Y/n] >"
+                    )
                     if answer.lower() in ['y', 'yes', '']:
-                        force_unlock(config_loader, lock_id, terraform_template_dir, False) # initialize = False
-                        return perform_apply(config_loader, is_destroy, terraform_template_dir, False) # initialize = False
+                        force_unlock(
+                            config_loader, lock_id, terraform_template_dir,
+                            False)  # initialize = False
+                        return perform_apply(
+                            config_loader, is_destroy, terraform_template_dir,
+                            False)  # initialize = False
                 print(
                     error_text +
                     f"\nIf you are sure there are no other Terraform processes running, this can be fixed by setting the FORCE_UNLOCK_ID environment variable to \"{lock_id}\" and rerunning the same command."

From 468c9840f99bdf3cd1292ea2147a430169d393b2 Mon Sep 17 00:00:00 2001
From: Nick Burgan <nick.burgan@exygy.com>
Date: Tue, 29 Aug 2023 17:29:35 -0700
Subject: [PATCH 12/13] Make help wording for --lock-table-digest-value clearer

---
 cloud/shared/bin/run.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloud/shared/bin/run.py b/cloud/shared/bin/run.py
index 182762e6..359c683f 100755
--- a/cloud/shared/bin/run.py
+++ b/cloud/shared/bin/run.py
@@ -40,7 +40,7 @@ def main():
     parser.add_argument(
         '--lock-table-digest-value',
         help=
-        'Digest value to set in the DynamoDB table to fix when an error occured and this value was not updated on a previous deploy. Only works on AWS deployments.'
+        'Digest value for the Terraform lock table to set in DynamoDB. If multiple processes are doing a deploy, or an error occurred in a previous deploy that prevented Terraform from cleaning up after itself, this value may need updating. Only works on AWS deployments.'
     )
 
     args = parser.parse_args()

From a129878ab621b2588607e4fabee561844c8db212 Mon Sep 17 00:00:00 2001
From: Nick Burgan <nick.burgan@exygy.com>
Date: Wed, 30 Aug 2023 11:15:10 -0700
Subject: [PATCH 13/13] Comment for is_tty, and make env var fix suggestions
 clearer

---
 cloud/shared/bin/lib/terraform.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/cloud/shared/bin/lib/terraform.py b/cloud/shared/bin/lib/terraform.py
index 6771e410..1a378ac9 100644
--- a/cloud/shared/bin/lib/terraform.py
+++ b/cloud/shared/bin/lib/terraform.py
@@ -51,6 +51,7 @@ def perform_init(
     print(f" - Run {init_cmd}")
     output, exit_code = capture_stderr(init_cmd)
     if exit_code > 0:
+        # Determine if we're running interactively
         is_tty = sys.stdin.isatty()
         # This is AWS-specific, and should be modified when we have actual
         # Azure deployments
@@ -69,7 +70,7 @@ def perform_init(
                             config_loader, terraform_template_dir, upgrade)
                         return
                 print(
-                    f"To fix the above error, set the LOCK_TABLE_DIGEST_VALUE environment variable to \"{digest}\" and rerun this command."
+                    f"To fix the above error, rerun the command with LOCK_TABLE_DIGEST_VALUE=\"{digest}\" before it."
                 )
             # Since we've handled the error and printed a message, exit immediately
             # rather than returning False and having it print a stack trace.
@@ -139,6 +140,7 @@ def perform_apply(
 
     output, exit_code = capture_stderr(terraform_apply_cmd)
     if exit_code > 0:
+        # Determine if we're running interactively
         is_tty = sys.stdin.isatty()
         if "Error acquiring the state lock" in output:
             # Lock ID is a standard UUID v4 in the form 00000000-0000-0000-0000-000000000000
@@ -165,13 +167,13 @@ def perform_apply(
                             False)  # initialize = False
                 print(
                     error_text +
-                    f"\nIf you are sure there are no other Terraform processes running, this can be fixed by setting the FORCE_UNLOCK_ID environment variable to \"{lock_id}\" and rerunning the same command."
+                    f"\nIf you are sure there are no other Terraform processes running, this can be fixed by rerunning the command with FORCE_UNLOCK_ID=\"{lock_id}\" before it."
                 )
             else:
                 print(
                     error_text +
                     "\nWe were unable to extract the lock ID from the error text. Inspect the error message above."
-                    "\nIf you are sure there are no other Terraform processes running, this error can be fixed by setting the FORCE_UNLOCK_ID environment variable to the lock ID value, and then rerunning the same command."
+                    "\nIf you are sure there are no other Terraform processes running, this error can be fixed by rerunning the command with FORCE_UNLOCK_ID=<Lock ID> before it."
                 )
             # Since we've handled the error and printed a message, exit immediately
             # rather than returning False and having it print a stack trace.