Merge pull request #425 from NHSDigital/dp-force-unlock

Dp force unlock

Author: dominicplatt2-nhs

Makefile

@@ -40,3 +40,6 @@ endif
 
 ansible: guard-cmd
 	@account=$(account) poetry run make --no-print-directory -C ansible $(cmd)
+
+remove-stale-locks:
+	@poetry run python ./scripts/terraform_force_unlock.py

azure/components/cleanup-ecs-pr-proxies-job.yml

@@ -9,23 +9,22 @@ steps:
           profile: "apm_ptl"
 
       - bash: |
+          make remove-stale-locks
           export retain_hours=72
-          ANSIBLE_FORCE_COLOR=yes make -C ansible remove-old-ecs-pr-deploys > /tmp/err.txt
+          ANSIBLE_FORCE_COLOR=yes make -C ansible remove-old-ecs-pr-deploys | tee /tmp/output.txt
           ERROR_CODE=$?
           ROLE_TIMEOUT_MSG="The AWS assume role session token is due to expire"
-          if grep -q "$ROLE_TIMEOUT_MSG" /tmp/err.txt ; then
+          if grep -q "$ROLE_TIMEOUT_MSG" /tmp/output.txt ; then
             echo "stderr for ansible has the error \"$ROLE_TIMEOUT_MSG\""
             echo "Re-assuming role and re-running step"
             echo "##vso[task.setvariable variable=has_aws_role_timedout;]true"
 
           elif [ $ERROR_CODE -ne 0 ] ; then 
-            echo "ansible has unhandled error, re-raising"
-            >&2 cat /tmp/err.txt
+            echo "\n\nansible has unhandled error, re-raising"
             exit -1
           else
             echo "##vso[task.setvariable variable=has_aws_role_timedout;]false"
           fi
-          cat /tmp/err.txt
 
         displayName: "cleanup older pr deploys"
         condition: or(eq( ${{ parameters.retry }}, '0'), eq(variables['has_aws_role_timedout'], 'true'))

poetry.lock

@@ -25,6 +25,7 @@ gitpython = "^3.1.11"
 pytest-xdist = "^1.10.0"
 lxml = "^4.6.2"
 docker = "^5.0.3"
+boto3 = "^1.26.20"
 
 [tool.poetry.dev-dependencies]
 ansible-lint = "^4.2.0"

pyproject.toml

@@ -0,0 +1,69 @@
+"""
+Script to force unlock locks in terraform with a given prefix and of a certain age.
+
+Warning this script should be used with care.
+
+This script has to be used because sometimes the ecs pr pipeline cleanup won't always properly release the lock when
+it fails.
+
+CLI arguments:
+    --min-age-hr
+    --key-prefix
+    --table-name
+    --profile
+
+Example usage:
+
+python ./terraform_force_unlock.py --min-age-hr=8 --key-prefix=nhsd-apm-management-ptl-terraform/env:/api-deployment:ptl: --table-name=terraform-state-lock --profile=apm_ptl
+
+"""
+
+import json
+import boto3
+import dateutil
+import datetime
+import click
+
+
+@click.command()
+@click.option("--min-age-hr", type=int, default=8)
+@click.option("--key-prefix", type=str, default="nhsd-apm-management-ptl-terraform/env:/api-deployment:ptl:")
+@click.option("--table-name", type=str, default="terraform-state-lock")
+@click.option("--profile", type=str, default="apm_ptl")
+def main(min_age_hr, key_prefix, table_name, profile):
+
+    accepted_envs = ["apm_ptl", "apm_prod"]
+
+    if profile not in accepted_envs:
+        raise ValueError("Profile must be apm_ptl or apm_prod")
+
+    terraform_lock_table = boto3.Session(profile_name=profile).resource("dynamodb").Table(table_name)
+
+    filter_expr = "begins_with(#n0, :v0) AND attribute_exists(#n1)"
+
+    ExpressionAttributeNames = {"#n0": "LockID", "#n1": "Info"}
+    ExpressionAttributeValues = {
+        ":v0": key_prefix,
+    }
+    items = terraform_lock_table.scan(FilterExpression=filter_expr, ExpressionAttributeNames=ExpressionAttributeNames, ExpressionAttributeValues=ExpressionAttributeValues)
+    print(f"Found {len(items['Items'])} locks which start with key prefix '{key_prefix}'")
+
+    removed_count = 0
+    for lock_item in items["Items"]:
+        lock_item_info = json.loads(lock_item["Info"])
+        lock_id = lock_item["LockID"]
+        created_at = dateutil.parser.parse(lock_item_info["Created"])
+
+        if datetime.datetime.now(datetime.timezone.utc) - created_at > datetime.timedelta(hours=min_age_hr):
+            print(f"{lock_id} {created_at=} is more than {min_age_hr} hours old, deleting lock...")
+            terraform_lock_table.delete_item(Key={"LockID": lock_id})
+            removed_count += 1
+
+        else:
+            print(f"{lock_id} {created_at=} is not more than {min_age_hr} hours old, leaving it alone!")
+
+    print(f"Removed {removed_count} locks")
+
+
+if __name__ == "__main__":
+    main()