From d4d9dc0461f8af5c3f3dadba3e4e0cad7a52b250 Mon Sep 17 00:00:00 2001
From: jameslinnell <james.linnell2@nhs.net>
Date: Thu, 1 Feb 2024 17:13:16 +0000
Subject: [PATCH 1/6] Upgrade python to 3.9

---
 .github/workflows/continuous-integration.yml                | 4 ++--
 .github/workflows/unit-tests.yml                            | 4 ++--
 docker/sds-api/integration-tests/Pipfile                    | 2 +-
 docker/sds-api/integration-tests/Pipfile.lock               | 4 ++--
 docker/sds-api/spine-directory-service/sds/Dockerfile       | 2 +-
 docker/sds-api/spine-directory-service/sds/Dockerfile_local | 2 +-
 docker/sds-api/spine-directory-service/sds/Pipfile          | 2 +-
 docker/sds-api/spine-directory-service/sds/Pipfile.lock     | 4 ++--
 pyproject.toml                                              | 4 ++--
 9 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml
index 3d7f5af32..b6e01f720 100644
--- a/.github/workflows/continuous-integration.yml
+++ b/.github/workflows/continuous-integration.yml
@@ -12,10 +12,10 @@ jobs:
         with:
           fetch-depth: 0  # This causes all history to be fetched, which is required for calculate-version to function
 
-      - name: Install Python 3.8
+      - name: Install Python 3.9
         uses: actions/setup-python@v1
         with:
-          python-version: 3.8
+          python-version: 3.9
 
       - name: Update apt repositories
         run: sudo apt update
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index 76e0c7a34..57f14358a 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -11,10 +11,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v1
-      - name: Set up Python 3.8
+      - name: Set up Python 3.9
         uses: actions/setup-python@v1
         with:
-          python-version: 3.8
+          python-version: 3.9
       - name: Install pipenv
         uses: dschep/install-pipenv-action@v1
         with:
diff --git a/docker/sds-api/integration-tests/Pipfile b/docker/sds-api/integration-tests/Pipfile
index d420fe280..840c79b8d 100644
--- a/docker/sds-api/integration-tests/Pipfile
+++ b/docker/sds-api/integration-tests/Pipfile
@@ -10,7 +10,7 @@ requests = "*"
 unittest-xml-reporting = "*"
 
 [requires]
-python_version = "3.8"
+python_version = "3.9"
 
 [scripts]
 inttests = 'python -m xmlrunner discover -o test-reports ./tests'
diff --git a/docker/sds-api/integration-tests/Pipfile.lock b/docker/sds-api/integration-tests/Pipfile.lock
index 1eb129910..09cf51d8e 100644
--- a/docker/sds-api/integration-tests/Pipfile.lock
+++ b/docker/sds-api/integration-tests/Pipfile.lock
@@ -1,11 +1,11 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "bcb6c637eff939b2e99cc53183abf56b04463b68cf07c8d95a9086f6be59e6c3"
+            "sha256": "f6748174d7800b666bf195a7924a74556979f6b58259e60a97305d69aba11bbe"
         },
         "pipfile-spec": 6,
         "requires": {
-            "python_version": "3.8"
+            "python_version": "3.9"
         },
         "sources": [
             {
diff --git a/docker/sds-api/spine-directory-service/sds/Dockerfile b/docker/sds-api/spine-directory-service/sds/Dockerfile
index 7c35937b0..88d7ae098 100644
--- a/docker/sds-api/spine-directory-service/sds/Dockerfile
+++ b/docker/sds-api/spine-directory-service/sds/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.8-slim-bullseye as base
+FROM python:3.9-slim-bullseye as base
 
 RUN apt-get update && \
     apt-get install build-essential -y && \
diff --git a/docker/sds-api/spine-directory-service/sds/Dockerfile_local b/docker/sds-api/spine-directory-service/sds/Dockerfile_local
index 8ddc09f83..9b886156c 100644
--- a/docker/sds-api/spine-directory-service/sds/Dockerfile_local
+++ b/docker/sds-api/spine-directory-service/sds/Dockerfile_local
@@ -1,4 +1,4 @@
-FROM python:3.8-slim-bullseye as base
+FROM python:3.9-slim-bullseye as base
 
 RUN apt-get update && \
     apt-get install build-essential -y && \
diff --git a/docker/sds-api/spine-directory-service/sds/Pipfile b/docker/sds-api/spine-directory-service/sds/Pipfile
index 42c63fc34..07c15c31e 100644
--- a/docker/sds-api/spine-directory-service/sds/Pipfile
+++ b/docker/sds-api/spine-directory-service/sds/Pipfile
@@ -14,7 +14,7 @@ isodate = "~=0.6"
 python-json-logger = "~=2.0.1"
 
 [requires]
-python_version = "3.8"
+python_version = "3.9"
 
 [scripts]
 unittests = 'python -m xmlrunner -o test-reports -v'
diff --git a/docker/sds-api/spine-directory-service/sds/Pipfile.lock b/docker/sds-api/spine-directory-service/sds/Pipfile.lock
index db3dd89d9..787160902 100644
--- a/docker/sds-api/spine-directory-service/sds/Pipfile.lock
+++ b/docker/sds-api/spine-directory-service/sds/Pipfile.lock
@@ -1,11 +1,11 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "e2c66dbb7cf0fb3e4c4f2320795e35c376af172a6ba85e913802d6e9f774d42f"
+            "sha256": "1034dcf8bd32f87ac3551787c40f6341ca5fd965c1b3886861bb0dcb99db4d23"
         },
         "pipfile-spec": 6,
         "requires": {
-            "python_version": "3.8"
+            "python_version": "3.9"
         },
         "sources": [
             {
diff --git a/pyproject.toml b/pyproject.toml
index 3c7273db7..479f41e1a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,5 +1,5 @@
 [project]
-python = "^3.8"
+python = "^3.9"
 
 
 [tool.poetry]
@@ -22,7 +22,7 @@ keywords = ["healthcare", "uk", "nhs"] #TODO add additional keywords
 
 
 [tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.9"
 pyyaml = "^5.2"
 docopt = "^0.6.2"
 jsonpath-rw = "^1.4.0"

From fa471c0f23fca6a58de77d90d782842574632a34 Mon Sep 17 00:00:00 2001
From: jameslinnell <james.linnell2@nhs.net>
Date: Thu, 1 Feb 2024 17:28:24 +0000
Subject: [PATCH 2/6] Upgrade python to 3.9

---
 poetry.lock | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index 62cb0d2ea..94a61c436 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1284,5 +1284,5 @@ multidict = ">=4.0"
 
 [metadata]
 lock-version = "2.0"
-python-versions = "^3.8"
-content-hash = "7a27d1177de4e671bd76d48c0848b9e7b5f9506b232584f41e2dce3f7d21badd"
+python-versions = "^3.9"
+content-hash = "082377499f94dc7de37e7cc579a7802ed82ec91f9a9a0b8b31fcf6999e38bc4c"

From 0cd783512effd0b18439fdd3ab57ef1aaf2c365a Mon Sep 17 00:00:00 2001
From: jameslinnell <james.linnell2@nhs.net>
Date: Thu, 1 Feb 2024 17:42:40 +0000
Subject: [PATCH 3/6] Upgrade python to 3.9

---
 .github/workflows/continuous-integration.yml | 2 +-
 .github/workflows/unit-tests.yml             | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml
index b6e01f720..58983676d 100644
--- a/.github/workflows/continuous-integration.yml
+++ b/.github/workflows/continuous-integration.yml
@@ -13,7 +13,7 @@ jobs:
           fetch-depth: 0  # This causes all history to be fetched, which is required for calculate-version to function
 
       - name: Install Python 3.9
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
 
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index 57f14358a..19d8f07e6 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -10,9 +10,9 @@ jobs:
         working-directory: ./docker/sds-api/spine-directory-service/sds
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
       - name: Set up Python 3.9
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
       - name: Install pipenv

From 71b4bc3d87ee6e70a0dbaa971338e8de4cdc979c Mon Sep 17 00:00:00 2001
From: jameslinnell <james.linnell2@nhs.net>
Date: Fri, 2 Feb 2024 10:56:56 +0000
Subject: [PATCH 4/6] Upgrade python to 3.9

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 01d043a81..e4488b7ca 100644
--- a/README.md
+++ b/README.md
@@ -59,6 +59,7 @@ Connect to the VPN,
 You may have issues with the container connecting via the VPN. If so please look at the answer given here https://superuser.com/questions/1579858/docker-bridge-network-sporadically-loosing-packets/1580017?_gl=1*wyte41*_ga*MjgwODQyNzEwLjE3MDYwMDMwNzQ.*_ga_S812YQPLT2*MTcwNjE5MjIyOC4yLjAuMTcwNjE5MjIyOC4wLjAuMA..#1580017 using `docker network create --subnet=172.20.0.0/24 --gateway=172.20.0.1 docker20`
 
 Finally run
+
 ```sh
 docker-compose up
 ```

From 0c96be173e4c4f99c8517dfc78d3ddc5d029b261 Mon Sep 17 00:00:00 2001
From: jameslinnell <james.linnell2@nhs.net>
Date: Fri, 2 Feb 2024 11:38:55 +0000
Subject: [PATCH 5/6] Upgrade python to 3.9 on Azure Pipeline

---
 azure/azure-infrastructure-build.yml | 426 +++++++++++++++++++++++++++
 1 file changed, 426 insertions(+)
 create mode 100644 azure/azure-infrastructure-build.yml

diff --git a/azure/azure-infrastructure-build.yml b/azure/azure-infrastructure-build.yml
new file mode 100644
index 000000000..9267a0452
--- /dev/null
+++ b/azure/azure-infrastructure-build.yml
@@ -0,0 +1,426 @@
+name: "$(SourceBranchName)+$(BuildID)"
+
+trigger:
+ branches:
+   include:
+     - tags/refs/v*
+ tags:
+   include:
+     - v*
+
+pr:
+  branches:
+    include: ['*']
+
+resources:
+  repositories:
+    - repository: nhsd_apm_all_idx_all_indexes
+      type: github
+      name: NHSDigital/mirror-sc_apps-nhsd_apm_all_idx_all_indexes
+      ref: refs/heads/master
+      endpoint: NHSDigital
+    - repository: nhsd_apm_all_sh_all_viz
+      type: github
+      name: NHSDigital/mirror-sc_apps-nhsd_apm_all_sh_all_viz
+      ref: refs/heads/master
+      endpoint: NHSDigital
+
+stages:
+
+  - stage: build
+    displayName: Build Infrastructure Artifact
+    jobs:
+      - job: build
+        displayName: Build And Create Artifact
+        timeoutInMinutes: 40
+
+        pool:
+          name: 'AWS-ECS'
+
+        workspace:
+          clean: all
+
+        steps:
+          - checkout: self
+            persistCredentials: true
+          - checkout: nhsd_apm_all_sh_all_viz
+            path: s/nhsd_apm_all_sh_all_viz
+          - checkout: nhsd_apm_all_idx_all_indexes
+            path: s/nhsd_apm_all_idx_all_indexes
+
+          - bash: |
+              instance_id="$(curl -s http://169.254.169.254/latest/meta-data/instance-id)"
+              echo instance-id: "${instance_id}"
+              echo connect to: https://eu-west-2.console.aws.amazon.com/systems-manager/session-manager/${instance_id}
+              echo sudo su - ubuntu
+              or
+              echo ssh ubuntu@${instance_id}
+              echo working directory: $(System.DefaultWorkingDirectory)
+            displayName: print aws info
+
+          - task: UsePythonVersion@0
+            displayName: "Use Python 3.9"
+            inputs:
+              versionSpec: "3.9"
+
+          - bash: |
+              set -e
+              # Force our custom installed version of poetry as
+              # UsePythonVersion prepends the agent-aware python
+              # version to PATH.
+              echo 'export PATH=${HOME}/.local/bin:${PATH}' >> ~/.profile
+              source ~/.profile
+              # Now check poetry version
+              echo Poetry version is "$(poetry --no-ansi --version)"
+              if [[ ! "$(poetry --no-ansi --version)" =~ 'version 1.7' ]]; then
+                 echo "Need up update poetry version..."
+                 poetry self update --preview && which poetry && poetry --version || echo "But that's not going to happen..."
+              fi
+              if [[ ! "$(poetry --no-ansi --version)" =~ 'version 1.7' ]]; then
+                 echo "Could not get the correct poetry version running."
+                 echo "**************************************************************"
+                 echo "The PATH is ${PATH}"
+                 echo "**************************************************************"
+                 echo "**************************************************************"
+                 echo "The .profile is:"
+                 cat ~/.profile
+                 echo "**************************************************************"
+                 echo "**************************************************************"
+                 echo "The .bashrc is:"
+                 cat ~/.bashrc
+                 echo "**************************************************************"
+                 exit 1
+              fi
+            displayName: print poetry info
+            workingDirectory: api-management-infrastructure
+
+          - task: s3-cache-action@1
+            inputs:
+              key: 'poetry | ./api-management-infrastructure/.venv | ./api-management-infrastructure/poetry.lock'
+              location: './api-management-infrastructure/.venv'
+              debug: true
+              alias: 'Poetry'
+            displayName: cache poetry dependencies
+
+          - bash: make install
+            condition: ne(variables['CacheRestored-Poetry'], 'true')
+            displayName: Install project dependencies
+            workingDirectory: api-management-infrastructure
+
+          - bash: |
+              # Clear down any running containers on this instance
+              set -euo pipefail
+              make -C docker down || true
+              docker kill $(docker ps -q) || true
+              docker network prune -f
+            displayName: pre-emptively tear down containers
+            workingDirectory: api-management-infrastructure
+
+          - template: components/aws-assume-role.yml
+            parameters:
+              role: "auto-ops"
+              profile: "apm_ptl"
+              aws_account: "ptl"
+
+          - template: ./components/get-aws-secrets-and-ssm-params.yml
+            parameters:
+              secret_ids:
+                - ptl/azure-devops/apigee-nonprod/APIGEE_OTP_KEY
+                - ptl/azure-devops/apigee-nonprod/APIGEE_PASSWORD
+                - ptl/azure-devops/apigee-nonprod/APIGEE_USERNAME
+                - ptl/keycloak/paas/PROXYGEN_KEYCLOAK_CLIENT_ID
+                - ptl/keycloak/paas/PROXYGEN_KEYCLOAK_CLIENT_SECRET
+                - ptl/keycloak/paas/PROXYGEN_MACHINE_USER_CLIENT_ID
+                - ptl/keycloak/paas/API_PRODUCER_USER_CLIENT_ID
+                - ptl/keycloak/paas/API_PRODUCER_USER_CLIENT_SECRET
+                - ptl/keycloak/paas/API_PRODUCER_USERNAME
+                - ptl/keycloak/paas/API_PRODUCER_PASSWORD
+                - ptl/keycloak/paas/api-producers/deathstar/DEATHSTAR_PASSWORD
+                - ptl/monitoring/status-endpoint-api-key
+                - ptl/azure-devops/MONITORING_API_KEY
+              secret_file_ids:
+                - ptl/keycloak/paas/PROXYGEN_MACHINE_USER_CLIENT_PRIVATE_KEY
+                - ptl/proxygen/paas/PROXYGEN_MTLS_TEST_CLIENT_KEY
+                - ptl/proxygen/paas/PROXYGEN_MTLS_TEST_CLIENT_CERTIFICATE
+                - ptl/proxygen/paas/PROXYGEN_MTLS_TEST_CA_CERTIFICATE
+                - ptl/proxygen/paas/PROXYGEN_MTLS_TEST_SERVER_CERTIFICATE
+                - ptl/proxygen/paas/PROXYGEN_MTLS_TEST_SERVER_KEY
+                - ptl/proxygen/paas/PROXYGEN_MTLS_TEST_CLIENT_PRIVATE_KEY_ABSOLUTE_PATH
+                - ptl/proxygen/paas/PROXYGEN_MTLS_TEST_CLIENT_CERTIFICATE_ABSOLUTE_PATH
+              config_ids:
+                - /ptl/azure-devops/apigee-nonprod/APIGEE_USERNAME
+              aws_profile: "apm_ptl"
+
+          - bash: |
+              export MFA_CODE=`oathtool --totp -b "$(APIGEE_OTP_KEY)"`
+              # basic auth value here is an apigee public value .. https://docs.apigee.com/api-platform/system-administration/management-api-tokens  (not secret)
+              curl -X POST https://login.apigee.com/oauth/token \
+                -H "Content-Type: application/x-www-form-urlencoded" \
+                -H "Accept: application/json;charset=utf-8" \
+                -H "Authorization: Basic ZWRnZWNsaTplZGdlY2xpc2VjcmV0" \
+                -d "username=$(APIGEE_USERNAME)&password=$(APIGEE_PASSWORD)&mfa_token=${MFA_CODE}&grant_type=password" | jq .access_token > .token
+              # Set token into variable
+              echo "##vso[task.setvariable variable=secret.AccessToken;issecret=true]`cat .token`"
+            displayName: "get access token"
+
+          - bash: |
+              set -e
+
+              export num_changes_in_ansible_collections=$(git diff --name-only origin/master ansible/collections | wc -l)
+              echo "##vso[task.setvariable variable=ansibleCollectionsChanges]${num_changes_in_ansible_collections}"
+              echo "${num_changes_in_ansible_collections} changes in ansible collections"
+
+              export num_scripts_changes=$(git diff --name-only origin/master scripts | wc -l)
+              echo "##vso[task.setvariable variable=scriptsChanges]${num_scripts_changes}"
+              echo "${num_scripts_changes} changes in scripts"
+
+              export num_changes_in_docker=$(git diff --name-only origin/master docker | wc -l)
+              echo "##vso[task.setvariable variable=dockerChanges]${num_changes_in_docker}"
+              echo "${num_changes_in_docker} changes in docker"
+
+              export num_changes_in_lambda=$(git diff --name-only origin/master lambda | wc -l)
+              echo "##vso[task.setvariable variable=lambdaChanges]${num_changes_in_lambda}"
+              echo "${num_changes_in_docker} changes in lambda"
+            displayName: compute changes
+            workingDirectory: api-management-infrastructure
+
+          - bash: |
+              set -eou pipefail
+              cd ansible/collections/ansible_collections/nhsd/apim_infra/
+              ANSIBLE_FORCE_COLOR=yes \
+              APIGEE_ACCESS_TOKEN=$(secret.AccessToken) \
+              make test
+            displayName: test ansible collection
+            workingDirectory: api-management-infrastructure
+            condition: and(succeeded(), gt(variables['ansibleCollectionsChanges'], 0))
+
+          - bash: |
+              set -eou pipefail
+              # Remove leading refs/tags (tag release) or refs/heads (branch release) or refs/pull (PR release)
+              export source_branch=$(echo $(Build.SourceBranch) | sed s,'refs/\(tags\|pull\|heads\)/',,)
+              export commit_hash="$(git rev-parse HEAD)"
+              export build_label="$(Build.BuildId)-sha$(git rev-parse --short HEAD)"
+              export account_id="$(aws --profile=apm_ptl sts get-caller-identity --query Account --output text)"
+
+              # populate .extra.tfvars for terraform
+              mkdir -p ./dist
+              echo "*********************************************************"
+              echo "Populating .extra.tfvars (for terraform deployment)"
+              echo "*********************************************************"
+              echo "" > ./dist/.extra.tfvars
+              echo "build_label = \"${build_label}\"" >> ./dist/.extra.tfvars
+              echo "source_branch = \"${source_branch}\"" >> ./dist/.extra.tfvars
+              echo "commit_hash = \"${commit_hash}\"" >> ./dist/.extra.tfvars
+              cat ./dist/.extra.tfvars
+              echo "*********************************************************"
+              echo ""
+              echo "*********************************************************"
+              echo "Populating .build_env_vars (for apigee deployment scripts)"
+              echo "*********************************************************"
+              echo .build_env_vars for apigee deployment scripts
+              echo "" > ./dist/.build_env_vars
+              echo "export build_label=\"${build_label}\"" >> ./dist/.build_env_vars
+              echo "export source_branch=\"${source_branch}\"" >> ./dist/.build_env_vars
+              echo "export commit_hash=\"${commit_hash}\"" >> ./dist/.build_env_vars
+              cat ./dist/.build_env_vars
+              echo "*********************************************************"
+              echo ""
+              echo "*********************************************************"
+              echo "Populating docker/.env"
+              echo "*********************************************************"
+              echo ""
+              echo "" > docker/.env
+              echo APIGEE_ACCESS_TOKEN=$(secret.AccessToken) >> docker/.env
+              echo APIGEE_NHSD_NONPROD_OTP_KEY=$(APIGEE_OTP_KEY) >> docker/.env
+              echo APIGEE_NHSD_NONPROD_PASSWORD=$(APIGEE_PASSWORD) >> docker/.env
+              echo APIGEE_NHSD_NONPROD_USERNAME=$(APIGEE_USERNAME) >> docker/.env
+              echo APIGEE_ORG=nhsd-nonprod >> docker/.env
+              echo API_PRODUCER_PASSWORD=$(API_PRODUCER_PASSWORD) >> docker/.env
+              echo API_PRODUCER_USERNAME=$(API_PRODUCER_USERNAME) >> docker/.env
+              echo API_PRODUCER_USER_CLIENT_ID=$(API_PRODUCER_USER_CLIENT_ID) >> docker/.env
+              echo API_PRODUCER_USER_CLIENT_SECRET=$(API_PRODUCER_USER_CLIENT_SECRET) >> docker/.env
+              echo API_REGISTRY_URL=http://localhost:9000 >> docker/.env
+              echo AWS_ENV=local >> docker/.env
+              echo BUILD_LABEL="${build_label}" >> docker/.env
+              echo DEATHSTAR_PASSWORD=$(DEATHSTAR_PASSWORD) >> docker/.env
+              echo ECR_PREFIX="${account_id}".dkr.ecr.eu-west-2.amazonaws.com/ >> docker/.env
+              echo HOST_GID=$(id -g) >> docker/.env
+              echo HOST_UID=$(id -u) >> docker/.env
+              echo KEYCLOAK_ENDPOINT_URL=https://identity.ptl.api.platform.nhs.uk >> docker/.env
+              echo LOCALSTACK_ACCESS_KEY_ID=abc >> docker/.env
+              echo LOCALSTACK_ENDPOINT_URL=http://localstack:4766 >> docker/.env
+              echo LOCALSTACK_SECRET_ACCESS_KEY=123 >> docker/.env
+              echo PROXYGEN_BUCKET_NAME=nhsd-apm-management-ptl-proxygen >> docker/.env
+              echo PROXYGEN_KEYCLOAK_CLIENT_ID=$(PROXYGEN_KEYCLOAK_CLIENT_ID) >> docker/.env
+              echo PROXYGEN_KEYCLOAK_CLIENT_SECRET=$(PROXYGEN_KEYCLOAK_CLIENT_SECRET) >> docker/.env
+              echo PROXYGEN_MACHINE_USER_CLIENT_ID=$(PROXYGEN_MACHINE_USER_CLIENT_ID) >> docker/.env
+              echo PROXYGEN_MACHINE_USER_CLIENT_PRIVATE_KEY_ABSOLUTE_PATH=$(Pipeline.Workspace)/secrets/$(PROXYGEN_MACHINE_USER_CLIENT_PRIVATE_KEY) >> docker/.env
+              echo PROXYGEN_MTLS_TEST_CLIENT_PRIVATE_KEY_ABSOLUTE_PATH=$(Pipeline.Workspace)/secrets/$(PROXYGEN_MTLS_TEST_CLIENT_KEY) >> docker/.env
+              echo PROXYGEN_MTLS_TEST_CLIENT_CERTIFICATE_ABSOLUTE_PATH=$(Pipeline.Workspace)/secrets/$(PROXYGEN_MTLS_TEST_CLIENT_CERTIFICATE) >> docker/.env
+              echo SPLUNK_HEC_TOKEN=abcd1234 >> docker/.env
+              echo SPLUNK_HEC_URL=https://localhost:8088 >> docker/.env
+              echo STATUS_ENDPOINT_API_KEY=$(status-endpoint-api-key) >> docker/.env
+              echo PTL_MONITORING_API_KEY=$(MONITORING_API_KEY) >> docker/.env
+              # The following variables contain the *contents* of files, which need to be concatenated
+              echo "PROXYGEN_MTLS_TEST_CLIENT_KEY='`cat $(Pipeline.Workspace)/secrets/$(PROXYGEN_MTLS_TEST_CLIENT_KEY)`'" >> docker/.env
+              echo "PROXYGEN_MTLS_TEST_CLIENT_CERTIFICATE='`cat $(Pipeline.Workspace)/secrets/$(PROXYGEN_MTLS_TEST_CLIENT_CERTIFICATE)`'" >> docker/.env
+              echo "PROXYGEN_MTLS_TEST_CA_CERTIFICATE='`cat $(Pipeline.Workspace)/secrets/$(PROXYGEN_MTLS_TEST_CA_CERTIFICATE)`'" >> docker/.env
+              echo "PROXYGEN_MTLS_TEST_SERVER_CERTIFICATE='`cat $(Pipeline.Workspace)/secrets/$(PROXYGEN_MTLS_TEST_SERVER_CERTIFICATE)`'" >> docker/.env
+              echo "PROXYGEN_MTLS_TEST_SERVER_KEY='`cat $(Pipeline.Workspace)/secrets/$(PROXYGEN_MTLS_TEST_SERVER_KEY)`'" >> docker/.env
+              # Azure Devops masks the secrets so it's safe to cat this file.
+              cat docker/.env
+              echo "*********************************************************"
+            workingDirectory: api-management-infrastructure
+            displayName: "populate .env, .extra.tfvars, .build_env_vars"
+
+          - bash: |
+              set -euo pipefail
+              make -C docker login touch
+            displayName: docker login
+            workingDirectory: api-management-infrastructure
+
+          - bash: |
+              set -euo pipefail
+              BUILD_LABEL=latest \
+              make -C docker pre-pull
+            displayName: docker pull pre-requsites
+            workingDirectory: api-management-infrastructure
+
+          - bash: |
+              set -euo pipefail
+              PUSH=1 \
+              make -C docker pre-build
+            displayName: docker build and push pre-requsites
+            workingDirectory: api-management-infrastructure
+
+          - bash: |
+              set -euo pipefail
+              # Start splunk + localstack now as they are very slow
+              make -C docker pre-up
+            displayName: docker up pre-requsites
+            workingDirectory: api-management-infrastructure
+            # Note that we need localstack running w/local terraform applied for the scripts tests
+            condition: and(succeeded(), or(gt(variables['dockerChanges'],0), gt(variables['scriptsChanges'],0)))
+
+          - bash: |
+              set -euo pipefail
+              BUILD_LABEL=latest \
+              make -C docker pull
+            workingDirectory: api-management-infrastructure
+            displayName: docker pull
+
+          - bash: |
+              set -euo pipefail
+              PUSH=1 make -C docker build
+            workingDirectory: api-management-infrastructure
+            displayName: docker build and push
+
+          - bash: |
+              set -euo pipefail
+              make -C docker up
+            workingDirectory: api-management-infrastructure
+            displayName: docker up
+            condition: and(succeeded(), gt(variables['dockerChanges'], 0))
+
+          - bash: |
+              set -euo pipefail
+              make -C docker test
+            workingDirectory: api-management-infrastructure
+            displayName: docker test
+            condition: and(succeeded(), gt(variables['dockerChanges'], 0))
+
+          - bash: |
+              set -euo pipefail
+              make test-lambda
+            workingDirectory: api-management-infrastructure
+            displayName: test lambda
+            condition: and(succeeded(), gt(variables['lambdaChanges'], 0))
+
+          - bash: |
+              set -euo pipefail
+              APIGEE_TOKEN=$(secret.AccessToken) \
+              make test-scripts
+            workingDirectory: api-management-infrastructure
+            condition: and(succeeded(), gt(variables['scriptsChanges'], 0))
+            displayName: test scripts
+
+          - bash: |
+              set -euo pipefail
+              make down
+            displayName: bring down containers
+            condition: always()
+            workingDirectory: api-management-infrastructure/docker
+
+          - bash: |
+              set -e
+              source ../dist/.build_env_vars
+              ANSIBLE_FORCE_COLOR=yes \
+              make check-secrets-exist-and-pin-versions
+            displayName: check secrets exist for kvms and tls certs and pin version
+            workingDirectory: api-management-infrastructure/ansible
+
+          - bash: |
+              set -e
+              source ../dist/.build_env_vars
+              ANSIBLE_FORCE_COLOR=yes \
+              make set-deployment-build-labels
+            displayName: set per-image build_labels
+            workingDirectory: api-management-infrastructure/ansible
+
+          - bash: |
+              set -e
+              mkdir -p ./dist
+              echo "" > ./dist/.ami_ids
+              account_id="$(aws --profile=apm_ptl sts get-caller-identity --query Account --output text)"
+
+              apm_ubuntu_base_ami="$(aws --profile=apm_ptl ec2 describe-images --owners ${account_id} --filters 'Name=name,Values=apm-ubuntu-base-*' --query 'sort_by(Images, &CreationDate)[].[ImageId,CreationDate]' --output text | sort -k2 -r | head -n 1 | cut -f1)"
+              apm_ecs_agent_ami="$(aws --profile=apm_ptl ec2 describe-images --owners ${account_id} --filters 'Name=name,Values=apm-ecs-agent-*' --query 'sort_by(Images, &CreationDate)[].[ImageId,CreationDate]' --output text | sort -k2 -r | head -n 1 | cut -f1)"
+              apm_build_agent_ami="$(aws --profile=apm_ptl  ec2 describe-images --owners ${account_id} --filters 'Name=name,Values=apm-build-agent-*' --query 'sort_by(Images, &CreationDate)[].[ImageId,CreationDate]' --output text | sort -k2 -r | head -n 1 | cut -f1)"
+
+              if [[ -z "${apm_ubuntu_base_ami}" ]] || [[ -z "${apm_ecs_agent_ami}" ]] || [[ -z "${apm_build_agent_ami}" ]]; then
+                echo failed to get ami ids
+              fi
+
+              echo "apm_ubuntu_base_ami = ${apm_ubuntu_base_ami}"
+              echo "apm_ecs_agent_ami = ${apm_ecs_agent_ami}"
+              echo "apm_build_agent_ami = ${apm_build_agent_ami}"
+
+              echo "apm_ubuntu_base_ami = \"${apm_ubuntu_base_ami}\"" >> ./dist/.ami_ids
+              echo "apm_ecs_agent_ami = \"${apm_ecs_agent_ami}\"" >> ./dist/.ami_ids
+              echo "apm_build_agent_ami = \"${apm_build_agent_ami}\"" >> ./dist/.ami_ids
+
+              cat ./dist/.ami_ids >> ./dist/.extra.tfvars
+
+            displayName: record AMI ids
+            workingDirectory: api-management-infrastructure
+
+          - bash: |
+              set -e
+              echo "extra .tfvars:"
+              cat ./dist/.extra.tfvars
+
+              cat ./dist/.extra.tfvars >> terraform/_shared/vars/ptl.tfvars
+              cat ./dist/.extra.tfvars >> terraform/_shared/vars/prod.tfvars
+
+            displayName: .extra.tfvars output and record
+            workingDirectory: api-management-infrastructure
+
+          - bash: |
+              set -e
+              mv ./dist ./dist_vars
+              make release
+              cp -r ./dist_vars/. ./dist
+            displayName: copy scripts to artifact
+            workingDirectory: api-management-infrastructure
+
+          - task: PublishPipelineArtifact@1
+            inputs:
+              targetPath: $(System.DefaultWorkingDirectory)/api-management-infrastructure/dist
+              artifactName: package
+
+          - bash: |
+              docker ps -q | xargs docker stop
+              docker container prune -f
+              docker network prune -f
+            displayName: Ensure docker containers and networks are brought down
+            condition: always()

From f316be08b51af09d0e32e1d84abe22897c815cbc Mon Sep 17 00:00:00 2001
From: jameslinnell <james.linnell2@nhs.net>
Date: Fri, 2 Feb 2024 13:27:21 +0000
Subject: [PATCH 6/6] trying to set 3.9

---
 azure/azure-infrastructure-build.yml | 426 ---------------------------
 azure/common/apigee-build.yml        | 284 ++++++++++++++++++
 2 files changed, 284 insertions(+), 426 deletions(-)
 delete mode 100644 azure/azure-infrastructure-build.yml
 create mode 100644 azure/common/apigee-build.yml

diff --git a/azure/azure-infrastructure-build.yml b/azure/azure-infrastructure-build.yml
deleted file mode 100644
index 9267a0452..000000000
--- a/azure/azure-infrastructure-build.yml
+++ /dev/null
@@ -1,426 +0,0 @@
-name: "$(SourceBranchName)+$(BuildID)"
-
-trigger:
- branches:
-   include:
-     - tags/refs/v*
- tags:
-   include:
-     - v*
-
-pr:
-  branches:
-    include: ['*']
-
-resources:
-  repositories:
-    - repository: nhsd_apm_all_idx_all_indexes
-      type: github
-      name: NHSDigital/mirror-sc_apps-nhsd_apm_all_idx_all_indexes
-      ref: refs/heads/master
-      endpoint: NHSDigital
-    - repository: nhsd_apm_all_sh_all_viz
-      type: github
-      name: NHSDigital/mirror-sc_apps-nhsd_apm_all_sh_all_viz
-      ref: refs/heads/master
-      endpoint: NHSDigital
-
-stages:
-
-  - stage: build
-    displayName: Build Infrastructure Artifact
-    jobs:
-      - job: build
-        displayName: Build And Create Artifact
-        timeoutInMinutes: 40
-
-        pool:
-          name: 'AWS-ECS'
-
-        workspace:
-          clean: all
-
-        steps:
-          - checkout: self
-            persistCredentials: true
-          - checkout: nhsd_apm_all_sh_all_viz
-            path: s/nhsd_apm_all_sh_all_viz
-          - checkout: nhsd_apm_all_idx_all_indexes
-            path: s/nhsd_apm_all_idx_all_indexes
-
-          - bash: |
-              instance_id="$(curl -s http://169.254.169.254/latest/meta-data/instance-id)"
-              echo instance-id: "${instance_id}"
-              echo connect to: https://eu-west-2.console.aws.amazon.com/systems-manager/session-manager/${instance_id}
-              echo sudo su - ubuntu
-              or
-              echo ssh ubuntu@${instance_id}
-              echo working directory: $(System.DefaultWorkingDirectory)
-            displayName: print aws info
-
-          - task: UsePythonVersion@0
-            displayName: "Use Python 3.9"
-            inputs:
-              versionSpec: "3.9"
-
-          - bash: |
-              set -e
-              # Force our custom installed version of poetry as
-              # UsePythonVersion prepends the agent-aware python
-              # version to PATH.
-              echo 'export PATH=${HOME}/.local/bin:${PATH}' >> ~/.profile
-              source ~/.profile
-              # Now check poetry version
-              echo Poetry version is "$(poetry --no-ansi --version)"
-              if [[ ! "$(poetry --no-ansi --version)" =~ 'version 1.7' ]]; then
-                 echo "Need up update poetry version..."
-                 poetry self update --preview && which poetry && poetry --version || echo "But that's not going to happen..."
-              fi
-              if [[ ! "$(poetry --no-ansi --version)" =~ 'version 1.7' ]]; then
-                 echo "Could not get the correct poetry version running."
-                 echo "**************************************************************"
-                 echo "The PATH is ${PATH}"
-                 echo "**************************************************************"
-                 echo "**************************************************************"
-                 echo "The .profile is:"
-                 cat ~/.profile
-                 echo "**************************************************************"
-                 echo "**************************************************************"
-                 echo "The .bashrc is:"
-                 cat ~/.bashrc
-                 echo "**************************************************************"
-                 exit 1
-              fi
-            displayName: print poetry info
-            workingDirectory: api-management-infrastructure
-
-          - task: s3-cache-action@1
-            inputs:
-              key: 'poetry | ./api-management-infrastructure/.venv | ./api-management-infrastructure/poetry.lock'
-              location: './api-management-infrastructure/.venv'
-              debug: true
-              alias: 'Poetry'
-            displayName: cache poetry dependencies
-
-          - bash: make install
-            condition: ne(variables['CacheRestored-Poetry'], 'true')
-            displayName: Install project dependencies
-            workingDirectory: api-management-infrastructure
-
-          - bash: |
-              # Clear down any running containers on this instance
-              set -euo pipefail
-              make -C docker down || true
-              docker kill $(docker ps -q) || true
-              docker network prune -f
-            displayName: pre-emptively tear down containers
-            workingDirectory: api-management-infrastructure
-
-          - template: components/aws-assume-role.yml
-            parameters:
-              role: "auto-ops"
-              profile: "apm_ptl"
-              aws_account: "ptl"
-
-          - template: ./components/get-aws-secrets-and-ssm-params.yml
-            parameters:
-              secret_ids:
-                - ptl/azure-devops/apigee-nonprod/APIGEE_OTP_KEY
-                - ptl/azure-devops/apigee-nonprod/APIGEE_PASSWORD
-                - ptl/azure-devops/apigee-nonprod/APIGEE_USERNAME
-                - ptl/keycloak/paas/PROXYGEN_KEYCLOAK_CLIENT_ID
-                - ptl/keycloak/paas/PROXYGEN_KEYCLOAK_CLIENT_SECRET
-                - ptl/keycloak/paas/PROXYGEN_MACHINE_USER_CLIENT_ID
-                - ptl/keycloak/paas/API_PRODUCER_USER_CLIENT_ID
-                - ptl/keycloak/paas/API_PRODUCER_USER_CLIENT_SECRET
-                - ptl/keycloak/paas/API_PRODUCER_USERNAME
-                - ptl/keycloak/paas/API_PRODUCER_PASSWORD
-                - ptl/keycloak/paas/api-producers/deathstar/DEATHSTAR_PASSWORD
-                - ptl/monitoring/status-endpoint-api-key
-                - ptl/azure-devops/MONITORING_API_KEY
-              secret_file_ids:
-                - ptl/keycloak/paas/PROXYGEN_MACHINE_USER_CLIENT_PRIVATE_KEY
-                - ptl/proxygen/paas/PROXYGEN_MTLS_TEST_CLIENT_KEY
-                - ptl/proxygen/paas/PROXYGEN_MTLS_TEST_CLIENT_CERTIFICATE
-                - ptl/proxygen/paas/PROXYGEN_MTLS_TEST_CA_CERTIFICATE
-                - ptl/proxygen/paas/PROXYGEN_MTLS_TEST_SERVER_CERTIFICATE
-                - ptl/proxygen/paas/PROXYGEN_MTLS_TEST_SERVER_KEY
-                - ptl/proxygen/paas/PROXYGEN_MTLS_TEST_CLIENT_PRIVATE_KEY_ABSOLUTE_PATH
-                - ptl/proxygen/paas/PROXYGEN_MTLS_TEST_CLIENT_CERTIFICATE_ABSOLUTE_PATH
-              config_ids:
-                - /ptl/azure-devops/apigee-nonprod/APIGEE_USERNAME
-              aws_profile: "apm_ptl"
-
-          - bash: |
-              export MFA_CODE=`oathtool --totp -b "$(APIGEE_OTP_KEY)"`
-              # basic auth value here is an apigee public value .. https://docs.apigee.com/api-platform/system-administration/management-api-tokens  (not secret)
-              curl -X POST https://login.apigee.com/oauth/token \
-                -H "Content-Type: application/x-www-form-urlencoded" \
-                -H "Accept: application/json;charset=utf-8" \
-                -H "Authorization: Basic ZWRnZWNsaTplZGdlY2xpc2VjcmV0" \
-                -d "username=$(APIGEE_USERNAME)&password=$(APIGEE_PASSWORD)&mfa_token=${MFA_CODE}&grant_type=password" | jq .access_token > .token
-              # Set token into variable
-              echo "##vso[task.setvariable variable=secret.AccessToken;issecret=true]`cat .token`"
-            displayName: "get access token"
-
-          - bash: |
-              set -e
-
-              export num_changes_in_ansible_collections=$(git diff --name-only origin/master ansible/collections | wc -l)
-              echo "##vso[task.setvariable variable=ansibleCollectionsChanges]${num_changes_in_ansible_collections}"
-              echo "${num_changes_in_ansible_collections} changes in ansible collections"
-
-              export num_scripts_changes=$(git diff --name-only origin/master scripts | wc -l)
-              echo "##vso[task.setvariable variable=scriptsChanges]${num_scripts_changes}"
-              echo "${num_scripts_changes} changes in scripts"
-
-              export num_changes_in_docker=$(git diff --name-only origin/master docker | wc -l)
-              echo "##vso[task.setvariable variable=dockerChanges]${num_changes_in_docker}"
-              echo "${num_changes_in_docker} changes in docker"
-
-              export num_changes_in_lambda=$(git diff --name-only origin/master lambda | wc -l)
-              echo "##vso[task.setvariable variable=lambdaChanges]${num_changes_in_lambda}"
-              echo "${num_changes_in_docker} changes in lambda"
-            displayName: compute changes
-            workingDirectory: api-management-infrastructure
-
-          - bash: |
-              set -eou pipefail
-              cd ansible/collections/ansible_collections/nhsd/apim_infra/
-              ANSIBLE_FORCE_COLOR=yes \
-              APIGEE_ACCESS_TOKEN=$(secret.AccessToken) \
-              make test
-            displayName: test ansible collection
-            workingDirectory: api-management-infrastructure
-            condition: and(succeeded(), gt(variables['ansibleCollectionsChanges'], 0))
-
-          - bash: |
-              set -eou pipefail
-              # Remove leading refs/tags (tag release) or refs/heads (branch release) or refs/pull (PR release)
-              export source_branch=$(echo $(Build.SourceBranch) | sed s,'refs/\(tags\|pull\|heads\)/',,)
-              export commit_hash="$(git rev-parse HEAD)"
-              export build_label="$(Build.BuildId)-sha$(git rev-parse --short HEAD)"
-              export account_id="$(aws --profile=apm_ptl sts get-caller-identity --query Account --output text)"
-
-              # populate .extra.tfvars for terraform
-              mkdir -p ./dist
-              echo "*********************************************************"
-              echo "Populating .extra.tfvars (for terraform deployment)"
-              echo "*********************************************************"
-              echo "" > ./dist/.extra.tfvars
-              echo "build_label = \"${build_label}\"" >> ./dist/.extra.tfvars
-              echo "source_branch = \"${source_branch}\"" >> ./dist/.extra.tfvars
-              echo "commit_hash = \"${commit_hash}\"" >> ./dist/.extra.tfvars
-              cat ./dist/.extra.tfvars
-              echo "*********************************************************"
-              echo ""
-              echo "*********************************************************"
-              echo "Populating .build_env_vars (for apigee deployment scripts)"
-              echo "*********************************************************"
-              echo .build_env_vars for apigee deployment scripts
-              echo "" > ./dist/.build_env_vars
-              echo "export build_label=\"${build_label}\"" >> ./dist/.build_env_vars
-              echo "export source_branch=\"${source_branch}\"" >> ./dist/.build_env_vars
-              echo "export commit_hash=\"${commit_hash}\"" >> ./dist/.build_env_vars
-              cat ./dist/.build_env_vars
-              echo "*********************************************************"
-              echo ""
-              echo "*********************************************************"
-              echo "Populating docker/.env"
-              echo "*********************************************************"
-              echo ""
-              echo "" > docker/.env
-              echo APIGEE_ACCESS_TOKEN=$(secret.AccessToken) >> docker/.env
-              echo APIGEE_NHSD_NONPROD_OTP_KEY=$(APIGEE_OTP_KEY) >> docker/.env
-              echo APIGEE_NHSD_NONPROD_PASSWORD=$(APIGEE_PASSWORD) >> docker/.env
-              echo APIGEE_NHSD_NONPROD_USERNAME=$(APIGEE_USERNAME) >> docker/.env
-              echo APIGEE_ORG=nhsd-nonprod >> docker/.env
-              echo API_PRODUCER_PASSWORD=$(API_PRODUCER_PASSWORD) >> docker/.env
-              echo API_PRODUCER_USERNAME=$(API_PRODUCER_USERNAME) >> docker/.env
-              echo API_PRODUCER_USER_CLIENT_ID=$(API_PRODUCER_USER_CLIENT_ID) >> docker/.env
-              echo API_PRODUCER_USER_CLIENT_SECRET=$(API_PRODUCER_USER_CLIENT_SECRET) >> docker/.env
-              echo API_REGISTRY_URL=http://localhost:9000 >> docker/.env
-              echo AWS_ENV=local >> docker/.env
-              echo BUILD_LABEL="${build_label}" >> docker/.env
-              echo DEATHSTAR_PASSWORD=$(DEATHSTAR_PASSWORD) >> docker/.env
-              echo ECR_PREFIX="${account_id}".dkr.ecr.eu-west-2.amazonaws.com/ >> docker/.env
-              echo HOST_GID=$(id -g) >> docker/.env
-              echo HOST_UID=$(id -u) >> docker/.env
-              echo KEYCLOAK_ENDPOINT_URL=https://identity.ptl.api.platform.nhs.uk >> docker/.env
-              echo LOCALSTACK_ACCESS_KEY_ID=abc >> docker/.env
-              echo LOCALSTACK_ENDPOINT_URL=http://localstack:4766 >> docker/.env
-              echo LOCALSTACK_SECRET_ACCESS_KEY=123 >> docker/.env
-              echo PROXYGEN_BUCKET_NAME=nhsd-apm-management-ptl-proxygen >> docker/.env
-              echo PROXYGEN_KEYCLOAK_CLIENT_ID=$(PROXYGEN_KEYCLOAK_CLIENT_ID) >> docker/.env
-              echo PROXYGEN_KEYCLOAK_CLIENT_SECRET=$(PROXYGEN_KEYCLOAK_CLIENT_SECRET) >> docker/.env
-              echo PROXYGEN_MACHINE_USER_CLIENT_ID=$(PROXYGEN_MACHINE_USER_CLIENT_ID) >> docker/.env
-              echo PROXYGEN_MACHINE_USER_CLIENT_PRIVATE_KEY_ABSOLUTE_PATH=$(Pipeline.Workspace)/secrets/$(PROXYGEN_MACHINE_USER_CLIENT_PRIVATE_KEY) >> docker/.env
-              echo PROXYGEN_MTLS_TEST_CLIENT_PRIVATE_KEY_ABSOLUTE_PATH=$(Pipeline.Workspace)/secrets/$(PROXYGEN_MTLS_TEST_CLIENT_KEY) >> docker/.env
-              echo PROXYGEN_MTLS_TEST_CLIENT_CERTIFICATE_ABSOLUTE_PATH=$(Pipeline.Workspace)/secrets/$(PROXYGEN_MTLS_TEST_CLIENT_CERTIFICATE) >> docker/.env
-              echo SPLUNK_HEC_TOKEN=abcd1234 >> docker/.env
-              echo SPLUNK_HEC_URL=https://localhost:8088 >> docker/.env
-              echo STATUS_ENDPOINT_API_KEY=$(status-endpoint-api-key) >> docker/.env
-              echo PTL_MONITORING_API_KEY=$(MONITORING_API_KEY) >> docker/.env
-              # The following variables contain the *contents* of files, which need to be concatenated
-              echo "PROXYGEN_MTLS_TEST_CLIENT_KEY='`cat $(Pipeline.Workspace)/secrets/$(PROXYGEN_MTLS_TEST_CLIENT_KEY)`'" >> docker/.env
-              echo "PROXYGEN_MTLS_TEST_CLIENT_CERTIFICATE='`cat $(Pipeline.Workspace)/secrets/$(PROXYGEN_MTLS_TEST_CLIENT_CERTIFICATE)`'" >> docker/.env
-              echo "PROXYGEN_MTLS_TEST_CA_CERTIFICATE='`cat $(Pipeline.Workspace)/secrets/$(PROXYGEN_MTLS_TEST_CA_CERTIFICATE)`'" >> docker/.env
-              echo "PROXYGEN_MTLS_TEST_SERVER_CERTIFICATE='`cat $(Pipeline.Workspace)/secrets/$(PROXYGEN_MTLS_TEST_SERVER_CERTIFICATE)`'" >> docker/.env
-              echo "PROXYGEN_MTLS_TEST_SERVER_KEY='`cat $(Pipeline.Workspace)/secrets/$(PROXYGEN_MTLS_TEST_SERVER_KEY)`'" >> docker/.env
-              # Azure Devops masks the secrets so it's safe to cat this file.
-              cat docker/.env
-              echo "*********************************************************"
-            workingDirectory: api-management-infrastructure
-            displayName: "populate .env, .extra.tfvars, .build_env_vars"
-
-          - bash: |
-              set -euo pipefail
-              make -C docker login touch
-            displayName: docker login
-            workingDirectory: api-management-infrastructure
-
-          - bash: |
-              set -euo pipefail
-              BUILD_LABEL=latest \
-              make -C docker pre-pull
-            displayName: docker pull pre-requsites
-            workingDirectory: api-management-infrastructure
-
-          - bash: |
-              set -euo pipefail
-              PUSH=1 \
-              make -C docker pre-build
-            displayName: docker build and push pre-requsites
-            workingDirectory: api-management-infrastructure
-
-          - bash: |
-              set -euo pipefail
-              # Start splunk + localstack now as they are very slow
-              make -C docker pre-up
-            displayName: docker up pre-requsites
-            workingDirectory: api-management-infrastructure
-            # Note that we need localstack running w/local terraform applied for the scripts tests
-            condition: and(succeeded(), or(gt(variables['dockerChanges'],0), gt(variables['scriptsChanges'],0)))
-
-          - bash: |
-              set -euo pipefail
-              BUILD_LABEL=latest \
-              make -C docker pull
-            workingDirectory: api-management-infrastructure
-            displayName: docker pull
-
-          - bash: |
-              set -euo pipefail
-              PUSH=1 make -C docker build
-            workingDirectory: api-management-infrastructure
-            displayName: docker build and push
-
-          - bash: |
-              set -euo pipefail
-              make -C docker up
-            workingDirectory: api-management-infrastructure
-            displayName: docker up
-            condition: and(succeeded(), gt(variables['dockerChanges'], 0))
-
-          - bash: |
-              set -euo pipefail
-              make -C docker test
-            workingDirectory: api-management-infrastructure
-            displayName: docker test
-            condition: and(succeeded(), gt(variables['dockerChanges'], 0))
-
-          - bash: |
-              set -euo pipefail
-              make test-lambda
-            workingDirectory: api-management-infrastructure
-            displayName: test lambda
-            condition: and(succeeded(), gt(variables['lambdaChanges'], 0))
-
-          - bash: |
-              set -euo pipefail
-              APIGEE_TOKEN=$(secret.AccessToken) \
-              make test-scripts
-            workingDirectory: api-management-infrastructure
-            condition: and(succeeded(), gt(variables['scriptsChanges'], 0))
-            displayName: test scripts
-
-          - bash: |
-              set -euo pipefail
-              make down
-            displayName: bring down containers
-            condition: always()
-            workingDirectory: api-management-infrastructure/docker
-
-          - bash: |
-              set -e
-              source ../dist/.build_env_vars
-              ANSIBLE_FORCE_COLOR=yes \
-              make check-secrets-exist-and-pin-versions
-            displayName: check secrets exist for kvms and tls certs and pin version
-            workingDirectory: api-management-infrastructure/ansible
-
-          - bash: |
-              set -e
-              source ../dist/.build_env_vars
-              ANSIBLE_FORCE_COLOR=yes \
-              make set-deployment-build-labels
-            displayName: set per-image build_labels
-            workingDirectory: api-management-infrastructure/ansible
-
-          - bash: |
-              set -e
-              mkdir -p ./dist
-              echo "" > ./dist/.ami_ids
-              account_id="$(aws --profile=apm_ptl sts get-caller-identity --query Account --output text)"
-
-              apm_ubuntu_base_ami="$(aws --profile=apm_ptl ec2 describe-images --owners ${account_id} --filters 'Name=name,Values=apm-ubuntu-base-*' --query 'sort_by(Images, &CreationDate)[].[ImageId,CreationDate]' --output text | sort -k2 -r | head -n 1 | cut -f1)"
-              apm_ecs_agent_ami="$(aws --profile=apm_ptl ec2 describe-images --owners ${account_id} --filters 'Name=name,Values=apm-ecs-agent-*' --query 'sort_by(Images, &CreationDate)[].[ImageId,CreationDate]' --output text | sort -k2 -r | head -n 1 | cut -f1)"
-              apm_build_agent_ami="$(aws --profile=apm_ptl  ec2 describe-images --owners ${account_id} --filters 'Name=name,Values=apm-build-agent-*' --query 'sort_by(Images, &CreationDate)[].[ImageId,CreationDate]' --output text | sort -k2 -r | head -n 1 | cut -f1)"
-
-              if [[ -z "${apm_ubuntu_base_ami}" ]] || [[ -z "${apm_ecs_agent_ami}" ]] || [[ -z "${apm_build_agent_ami}" ]]; then
-                echo failed to get ami ids
-              fi
-
-              echo "apm_ubuntu_base_ami = ${apm_ubuntu_base_ami}"
-              echo "apm_ecs_agent_ami = ${apm_ecs_agent_ami}"
-              echo "apm_build_agent_ami = ${apm_build_agent_ami}"
-
-              echo "apm_ubuntu_base_ami = \"${apm_ubuntu_base_ami}\"" >> ./dist/.ami_ids
-              echo "apm_ecs_agent_ami = \"${apm_ecs_agent_ami}\"" >> ./dist/.ami_ids
-              echo "apm_build_agent_ami = \"${apm_build_agent_ami}\"" >> ./dist/.ami_ids
-
-              cat ./dist/.ami_ids >> ./dist/.extra.tfvars
-
-            displayName: record AMI ids
-            workingDirectory: api-management-infrastructure
-
-          - bash: |
-              set -e
-              echo "extra .tfvars:"
-              cat ./dist/.extra.tfvars
-
-              cat ./dist/.extra.tfvars >> terraform/_shared/vars/ptl.tfvars
-              cat ./dist/.extra.tfvars >> terraform/_shared/vars/prod.tfvars
-
-            displayName: .extra.tfvars output and record
-            workingDirectory: api-management-infrastructure
-
-          - bash: |
-              set -e
-              mv ./dist ./dist_vars
-              make release
-              cp -r ./dist_vars/. ./dist
-            displayName: copy scripts to artifact
-            workingDirectory: api-management-infrastructure
-
-          - task: PublishPipelineArtifact@1
-            inputs:
-              targetPath: $(System.DefaultWorkingDirectory)/api-management-infrastructure/dist
-              artifactName: package
-
-          - bash: |
-              docker ps -q | xargs docker stop
-              docker container prune -f
-              docker network prune -f
-            displayName: Ensure docker containers and networks are brought down
-            condition: always()
diff --git a/azure/common/apigee-build.yml b/azure/common/apigee-build.yml
new file mode 100644
index 000000000..471c90572
--- /dev/null
+++ b/azure/common/apigee-build.yml
@@ -0,0 +1,284 @@
+parameters:
+  - name: service_name
+    type: string
+  - name: short_service_name
+    type: string
+  - name: variables
+    type: object
+    default: []
+  - name: secret_file_ids
+    type: object
+    default: []
+  - name: secret_ids
+    type: object
+    default: []
+  - name: config_ids
+    type: object
+    default: []
+  - name: test_steps
+    type: stepList
+    default: []
+  - name: post_ecs_push
+    type: stepList
+    default: []
+  - name: notify
+    type: boolean
+    default: true
+  - name: cache_steps
+    type: stepList
+    default: []
+
+jobs:
+  - job: build
+    displayName: Build & Test
+    timeoutInMinutes: 30
+    pool:
+      name: 'AWS-ECS'
+    workspace:
+      clean: all
+    variables:
+      ${{ each var in parameters.variables }}:
+        ${{ var.key }}: ${{ var.value }}
+    steps:
+
+      - bash: |
+          if [ ! -z "$(ls -A \"$(Pipeline.Workspace)/s/${{ parameters.service_name }}\" 2>/dev/null)" ]; then
+            echo "workspace directory is not empty!"
+            exit 1
+          fi
+        displayName: "check workspace is clean"
+
+      - bash: |
+          instance_id="$(curl -s http://169.254.169.254/latest/meta-data/instance-id)"
+          echo instance-id: "${instance_id}"
+          echo connect to: https://eu-west-2.console.aws.amazon.com/systems-manager/session-manager/${instance_id}
+          echo sudo su - ubuntu
+          or
+          echo ssh ubuntu@${instance_id}
+          echo working directory: $(System.DefaultWorkingDirectory)
+        displayName: print aws info
+
+      - template: ../components/aws-clean-config.yml
+
+      - ${{ if parameters.notify }}:
+        - template: ../components/aws-assume-role.yml
+          parameters:
+            role: "auto-ops"
+            profile: "apm_ptl"
+
+        - template: ../components/get-aws-secrets-and-ssm-params.yml
+          parameters:
+            secret_file_ids:
+              - ${{ each secret_file_id in parameters.secret_file_ids }}:
+                - ${{ secret_file_id }}
+            secret_ids:
+              - ptl/access-tokens/github/repo-status-update/GITHUB_ACCESS_TOKEN
+              - ${{ each secret_id in parameters.secret_ids }}:
+                - ${{ secret_id }}
+            config_ids:
+              - /ptl/azure-devops/GITHUB_USER
+              - ${{ each config_id in parameters.config_ids }}:
+                - ${{ config_id }}
+
+        - bash: |
+            echo "Build.SourceBranch: $(Build.SourceBranch)"
+            echo "Build.SourceBranchName: $(Build.SourceBranchName)"
+            echo "Build.SourceVersion: $(Build.SourceVersion)"
+            echo "Build.SourceVersionMessage: $(Build.SourceVersionMessage)"
+
+            if [[ ! -z $(NOTIFY_COMMIT_SHA) ]]; then
+                echo "##[debug]Using already provided NOTIFY_COMMIT_SHA=$(NOTIFY_COMMIT_SHA)"
+            else
+                NOTIFY_COMMIT_SHA=""
+
+                if [[ "$(Build.SourceBranch)" =~ ^refs/tags/.+$ ]]; then
+                    echo "##[debug]Build appears to be a tag build"
+                    echo "##[debug]Using Build.SourceVersion as NOTIFY_COMMIT_SHA"
+                    NOTIFY_COMMIT_SHA="$(Build.SourceVersion)"
+                fi
+
+                if [[ "$(Build.SourceBranch)" =~ ^refs/pull/.+$ ]]; then
+                    echo "##[debug]Build appears to be a pull request build"
+                    echo "##[debug]Extracting NOTIFY_COMMIT_SHA from Build.SourceVersionMessage"
+                    NOTIFY_COMMIT_SHA=`echo "$(Build.SourceVersionMessage)" | cut -d' ' -f2`
+                fi
+
+                if [[ -z $NOTIFY_COMMIT_SHA ]]; then
+                    echo "##[debug]Build does not appear to be pull or tag build"
+                    echo "##[debug]Using Build.SourceVersion as NOTIFY_COMMIT_SHA"
+                    NOTIFY_COMMIT_SHA="$(Build.SourceVersion)"
+                fi
+
+                echo "##vso[task.setvariable variable=NOTIFY_COMMIT_SHA]$NOTIFY_COMMIT_SHA"
+            fi
+          displayName: Set NOTIFY_COMMIT_SHA
+          condition: always()
+
+        - template: '../components/update-github-status.yml'
+          parameters:
+            state: pending
+            description: "Build started"
+
+      - bash: |
+          if [[ ! -z $(UTILS_PR_NUMBER) ]]; then
+            echo "##[debug]Triggered from utils repository, PR_NUMBER=$(UTILS_PR_NUMBER)"
+            echo "##vso[task.setvariable variable=PR_NUMBER]$(UTILS_PR_NUMBER)"
+          else
+            echo "##[debug]PR_NUMBER=$(System.PullRequest.PullRequestNumber)"
+            echo "##vso[task.setvariable variable=PR_NUMBER]$(System.PullRequest.PullRequestNumber)"
+          fi
+        displayName: Set PR_NUMBER
+
+      - checkout: self
+        path: "s/${{ parameters.service_name }}"
+        submodules: true
+
+      - template: ../templates/setup-build-name.yml
+        parameters:
+          service_name: "${{ parameters.service_name }}"
+
+      - task: UsePythonVersion@0
+        displayName: "Use Python 3.9"
+        inputs:
+          versionSpec: "3.9"
+
+      - ${{ each cache_step in parameters.cache_steps }}:
+          - ${{ cache_step }}
+
+      - bash: |
+          n=0
+          until [ "$n" -ge 3 ]
+          do
+            make install && break
+            n=$((n+1))
+          done
+        workingDirectory: "${{ parameters.service_name }}"
+        displayName: Install project dependencies
+
+      - bash: "make lint"
+        workingDirectory: "${{ parameters.service_name }}"
+        displayName: Lint
+
+      - ${{ each test_step in parameters.test_steps }}:
+          - ${{ test_step }}
+
+      - bash: "make publish"
+        workingDirectory: "${{ parameters.service_name }}"
+        displayName: Compile spec
+
+      - bash: "make release"
+        workingDirectory: "${{ parameters.service_name }}"
+        displayName: Build release package
+
+      - bash: |
+          if [[ -f ecs-proxies-containers.yml ]]; then
+            echo "##vso[task.setvariable variable=build_containers]true"
+          else
+            echo "##vso[task.setvariable variable=build_containers]false"
+          fi
+        workingDirectory: ${{ parameters.service_name }}
+        displayName: "Check for ECS proxy"
+
+      - checkout: common
+        path: "s/${{ parameters.service_name }}/utils"
+
+      - task: s3-cache-action@1
+        inputs:
+          key: poetry | $(System.DefaultWorkingDirectory) | ${{ parameters.service_name }}/utils/poetry.lock
+          location: "${{ parameters.service_name }}/utils/.venv"
+          debug: true
+          alias: 'Utils'
+        displayName: cache utils dependencies
+
+      - bash: |
+            make install
+            sleep 5
+        workingDirectory: "${{ parameters.service_name }}/utils"
+        condition: ne(variables['CacheRestored-Utils'], 'true')
+        displayName: "Install utils "
+
+      - bash: |
+          export out_dir="$(realpath ${{ parameters.service_name }}/dist)"
+          export commit_hash="$(git -C ${{ parameters.service_name }} rev-parse --short HEAD)"
+          # sha prefix is required docker gets upset if names contain -0 iirc
+          export build_label="$(Build.BuildId)-sha${commit_hash}"
+          export pr_number="$(PR_NUMBER)"
+          export service_name="${{ parameters.service_name }}"
+          export service_id="${{ parameters.short_service_name }}"
+          export ANSIBLE_FORCE_COLOR=yes
+          make --no-print-directory -C ${{ parameters.service_name }}/utils/ansible create-build-env-vars
+        displayName: output build env vars for artifact
+        condition: and(succeeded(), eq(variables['build_containers'], 'true'))
+
+      - bash: |
+          tfenv use 0.14.6
+        displayName: use terraforn
+        condition: and(succeeded(), eq(variables['build_containers'], 'true'))
+
+      - bash: |
+          source "${{ parameters.service_name }}/dist/.build_env_vars"
+          account=ptl make --no-print-directory -C ${{ parameters.service_name }}/utils/ansible create-ecr-build-role
+          echo "##vso[task.setvariable variable=BUILD_ROLE]build-${service_id}"
+        displayName: "ensure build role"
+        condition: and(succeeded(), eq(variables['build_containers'], 'true'))
+
+      - template: ../components/aws-assume-role.yml
+        parameters:
+          role: "$(BUILD_ROLE)"
+          profile: "$(BUILD_ROLE)"
+
+      - bash: |
+          source "${{ parameters.service_name }}/dist/.build_env_vars"
+          CONTAINER_VARS_FILE="$(realpath ${{ parameters.service_name }}/ecs-proxies-containers.yml)" \
+          make --no-print-directory -C ${{ parameters.service_name }}/utils/ansible build-ecs-proxies
+        displayName: "Build and push ECS proxies"
+        condition: and(succeeded(), eq(variables['build_containers'], 'true'))
+
+      - ${{ each post_ecs_push in parameters.post_ecs_push }}:
+          - ${{ post_ecs_push }}
+
+      - bash: |
+          cp ${{ parameters.service_name }}/ecs-*.yml ${{ parameters.service_name }}/dist || true
+        displayName: "Copy ecs-proxies-deploy configs into build artifact"
+        condition: and(succeeded(), eq(variables['build_containers'], 'true'))
+
+      - bash: |
+          rsync -a utils dist --exclude .venv
+        workingDirectory: "${{ parameters.service_name }}"
+        displayName: "Copy utils into artifact"
+
+      - bash: |
+          set -euo pipefail
+          export DIST_DIR=`pwd`/dist
+          export ANSIBLE_FORCE_COLOR=yes
+          if [ -f manifest.yml ]; then
+             rsync -a manifest.yml dist
+             make -C utils/ansible validate-manifest
+          elif [ -f manifest_template.yml ]; then
+             rsync -a manifest_template.yml dist
+             make -C utils/ansible template-manifest
+             make -C utils/ansible validate-manifest
+          fi
+        workingDirectory: "${{ parameters.service_name }}"
+        displayName: "Validate manifest (template) and copy to artifact"
+
+      - bash: |
+          cd ${{ parameters.service_name }}/utils
+          git rev-parse HEAD > ../dist/.utils-version
+        displayName: Snapshot utils version
+
+      - publish: ${{ parameters.service_name}}/dist
+        artifact: "$(Build.BuildNumber)"
+
+      - ${{ if parameters.notify }}:
+        - template: '../components/update-github-status.yml'
+          parameters:
+            state: success
+            on_success: true
+            description: "Build succeeded"
+
+        - template: '../components/update-github-status.yml'
+          parameters:
+            state: failure
+            on_failure: true
+            description: "Build failed"