diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 3d7f5af32..58983676d 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -12,10 +12,10 @@ jobs: with: fetch-depth: 0 # This causes all history to be fetched, which is required for calculate-version to function - - name: Install Python 3.8 - uses: actions/setup-python@v1 + - name: Install Python 3.9 + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Update apt repositories run: sudo apt update diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 76e0c7a34..19d8f07e6 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -10,11 +10,11 @@ jobs: working-directory: ./docker/sds-api/spine-directory-service/sds steps: - - uses: actions/checkout@v1 - - name: Set up Python 3.8 - uses: actions/setup-python@v1 + - uses: actions/checkout@v2 + - name: Set up Python 3.9 + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Install pipenv uses: dschep/install-pipenv-action@v1 with: diff --git a/README.md b/README.md index 01d043a81..e4488b7ca 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ Connect to the VPN, You may have issues with the container connecting via the VPN. If so please look at the answer given here https://superuser.com/questions/1579858/docker-bridge-network-sporadically-loosing-packets/1580017?_gl=1*wyte41*_ga*MjgwODQyNzEwLjE3MDYwMDMwNzQ.*_ga_S812YQPLT2*MTcwNjE5MjIyOC4yLjAuMTcwNjE5MjIyOC4wLjAuMA..#1580017 using `docker network create --subnet=172.20.0.0/24 --gateway=172.20.0.1 docker20` Finally run + ```sh docker-compose up ``` diff --git a/azure/common/apigee-build.yml b/azure/common/apigee-build.yml new file mode 100644 index 000000000..471c90572 --- /dev/null +++ b/azure/common/apigee-build.yml @@ -0,0 +1,284 @@ +parameters: + - name: service_name + type: string + - name: short_service_name + type: string + - name: variables + type: object + default: [] + - name: secret_file_ids + type: object + default: [] + - name: secret_ids + type: object + default: [] + - name: config_ids + type: object + default: [] + - name: test_steps + type: stepList + default: [] + - name: post_ecs_push + type: stepList + default: [] + - name: notify + type: boolean + default: true + - name: cache_steps + type: stepList + default: [] + +jobs: + - job: build + displayName: Build & Test + timeoutInMinutes: 30 + pool: + name: 'AWS-ECS' + workspace: + clean: all + variables: + ${{ each var in parameters.variables }}: + ${{ var.key }}: ${{ var.value }} + steps: + + - bash: | + if [ ! -z "$(ls -A \"$(Pipeline.Workspace)/s/${{ parameters.service_name }}\" 2>/dev/null)" ]; then + echo "workspace directory is not empty!" + exit 1 + fi + displayName: "check workspace is clean" + + - bash: | + instance_id="$(curl -s http://169.254.169.254/latest/meta-data/instance-id)" + echo instance-id: "${instance_id}" + echo connect to: https://eu-west-2.console.aws.amazon.com/systems-manager/session-manager/${instance_id} + echo sudo su - ubuntu + or + echo ssh ubuntu@${instance_id} + echo working directory: $(System.DefaultWorkingDirectory) + displayName: print aws info + + - template: ../components/aws-clean-config.yml + + - ${{ if parameters.notify }}: + - template: ../components/aws-assume-role.yml + parameters: + role: "auto-ops" + profile: "apm_ptl" + + - template: ../components/get-aws-secrets-and-ssm-params.yml + parameters: + secret_file_ids: + - ${{ each secret_file_id in parameters.secret_file_ids }}: + - ${{ secret_file_id }} + secret_ids: + - ptl/access-tokens/github/repo-status-update/GITHUB_ACCESS_TOKEN + - ${{ each secret_id in parameters.secret_ids }}: + - ${{ secret_id }} + config_ids: + - /ptl/azure-devops/GITHUB_USER + - ${{ each config_id in parameters.config_ids }}: + - ${{ config_id }} + + - bash: | + echo "Build.SourceBranch: $(Build.SourceBranch)" + echo "Build.SourceBranchName: $(Build.SourceBranchName)" + echo "Build.SourceVersion: $(Build.SourceVersion)" + echo "Build.SourceVersionMessage: $(Build.SourceVersionMessage)" + + if [[ ! -z $(NOTIFY_COMMIT_SHA) ]]; then + echo "##[debug]Using already provided NOTIFY_COMMIT_SHA=$(NOTIFY_COMMIT_SHA)" + else + NOTIFY_COMMIT_SHA="" + + if [[ "$(Build.SourceBranch)" =~ ^refs/tags/.+$ ]]; then + echo "##[debug]Build appears to be a tag build" + echo "##[debug]Using Build.SourceVersion as NOTIFY_COMMIT_SHA" + NOTIFY_COMMIT_SHA="$(Build.SourceVersion)" + fi + + if [[ "$(Build.SourceBranch)" =~ ^refs/pull/.+$ ]]; then + echo "##[debug]Build appears to be a pull request build" + echo "##[debug]Extracting NOTIFY_COMMIT_SHA from Build.SourceVersionMessage" + NOTIFY_COMMIT_SHA=`echo "$(Build.SourceVersionMessage)" | cut -d' ' -f2` + fi + + if [[ -z $NOTIFY_COMMIT_SHA ]]; then + echo "##[debug]Build does not appear to be pull or tag build" + echo "##[debug]Using Build.SourceVersion as NOTIFY_COMMIT_SHA" + NOTIFY_COMMIT_SHA="$(Build.SourceVersion)" + fi + + echo "##vso[task.setvariable variable=NOTIFY_COMMIT_SHA]$NOTIFY_COMMIT_SHA" + fi + displayName: Set NOTIFY_COMMIT_SHA + condition: always() + + - template: '../components/update-github-status.yml' + parameters: + state: pending + description: "Build started" + + - bash: | + if [[ ! -z $(UTILS_PR_NUMBER) ]]; then + echo "##[debug]Triggered from utils repository, PR_NUMBER=$(UTILS_PR_NUMBER)" + echo "##vso[task.setvariable variable=PR_NUMBER]$(UTILS_PR_NUMBER)" + else + echo "##[debug]PR_NUMBER=$(System.PullRequest.PullRequestNumber)" + echo "##vso[task.setvariable variable=PR_NUMBER]$(System.PullRequest.PullRequestNumber)" + fi + displayName: Set PR_NUMBER + + - checkout: self + path: "s/${{ parameters.service_name }}" + submodules: true + + - template: ../templates/setup-build-name.yml + parameters: + service_name: "${{ parameters.service_name }}" + + - task: UsePythonVersion@0 + displayName: "Use Python 3.9" + inputs: + versionSpec: "3.9" + + - ${{ each cache_step in parameters.cache_steps }}: + - ${{ cache_step }} + + - bash: | + n=0 + until [ "$n" -ge 3 ] + do + make install && break + n=$((n+1)) + done + workingDirectory: "${{ parameters.service_name }}" + displayName: Install project dependencies + + - bash: "make lint" + workingDirectory: "${{ parameters.service_name }}" + displayName: Lint + + - ${{ each test_step in parameters.test_steps }}: + - ${{ test_step }} + + - bash: "make publish" + workingDirectory: "${{ parameters.service_name }}" + displayName: Compile spec + + - bash: "make release" + workingDirectory: "${{ parameters.service_name }}" + displayName: Build release package + + - bash: | + if [[ -f ecs-proxies-containers.yml ]]; then + echo "##vso[task.setvariable variable=build_containers]true" + else + echo "##vso[task.setvariable variable=build_containers]false" + fi + workingDirectory: ${{ parameters.service_name }} + displayName: "Check for ECS proxy" + + - checkout: common + path: "s/${{ parameters.service_name }}/utils" + + - task: s3-cache-action@1 + inputs: + key: poetry | $(System.DefaultWorkingDirectory) | ${{ parameters.service_name }}/utils/poetry.lock + location: "${{ parameters.service_name }}/utils/.venv" + debug: true + alias: 'Utils' + displayName: cache utils dependencies + + - bash: | + make install + sleep 5 + workingDirectory: "${{ parameters.service_name }}/utils" + condition: ne(variables['CacheRestored-Utils'], 'true') + displayName: "Install utils " + + - bash: | + export out_dir="$(realpath ${{ parameters.service_name }}/dist)" + export commit_hash="$(git -C ${{ parameters.service_name }} rev-parse --short HEAD)" + # sha prefix is required docker gets upset if names contain -0 iirc + export build_label="$(Build.BuildId)-sha${commit_hash}" + export pr_number="$(PR_NUMBER)" + export service_name="${{ parameters.service_name }}" + export service_id="${{ parameters.short_service_name }}" + export ANSIBLE_FORCE_COLOR=yes + make --no-print-directory -C ${{ parameters.service_name }}/utils/ansible create-build-env-vars + displayName: output build env vars for artifact + condition: and(succeeded(), eq(variables['build_containers'], 'true')) + + - bash: | + tfenv use 0.14.6 + displayName: use terraforn + condition: and(succeeded(), eq(variables['build_containers'], 'true')) + + - bash: | + source "${{ parameters.service_name }}/dist/.build_env_vars" + account=ptl make --no-print-directory -C ${{ parameters.service_name }}/utils/ansible create-ecr-build-role + echo "##vso[task.setvariable variable=BUILD_ROLE]build-${service_id}" + displayName: "ensure build role" + condition: and(succeeded(), eq(variables['build_containers'], 'true')) + + - template: ../components/aws-assume-role.yml + parameters: + role: "$(BUILD_ROLE)" + profile: "$(BUILD_ROLE)" + + - bash: | + source "${{ parameters.service_name }}/dist/.build_env_vars" + CONTAINER_VARS_FILE="$(realpath ${{ parameters.service_name }}/ecs-proxies-containers.yml)" \ + make --no-print-directory -C ${{ parameters.service_name }}/utils/ansible build-ecs-proxies + displayName: "Build and push ECS proxies" + condition: and(succeeded(), eq(variables['build_containers'], 'true')) + + - ${{ each post_ecs_push in parameters.post_ecs_push }}: + - ${{ post_ecs_push }} + + - bash: | + cp ${{ parameters.service_name }}/ecs-*.yml ${{ parameters.service_name }}/dist || true + displayName: "Copy ecs-proxies-deploy configs into build artifact" + condition: and(succeeded(), eq(variables['build_containers'], 'true')) + + - bash: | + rsync -a utils dist --exclude .venv + workingDirectory: "${{ parameters.service_name }}" + displayName: "Copy utils into artifact" + + - bash: | + set -euo pipefail + export DIST_DIR=`pwd`/dist + export ANSIBLE_FORCE_COLOR=yes + if [ -f manifest.yml ]; then + rsync -a manifest.yml dist + make -C utils/ansible validate-manifest + elif [ -f manifest_template.yml ]; then + rsync -a manifest_template.yml dist + make -C utils/ansible template-manifest + make -C utils/ansible validate-manifest + fi + workingDirectory: "${{ parameters.service_name }}" + displayName: "Validate manifest (template) and copy to artifact" + + - bash: | + cd ${{ parameters.service_name }}/utils + git rev-parse HEAD > ../dist/.utils-version + displayName: Snapshot utils version + + - publish: ${{ parameters.service_name}}/dist + artifact: "$(Build.BuildNumber)" + + - ${{ if parameters.notify }}: + - template: '../components/update-github-status.yml' + parameters: + state: success + on_success: true + description: "Build succeeded" + + - template: '../components/update-github-status.yml' + parameters: + state: failure + on_failure: true + description: "Build failed" diff --git a/docker/sds-api/integration-tests/Pipfile b/docker/sds-api/integration-tests/Pipfile index d420fe280..840c79b8d 100644 --- a/docker/sds-api/integration-tests/Pipfile +++ b/docker/sds-api/integration-tests/Pipfile @@ -10,7 +10,7 @@ requests = "*" unittest-xml-reporting = "*" [requires] -python_version = "3.8" +python_version = "3.9" [scripts] inttests = 'python -m xmlrunner discover -o test-reports ./tests' diff --git a/docker/sds-api/integration-tests/Pipfile.lock b/docker/sds-api/integration-tests/Pipfile.lock index 1eb129910..09cf51d8e 100644 --- a/docker/sds-api/integration-tests/Pipfile.lock +++ b/docker/sds-api/integration-tests/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "bcb6c637eff939b2e99cc53183abf56b04463b68cf07c8d95a9086f6be59e6c3" + "sha256": "f6748174d7800b666bf195a7924a74556979f6b58259e60a97305d69aba11bbe" }, "pipfile-spec": 6, "requires": { - "python_version": "3.8" + "python_version": "3.9" }, "sources": [ { diff --git a/docker/sds-api/spine-directory-service/sds/Dockerfile b/docker/sds-api/spine-directory-service/sds/Dockerfile index 7c35937b0..88d7ae098 100644 --- a/docker/sds-api/spine-directory-service/sds/Dockerfile +++ b/docker/sds-api/spine-directory-service/sds/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-slim-bullseye as base +FROM python:3.9-slim-bullseye as base RUN apt-get update && \ apt-get install build-essential -y && \ diff --git a/docker/sds-api/spine-directory-service/sds/Dockerfile_local b/docker/sds-api/spine-directory-service/sds/Dockerfile_local index 8ddc09f83..9b886156c 100644 --- a/docker/sds-api/spine-directory-service/sds/Dockerfile_local +++ b/docker/sds-api/spine-directory-service/sds/Dockerfile_local @@ -1,4 +1,4 @@ -FROM python:3.8-slim-bullseye as base +FROM python:3.9-slim-bullseye as base RUN apt-get update && \ apt-get install build-essential -y && \ diff --git a/docker/sds-api/spine-directory-service/sds/Pipfile b/docker/sds-api/spine-directory-service/sds/Pipfile index 42c63fc34..07c15c31e 100644 --- a/docker/sds-api/spine-directory-service/sds/Pipfile +++ b/docker/sds-api/spine-directory-service/sds/Pipfile @@ -14,7 +14,7 @@ isodate = "~=0.6" python-json-logger = "~=2.0.1" [requires] -python_version = "3.8" +python_version = "3.9" [scripts] unittests = 'python -m xmlrunner -o test-reports -v' diff --git a/docker/sds-api/spine-directory-service/sds/Pipfile.lock b/docker/sds-api/spine-directory-service/sds/Pipfile.lock index db3dd89d9..787160902 100644 --- a/docker/sds-api/spine-directory-service/sds/Pipfile.lock +++ b/docker/sds-api/spine-directory-service/sds/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "e2c66dbb7cf0fb3e4c4f2320795e35c376af172a6ba85e913802d6e9f774d42f" + "sha256": "1034dcf8bd32f87ac3551787c40f6341ca5fd965c1b3886861bb0dcb99db4d23" }, "pipfile-spec": 6, "requires": { - "python_version": "3.8" + "python_version": "3.9" }, "sources": [ { diff --git a/poetry.lock b/poetry.lock index 62cb0d2ea..94a61c436 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1284,5 +1284,5 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" -python-versions = "^3.8" -content-hash = "7a27d1177de4e671bd76d48c0848b9e7b5f9506b232584f41e2dce3f7d21badd" +python-versions = "^3.9" +content-hash = "082377499f94dc7de37e7cc579a7802ed82ec91f9a9a0b8b31fcf6999e38bc4c" diff --git a/pyproject.toml b/pyproject.toml index 3c7273db7..479f41e1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [project] -python = "^3.8" +python = "^3.9" [tool.poetry] @@ -22,7 +22,7 @@ keywords = ["healthcare", "uk", "nhs"] #TODO add additional keywords [tool.poetry.dependencies] -python = "^3.8" +python = "^3.9" pyyaml = "^5.2" docopt = "^0.6.2" jsonpath-rw = "^1.4.0"