Automated Tests

.github/workflows/lint.yml

@@ -1,8 +1,8 @@
 name: Lint this repository
 on:
   pull_request:
-    types: [ opened, reopened, synchronize, ready_for_review ]
-    branches: [ main ]
+    types: [opened, reopened, synchronize, ready_for_review]
+    branches: [main]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -16,7 +16,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@ff7abcd0c3c05ccf6adc123a8cd1fd4fb30fb493
-      
+
       - name: Check documentation exists
         shell: python
         run: |
@@ -31,10 +31,10 @@ jobs:
 
           for reusable_workflow in reusable_workflow_filenames:
               workflow_name = reusable_workflow.stem
-              expected_doc_path = pathlib.Path(f'docs/{workflow_name}.md')
+              expected_doc_path = pathlib.Path(f'docs/workflows/{workflow_name}.md')
               if not expected_doc_path.exists():
                   missing_documentation.append(str(expected_doc_path))
-          
+
           if missing_documentation:
               raise ValueError(f"Missing documentation for reusable workflows: {', '.join(missing_documentation)}")
           else:

.github/workflows/reusable-pr-label-by-branch.yml

@@ -22,7 +22,7 @@ jobs:
           jq -e 'if map(select(.name | contains ("major"))) == [] then null else "Label major exists" end' labels.json || gh label create major --color "b60205"
   label-pr:
     name: Label Pull Request
-    needs: [ configure-labels ]
+    needs: [configure-labels]
     permissions:
       contents: read
       pull-requests: write
@@ -50,7 +50,7 @@ jobs:
               echo "updates_configuration=false" >> $GITHUB_OUTPUT
             fi
           if [[ $PR_NUMBER -eq "1" ]]; then
-            if git cat-file -e ${{ github.event.repository.default_branch }}:.github/release-drafter.yml 2>/dev/null; then
+            if git cat-file -e origin/${{ github.event.repository.default_branch }}:.github/release-drafter.yml 2>/dev/null; then
               # First PR, but we have a release-drafter.yml file on the main branch so we can let it proceed as normal.
               echo "auto_label=true" >> $GITHUB_OUTPUT
             else
@@ -63,7 +63,7 @@ jobs:
             # Not the first PR, so we can attempt to auto-label it.
             echo "auto_label=true" >> $GITHUB_OUTPUT
           fi
-          
+
           rm first-time-comment.md
       - uses: release-drafter/release-drafter@6db134d15f3909ccc9eefd369f02bd1e9cffdf97
         if: steps.prerequisite-check.outputs.auto_label == 'true'

.github/workflows/reusable-release-on-merge.yml

@@ -23,11 +23,6 @@ on:
         description: 'Whether to disable the autolabeler. Autolabeler should remain disabled if you label the pull requests with another workflow. Default: true'
         required: false
         default: true
-      run_post_release_workflow:
-        type: boolean
-        description: 'Trigger a post-release workflow at .github/workflows/release-post-actions.yml within your repository that receives a "version" input with the version that was just released. This workflow can do whatever you need, but generally is used to e.g. release a package to a package regsitry. Default: false'
-        required: false
-        default: false
     outputs:
       release_version:
         description: 'The version of the release that was created by this workflow. This is the tag name of the release.'

.github/workflows/test.yml

@@ -0,0 +1,57 @@
+name: Run Tests
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+
+permissions:
+  contents: read
+  checks: write
+  pull-requests: write
+
+jobs:
+  build:
+    name: Run Tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ "3.13" ]
+    steps:
+    - uses: actions/checkout@8edcb1bdb4e267140fa742c62e395cd74f332709
+
+    - name: Restore cached asdf tools
+      uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684
+      id: cache
+      with:
+        path: ~/.asdf
+        key: ${{ runner.os }}-tool-versions-${{ hashFiles('.tool-versions') }}
+
+    - name: asdf install
+      uses: asdf-vm/actions/install@1902764435ca0dd2f3388eea723a4f92a4eb8302 
+
+    - name: Cache asdf tools
+      uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684
+      id: save-cache
+      if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+      with:
+        path: ~/.asdf
+        key: ${{ runner.os }}-tool-versions-${{ hashFiles('.tool-versions') }}
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e
+      id: app-token
+      with:
+        app-id: ${{ vars.APP_ID }}
+        owner: ${{ vars.APP_OWNER }}
+        private-key: ${{ secrets.APP_PRIVATE_KEY }}
+
+    - name: Test with pytest
+      env:
+        LAUNCH_WORKFLOWS_REF_TO_TEST: ${{ github.head_ref }}
+        GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
+      run: |
+        uv run pytest

.gitignore

@@ -0,0 +1,4 @@
+.pytest_cache
+__pycache__
+*.py[cod]
+.venv

.tool-versions

@@ -0,0 +1 @@
+uv 0.8.4

docs/testing/1-configuring-organization.md

@@ -0,0 +1,26 @@
+# Configure an Organization for running Workflow Tests
+
+Our tests will end up creating repositories, and so we isolate them to a particular GitHub Organization. You will need to be an `Owner` of the organization to set this up correctly.
+
+## Ensure the Organization is configured for GitHub Actions
+
+First, enter the Organization's Settings page. Navigate to Actions > General blade and set the following:
+
+- **Policies** must be set to allow usage of our actions from the `launchbynttdata` organization. Additionally, we use third-party actions contained in other organizations, so you must either explicitly allow the third-party actions we use or allow all actions. Since this Organization should be isolated for testing and have no other use, the recommendation is to set `Allow all actions and reusable workflows`.
+
+- **Require actions to be pinned to a full-length commit SHA** must NOT be set.
+
+- **Approval for running fork pull request workflows from contributors** should be set to `Require approval for all external contributors`
+
+- **Workflow permissions** should be set to `Read repository contents and packages permissions`
+
+- **Allow GitHub Actions to create and approve pull requests** must NOT be set.
+
+- Ensure you save your changes before continuing!
+
+## Install the GitHub App
+
+Our workflow testing utilizes a [GitHub App](https://github.com/apps/launch-workflow-testing) to provide authentication and manage permissions. You will need to install this App to the organization you wish to use.
+
+> [!TIP]
+> The basics are now configured for your organization, but there is still work to do. You must configure OIDC between the organization and the cloud providers, which is covered in the [next sections](2-configuring-aws-oidc.md).

docs/testing/2-configuring-aws-oidc.md

@@ -0,0 +1,54 @@
+# Configuring OIDC to an AWS Account
+
+This document provides a quick overview of the process. You can also refer to the detailed [documentation from GitHub](https://docs.github.com/en/actions/how-tos/secure-your-work/security-harden-deployments/oidc-in-aws).
+
+## Add Identity Provider to AWS
+
+Create a new Identity Provider configuration within AWS IAM. Select the `OpenID Connect` option, and provide the following values:
+
+> **Provider URL**: https://token.actions.githubusercontent.com
+> **Audience**: sts.amazonaws.com
+
+## Create a Policy to control access to AWS resources
+
+In the IAM section, you will need to create a new Policy object that allows actions you will need for testing. Keep in mind that the identity that is creating your test resources is also creating an S3 bucket for state management and entitle it appropriately.
+
+## Create a Role and establish Trust
+
+Create a Role object and assign it to the policy you just created. Create a Trust Relationship for your role, replacing the `{templated}` items with values that correspond to your environment and IAM objects:
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Principal": {
+                "Federated": "arn:aws:iam::{your AWS account number}}:oidc-provider/token.actions.githubusercontent.com"
+            },
+            "Action": "sts:AssumeRoleWithWebIdentity",
+            "Condition": {
+                "StringEquals": {
+                    "token.actions.githubusercontent.com:aud": "sts.amazonaws.com"
+                },
+                "StringLike": {
+                    "token.actions.githubusercontent.com:sub": [
+                        "repo:{test organization name}/test-repo-*:*",
+                    ]
+                }
+            }
+        }
+    ]
+}
+```
+
+## Configure GitHub Organization
+
+Finally, the workflows are expecting a setting to configure which Role they assume when making the OIDC request. In GitHub, open the Organization's Settings page, navigate to the Secrets and Variables > Actions section. Create a new **Variable** (not a secret) as shown below:
+
+> DEPLOY_ROLE_ARN = {ARN of the role you created earlier}
+
+Ensure that you update the **Repository Access** setting to `All repositories`.
+
+> [!TIP]
+> AWS OIDC is now configured for your organization. To configure Azure, see the [next section](3-configuring-azure-oidc.md).

docs/testing/3-configuring-azure-oidc.md

@@ -0,0 +1,61 @@
+# Configuring OIDC to an Azure Tenant
+
+This document provides a quick overview of the process. You can also refer to the detailed [documentation from GitHub](https://docs.github.com/en/actions/how-tos/secure-your-work/security-harden-deployments/oidc-in-azure).
+
+## Create an App Registration and Federated Identity Credential
+
+> [!NOTE]
+> This requires elevated permissions in your Azure tenant.
+
+Start by navigating to **App Registrations** and create a new one with the name of your choice. The value of **Supported account types** will vary depending on your Azure setup, but for Launch, we use the **Single Tenant** option. It is not necessary to supply a redirect URL.
+
+
+
+Create a new Identity Provider configuration within AWS IAM. Select the `OpenID Connect` option, and provide the following values:
+
+> **Provider URL**: https://token.actions.githubusercontent.com
+> **Audience**: sts.amazonaws.com
+
+## Create a Policy to control access to AWS resources
+
+In the IAM section, you will need to create a new Policy object that allows actions you will need for testing. Keep in mind that the identity that is creating your test resources is also creating an S3 bucket for state management and entitle it appropriately.
+
+## Create a Role and establish Trust
+
+Create a Role object and assign it to the policy you just created. Create a Trust Relationship for your role, replacing the `{templated}` items with values that correspond to your environment and IAM objects:
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Principal": {
+                "Federated": "arn:aws:iam::{your AWS account number}}:oidc-provider/token.actions.githubusercontent.com"
+            },
+            "Action": "sts:AssumeRoleWithWebIdentity",
+            "Condition": {
+                "StringEquals": {
+                    "token.actions.githubusercontent.com:aud": "sts.amazonaws.com"
+                },
+                "StringLike": {
+                    "token.actions.githubusercontent.com:sub": [
+                        "repo:{test organization name}/test-repo-*:*",
+                    ]
+                }
+            }
+        }
+    ]
+}
+```
+
+## Configure GitHub Organization
+
+Finally, the workflows are expecting a setting to configure which Role they assume when making the OIDC request. In GitHub, open the Organization's Settings page, navigate to the Secrets and Variables > Actions section. Create a new **Variable** (not a secret) as shown below:
+
+> DEPLOY_ROLE_ARN = {ARN of the role you created earlier}
+
+Ensure that you update the **Repository Access** setting to `All repositories`.
+
+> [!TIP]
+> AWS OIDC is now configured for your organization. To configure Azure, see the [next section](3-configuring-azure-oidc.md).

docs/reusable-release-draft-on-merge.md → docs/workflows/reusable-release-draft-on-merge.md

@@ -54,3 +54,16 @@ jobs:
 ```
 
 Be sure you replace `ref` with an appropriate ref to this repository.
+
+If this workflow runs in a repository without a prior Release being published, the first release will always be drafted with version 0.1.0. This is expected behavior and conforms to the SemVer spec, which permits a major version zero. From the SemVer documentation:
+
+> How should I deal with revisions in the 0.y.z initial development phase?
+> The simplest thing to do is start your initial development release at 0.1.0 and then increment the minor version for each subsequent release.
+>
+> How do I know when to release 1.0.0?
+> If your software is being used in production, it should probably already be 1.0.0. If you have a stable API on which users have come to depend, you should be 1.0.0. If you’re worrying a lot about backward compatibility, you should probably already be 1.0.0.
+>
+> Doesn’t this discourage rapid development and fast iteration?
+> Major version zero is all about rapid development. If you’re changing the API every day you should either still be in version 0.y.z or on a separate development branch working on the next major version.
+
+Given that this workflow only drafts the release, you may manually edit the drafted release to rename and re-tag it with the version of your choosing if you wanted your first release to reflect 1.0.0 prior to publishing. If you are integrating this workflow into a repository that has tags, but no releases, you will need to perform this manual step to make sure your first true release follows your existing tags. If you integrate this workflow into a repository that has prior releases, the next drafted release will take the prior releases into account and will increment its version according to the tags found on the pull request that initiates the workflow run.

docs/reusable-release-on-merge.md → docs/workflows/reusable-release-on-merge.md

@@ -51,3 +51,16 @@ jobs:
 ```
 
 Be sure you replace `ref` with an appropriate ref to this repository.
+
+If this workflow runs in a repository without a prior Release being published, the first release will always be drafted with version 0.1.0. This is expected behavior and conforms to the SemVer spec, which permits a major version zero. From the SemVer documentation:
+
+> How should I deal with revisions in the 0.y.z initial development phase?
+> The simplest thing to do is start your initial development release at 0.1.0 and then increment the minor version for each subsequent release.
+>
+> How do I know when to release 1.0.0?
+> If your software is being used in production, it should probably already be 1.0.0. If you have a stable API on which users have come to depend, you should be 1.0.0. If you’re worrying a lot about backward compatibility, you should probably already be 1.0.0.
+>
+> Doesn’t this discourage rapid development and fast iteration?
+> Major version zero is all about rapid development. If you’re changing the API every day you should either still be in version 0.y.z or on a separate development branch working on the next major version.
+
+Since this workflow publishes the release when a PR is merged (rather than leaving it in a drafted state), the first PR merge will generally result in version 0.1.0 being published. Do not integrate this workflow into a repository that contains tags but no releases, you must first create a Release with the last tagged version in order for the next PR merge to provide the correct version bump. If you integrate this workflow into a repository that has prior releases, the next release will take the prior releases into account and will increment its version according to the tags found on the pull request that initiates the workflow run.

pyproject.toml

@@ -0,0 +1,20 @@
+[project]
+name = "launch-workflows-testing"
+version = "0.0.0"
+description = "Tests for the launch-workflows project"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "gitpython>=3.1.45",
+    "pygithub>=2.7.0",
+    "pytest>=8.4.1",
+    "pytest-xdist>=3.8.0",
+]
+
+[tool.pytest.ini_options]
+pythonpath = "src"
+minversion = "8.0"
+addopts = "-ra -n auto"
+testpaths = [
+    "test"
+]