From d634825b2d5278f904d55c7b51cbe31b5f9b9abd Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Wed, 18 Mar 2026 14:17:23 +0000
Subject: [PATCH 01/30] fix: no logs in terminal when not logging locally

---
 src/main.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/main.py b/src/main.py
index d75891a..f20606d 100644
--- a/src/main.py
+++ b/src/main.py
@@ -406,6 +406,14 @@ def handler(event: dict, context) -> str:  # pylint: disable=unused-argument, to
         logging.basicConfig(
             filename="debug.log",
             filemode="w",
+            format="%(asctime)s %(levelname)s %(message)s",
+
+        )
+    else:
+        # Ensure INFO logs show in the terminal when not logging to a file
+        logging.basicConfig(
+            level=logging.INFO,
+            format="%(asctime)s %(levelname)s %(message)s",
         )
 
     # Create an S3 client
@@ -485,5 +493,5 @@ def handler(event: dict, context) -> str:  # pylint: disable=unused-argument, to
 
 # # Dev Only
 # # Uncomment the following line to run the script locally
-# if __name__ == "__main__":
-#     handler(None, None)
+if __name__ == "__main__":
+    handler(None, None)

From 8d3d604c1303dc9dc3bd5c842ca6ecfbcf7111e4 Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Wed, 18 Mar 2026 15:43:05 +0000
Subject: [PATCH 02/30] refactor: change usage data source to new endpoint

---
 src/main.py | 38 ++++++++++++++++++++++----------------
 1 file changed, 22 insertions(+), 16 deletions(-)

diff --git a/src/main.py b/src/main.py
index f20606d..9a6eea4 100644
--- a/src/main.py
+++ b/src/main.py
@@ -14,6 +14,7 @@
 import github_api_toolkit
 from botocore.exceptions import ClientError
 from requests import Response
+import urllib.request
 
 # GitHub Organisation
 org = os.getenv("GITHUB_ORG")
@@ -129,27 +130,32 @@ def get_and_update_historic_usage(
         tuple: A tuple containing the updated historic usage data and a list of dates added.
     """
     # Get the usage data
-    usage_data = gh.get(f"/orgs/{org}/copilot/metrics")
-    usage_data = usage_data.json()
+    api_response = gh.get(f"/orgs/{org}/copilot/metrics/reports/organization-28-day/latest").json()
+    usage_data = json.loads(urllib.request.urlopen(api_response["download_links"][0]).read())
 
     logger.info("Usage data retrieved")
 
-    try:
-        response = s3.get_object(Bucket=BUCKET_NAME, Key=OBJECT_NAME)
-        historic_usage = json.loads(response["Body"].read().decode("utf-8"))
-    except ClientError as e:
-        logger.error("Error getting %s: %s. Using empty list.", OBJECT_NAME, e)
+    # try:
+    #     response = s3.get_object(Bucket=BUCKET_NAME, Key=OBJECT_NAME)
+    #     historic_usage = json.loads(response["Body"].read().decode("utf-8"))
+    # except ClientError as e:
+    #     logger.error("Error getting %s: %s. Using empty list.", OBJECT_NAME, e)
 
-        historic_usage = []
+    #     historic_usage = []
+
+    historic_usage = {"day_totals": []}
 
     dates_added = []
 
-    # Append the new usage data to the historic_usage_data.json
-    for date in usage_data:
-        if not any(d["date"] == date["date"] for d in historic_usage):
-            historic_usage.append(date)
+    # If historic data exists, append new usage data to historic_usage_data.json
+    if historic_usage:
+        for day in usage_data["day_totals"]:
+            if not any(d["day"] == day["day"] for d in historic_usage["day_totals"]):
+                historic_usage["day_totals"].append(day)
+
+                dates_added.append(day["day"])
 
-            dates_added.append(date["date"])
+    sorted_historic_usage = sorted(historic_usage["day_totals"], key=lambda x: x["day"])
 
     logger.info(
         "New usage data added to %s",
@@ -159,15 +165,15 @@ def get_and_update_historic_usage(
 
     if not write_data_locally:
         # Write the updated historic_usage to historic_usage_data.json
-        update_s3_object(s3, BUCKET_NAME, OBJECT_NAME, historic_usage)
+        update_s3_object(s3, BUCKET_NAME, OBJECT_NAME, sorted_historic_usage)
     else:
         local_path = f"output/{OBJECT_NAME}"
         os.makedirs("output", exist_ok=True)
         with open(local_path, "w", encoding="utf-8") as f:
-            json.dump(historic_usage, f, indent=4)
+            json.dump(sorted_historic_usage, f, indent=4)
         logger.info("Historic usage data written locally to %s (S3 skipped)", local_path)
 
-    return historic_usage, dates_added
+    return sorted_historic_usage, dates_added
 
 
 def get_and_update_copilot_teams(

From 51953af39acfdfdc657c3e2d838a961d860cc44e Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Wed, 18 Mar 2026 21:18:01 +0000
Subject: [PATCH 03/30] fix: resolve data structure issues and improve logging

---
 src/main.py | 44 ++++++++++++++++++++++----------------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/src/main.py b/src/main.py
index 9a6eea4..0ce6fdb 100644
--- a/src/main.py
+++ b/src/main.py
@@ -5,6 +5,7 @@
 for an organization. Data is retrieved from the GitHub API and stored in S3.
 """
 
+from itertools import count
 import json
 import logging
 import os
@@ -30,7 +31,7 @@
 
 # AWS Bucket Path
 BUCKET_NAME = f"{account}-copilot-usage-dashboard"
-OBJECT_NAME = "historic_usage_data.json"
+OBJECT_NAME = "org_history.json"
 
 logger = logging.getLogger()
 
@@ -131,31 +132,30 @@ def get_and_update_historic_usage(
     """
     # Get the usage data
     api_response = gh.get(f"/orgs/{org}/copilot/metrics/reports/organization-28-day/latest").json()
-    usage_data = json.loads(urllib.request.urlopen(api_response["download_links"][0]).read())
-
+    usage_data = json.loads(urllib.request.urlopen(api_response["download_links"][0]).read())["day_totals"]
+    
     logger.info("Usage data retrieved")
 
-    # try:
-    #     response = s3.get_object(Bucket=BUCKET_NAME, Key=OBJECT_NAME)
-    #     historic_usage = json.loads(response["Body"].read().decode("utf-8"))
-    # except ClientError as e:
-    #     logger.error("Error getting %s: %s. Using empty list.", OBJECT_NAME, e)
-
-    #     historic_usage = []
-
-    historic_usage = {"day_totals": []}
+    try:
+        response = s3.get_object(Bucket=BUCKET_NAME, Key=OBJECT_NAME)
+        historic_usage = json.loads(response["Body"].read().decode("utf-8"))
+    except ClientError as e:
+        logger.error("Error getting %s: %s. Using empty list.", OBJECT_NAME, e)
+        historic_usage = []
 
     dates_added = []
-
-    # If historic data exists, append new usage data to historic_usage_data.json
-    if historic_usage:
-        for day in usage_data["day_totals"]:
-            if not any(d["day"] == day["day"] for d in historic_usage["day_totals"]):
-                historic_usage["day_totals"].append(day)
-
-                dates_added.append(day["day"])
-
-    sorted_historic_usage = sorted(historic_usage["day_totals"], key=lambda x: x["day"])
+    count = 0
+
+    for day in usage_data:
+        if not any(d["day"] == day["day"] for d in historic_usage):
+            historic_usage.append(day)
+            dates_added.append(day["day"])
+            logger.info("Added data for day %s", day["day"])
+            count += 1
+    
+    logger.info("Total new days added: %d", count)
+
+    sorted_historic_usage = sorted(historic_usage, key=lambda x: x["day"])
 
     logger.info(
         "New usage data added to %s",

From 5ecdad517fe7904bf507a0fa8da8ae4051405d7a Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Thu, 19 Mar 2026 15:04:04 +0000
Subject: [PATCH 04/30] refactor: update TestGetAndUpdateHistoricUsage

---
 tests/test_main.py | 116 +++++++++++++++++++++++++++++++--------------
 1 file changed, 80 insertions(+), 36 deletions(-)

diff --git a/tests/test_main.py b/tests/test_main.py
index 628f57f..487cf98 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -1,6 +1,7 @@
 import json
 import os
 from unittest.mock import MagicMock, call, patch
+from io import BytesIO
 
 from botocore.exceptions import ClientError
 from requests import Response
@@ -362,39 +363,61 @@ def teardown_method(self):
     def test_get_and_update_historic_usage_success(self):
         s3 = MagicMock()
         gh = MagicMock()
-        # Mock usage data returned from GitHub API
-        usage_data = [
-            {"date": "2024-01-01", "usage": 10},
-            {"date": "2024-01-02", "usage": 20},
-        ]
-        gh.get.return_value.json.return_value = usage_data
+
+        # Mock API response
+        api_response = {
+            "download_links": [
+                "https://example.com/org_history_api_response.json"
+            ]
+            # There are other fields in the API response, but we don't need them for this test
+        }
+
+        # Mock usage data returned from GitHub API 
+        fetched_usage_data = {"day_totals": [
+            {"day": "2024-01-01", "usage": 10},
+            {"day": "2024-01-02", "usage": 20},
+        ]}
+        
+        gh.get.return_value.json.return_value = api_response
 
         # Mock S3 get_object returns existing historic usage with one date
-        existing_usage = [{"date": "2024-01-01", "usage": 10}]
+        existing_usage = [{"day": "2024-01-01", "usage": 10}]
         s3.get_object.return_value = {
-            "Body": MagicMock(
-                read=MagicMock(return_value=json.dumps(existing_usage).encode("utf-8"))
-            )
+            "Body": BytesIO(json.dumps(existing_usage).encode("utf-8"))
         }
 
-        result, dates_added = get_and_update_historic_usage(s3, gh, False)
+        # Mock urllib.request.urlopen returns usage data from download_links
+        # We always patch dependencies imported inside the function we're testing.
+        # Test environment initialisation ends here.
+        with patch("src.main.urllib.request.urlopen") as mock_urlopen:
+            mock_urlopen.return_value = BytesIO(json.dumps(fetched_usage_data).encode("utf-8"))
+            result, dates_added = get_and_update_historic_usage(s3, gh, False)
+
         assert result == [
-            {"date": "2024-01-01", "usage": 10},
-            {"date": "2024-01-02", "usage": 20},
+            {"day": "2024-01-01", "usage": 10},
+            {"day": "2024-01-02", "usage": 20},
         ]
         assert dates_added == ["2024-01-02"]
         s3.get_object.assert_called_once()
         s3.put_object.assert_called_once()
         args, kwargs = s3.put_object.call_args
         assert kwargs["Bucket"].endswith("copilot-usage-dashboard")
-        assert kwargs["Key"] == "historic_usage_data.json"
+        assert kwargs["Key"] == "org_history.json"
         assert json.loads(kwargs["Body"].decode("utf-8")) == result
 
     def test_get_and_update_historic_usage_no_existing_data(self, caplog):
         s3 = MagicMock()
         gh = MagicMock()
-        usage_data = [{"date": "2024-01-01", "usage": 10}]
-        gh.get.return_value.json.return_value = usage_data
+        api_response = {
+            "download_links": [
+                "https://example.com/org_history_api_response.json"
+            ]
+        }
+        fetched_usage_data = {"day_totals": [
+            {"day": "2024-01-01", "usage": 10},
+        ]}
+
+        gh.get.return_value.json.return_value = api_response
 
         # S3 get_object raises ClientError
         s3.get_object.side_effect = ClientError(
@@ -402,39 +425,58 @@ def test_get_and_update_historic_usage_no_existing_data(self, caplog):
             operation_name="GetObject",
         )
 
-        result, dates_added = get_and_update_historic_usage(s3, gh, False)
-        assert result == [{"date": "2024-01-01", "usage": 10}]
+        with patch("src.main.urllib.request.urlopen") as mock_urlopen:
+            mock_urlopen.return_value = BytesIO(json.dumps(fetched_usage_data).encode("utf-8"))
+            result, dates_added = get_and_update_historic_usage(s3, gh, False)
+
+        assert result == [{"day": "2024-01-01", "usage": 10}]
         assert dates_added == ["2024-01-01"]
         s3.put_object.assert_called_once()
         assert any(
-            "Error getting historic_usage_data.json" in record.getMessage()
+            "Error getting org_history.json" in record.getMessage()
             for record in caplog.records
         )
 
     def test_get_and_update_historic_usage_no_new_dates(self):
         s3 = MagicMock()
         gh = MagicMock()
-        usage_data = [{"date": "2024-01-01", "usage": 10}]
-        gh.get.return_value.json.return_value = usage_data
+        api_response = {
+            "download_links": [
+                "https://example.com/org_history_api_response.json"
+            ]
+        }
+        fetched_usage_data = {"day_totals": [
+            {"day": "2024-01-01", "usage": 10},
+        ]}
+        
+        gh.get.return_value.json.return_value = api_response
 
         # S3 get_object returns same date as usage_data
-        existing_usage = [{"date": "2024-01-01", "usage": 10}]
+        existing_usage = [{"day": "2024-01-01", "usage": 10}]
         s3.get_object.return_value = {
-            "Body": MagicMock(
-                read=MagicMock(return_value=json.dumps(existing_usage).encode("utf-8"))
-            )
+            "Body": BytesIO(json.dumps(existing_usage).encode("utf-8"))
         }
+        with patch("src.main.urllib.request.urlopen") as mock_urlopen:
+            mock_urlopen.return_value = BytesIO(json.dumps(fetched_usage_data).encode("utf-8"))
+            result, dates_added = get_and_update_historic_usage(s3, gh, False)
 
-        result, dates_added = get_and_update_historic_usage(s3, gh, False)
-        assert result == [{"date": "2024-01-01", "usage": 10}]
+        assert result == [{"day": "2024-01-01", "usage": 10}]
         assert dates_added == []
         s3.put_object.assert_called_once()
 
     def test_write_data_locally_creates_file(self, tmp_path):
         s3 = MagicMock()
         gh = MagicMock()
-        usage_data = [{"date": "2024-01-01", "usage": 10}]
-        gh.get.return_value.json.return_value = usage_data
+        api_response = {
+            "download_links": [
+                "https://example.com/org_history_api_response.json"
+            ]
+        }
+        fetched_usage_data = {"day_totals": [
+            {"day": "2024-01-01", "usage": 10},
+        ]}
+        
+        gh.get.return_value.json.return_value = api_response
 
         # S3 get_object raises ClientError
         s3.get_object.side_effect = ClientError(
@@ -444,13 +486,15 @@ def test_write_data_locally_creates_file(self, tmp_path):
 
         # Patch os.makedirs and open to use tmp_path
         with patch("src.main.os.makedirs") as mock_makedirs, \
-                patch("src.main.open", create=True) as mock_open:
-            result, dates_added = get_and_update_historic_usage(s3, gh, True)
-            assert result == [{"date": "2024-01-01", "usage": 10}]
-            assert dates_added == ["2024-01-01"]
-            mock_makedirs.assert_called_once_with("output", exist_ok=True)
-            mock_open.assert_called_once()
-            s3.put_object.assert_not_called()
+                patch("src.main.open", create=True) as mock_open, \
+                    patch("src.main.urllib.request.urlopen") as mock_urlopen:
+                        mock_urlopen.return_value = BytesIO(json.dumps(fetched_usage_data).encode("utf-8"))
+                        result, dates_added = get_and_update_historic_usage(s3, gh, True)
+                        assert result == [{"day": "2024-01-01", "usage": 10}]
+                        assert dates_added == ["2024-01-01"]
+                        mock_makedirs.assert_called_once_with("output", exist_ok=True)
+                        mock_open.assert_called_once()
+                        s3.put_object.assert_not_called()
 
 
 class TestCreateDictionary:

From c73c4d352016dcd58b2fe543311e3b3c1497923c Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Thu, 19 Mar 2026 15:19:52 +0000
Subject: [PATCH 05/30] refactor: remove functions responsible for handling
 team data

---
 src/main.py | 221 +---------------------------------------------------
 1 file changed, 4 insertions(+), 217 deletions(-)

diff --git a/src/main.py b/src/main.py
index 0ce6fdb..f28efe8 100644
--- a/src/main.py
+++ b/src/main.py
@@ -59,64 +59,6 @@
 # }
 
 
-def get_copilot_team_date(gh: github_api_toolkit.github_interface, page: int) -> list:
-    """Gets a list of GitHub Teams with Copilot Data for a given API page.
-
-    Args:
-        gh (github_api_toolkit.github_interface): An instance of the github_interface class.
-        page (int): The page number of the API request.
-
-    Returns:
-        list: A list of GitHub Teams with Copilot Data.
-    """
-    copilot_teams = []
-
-    response = gh.get(f"/orgs/{org}/teams", params={"per_page": 100, "page": page})
-    teams = response.json()
-    for team in teams:
-        usage_data = gh.get(f"/orgs/{org}/team/{team['name']}/copilot/metrics")
-
-        if not isinstance(usage_data, Response):
-
-            # If the response is not a Response object, no copilot data is available for this team
-            # We can then skip this team
-
-            # We don't log this as an error, as it is expected and it'd be too noisy within logs
-
-            continue
-
-        # If the response has data, append the team to the list
-        # If there is no data, .json() will return an empty list
-        if usage_data.json():
-
-            team_name = team.get("name", "")
-            team_slug = team.get("slug", "")
-            team_description = team.get("description", "")
-            team_html_url = team.get("html_url", "")
-
-            logger.info(
-                "Team %s has Copilot data",
-                team_name,
-                extra={
-                    "team_name": team_name,
-                    "team_slug": team_slug,
-                    "team_description": team_description,
-                    "team_html_url": team_html_url,
-                },
-            )
-
-            copilot_teams.append(
-                {
-                    "name": team_name,
-                    "slug": team_slug,
-                    "description": team_description,
-                    "url": team_html_url,
-                }
-            )
-
-    return copilot_teams
-
-
 def get_and_update_historic_usage(
     s3: boto3.client, gh: github_api_toolkit.github_interface, write_data_locally: bool
 ) -> tuple:
@@ -176,104 +118,6 @@ def get_and_update_historic_usage(
     return sorted_historic_usage, dates_added
 
 
-def get_and_update_copilot_teams(
-    s3: boto3.client, gh: github_api_toolkit.github_interface, write_data_locally: bool
-) -> list:
-    """Get and update GitHub Teams with Copilot Data.
-
-    Args:
-        s3 (boto3.client): An S3 client.
-        gh (github_api_toolkit.github_interface): An instance of the github_interface class.
-        write_data_locally (bool): Whether to write data locally instead of to an S3 bucket.
-
-    Returns:
-        list: A list of GitHub Teams with Copilot Data.
-    """
-    logger.info("Getting GitHub Teams with Copilot Data")
-
-    copilot_teams = []
-
-    response = gh.get(f"/orgs/{org}/teams", params={"per_page": 100})
-
-    # Get the last page of teams
-    try:
-        last_page = int(response.links["last"]["url"].split("=")[-1])
-    except KeyError:
-        last_page = 1
-
-    for page in range(1, last_page + 1):
-        page_teams = get_copilot_team_date(gh, page)
-
-        copilot_teams = copilot_teams + page_teams
-
-    logger.info(
-        "Fetched GitHub Teams with Copilot Data",
-        extra={"no_teams": len(copilot_teams)},
-    )
-
-    if not write_data_locally:
-        update_s3_object(s3, BUCKET_NAME, "copilot_teams.json", copilot_teams)
-    else:
-        local_path = "output/copilot_teams.json"
-        os.makedirs("output", exist_ok=True)
-        with open(local_path, "w", encoding="utf-8") as f:
-            json.dump(copilot_teams, f, indent=4)
-        logger.info("Copilot teams data written locally to %s (S3 skipped)", local_path)
-
-    return copilot_teams
-
-
-def create_dictionary(
-    gh: github_api_toolkit.github_interface, copilot_teams: list, existing_team_history: list
-) -> list:
-    """Create a dictionary for quick lookup of existing team data using the `name` field.
-
-    Args:
-        gh (github_api_toolkit.github_interface): An instance of the github_interface class.
-        copilot_teams (list): List of teams with Copilot data.
-        existing_team_history (list): List of existing team history data.
-
-    Returns:
-        list: A list of dictionaries containing team data and their history.
-    """
-    existing_team_data_map = {
-        single_team["team"]["name"]: single_team for single_team in existing_team_history
-    }
-
-    # Iterate through identified teams
-    for team in copilot_teams:
-        team_name = team.get("name", "")
-        if not team_name:
-            logger.warning("Skipping team with no name")
-            continue
-
-        # Determine the last known date for the team
-        last_known_date = None
-        if team_name in existing_team_data_map:
-            existing_dates = [entry["date"] for entry in existing_team_data_map[team_name]["data"]]
-            if existing_dates:
-                last_known_date = max(existing_dates)  # Get the most recent date
-
-        # Assign the last known date to the `since` query parameter
-        query_params = {}
-        if last_known_date:
-            query_params["since"] = last_known_date
-
-        single_team_history = get_team_history(gh, team_name, query_params)
-        if not single_team_history:
-            logger.info("No new history found for team %s", team_name)
-            continue
-
-        # Append new data to the existing team history
-        new_team_data = single_team_history
-        if team_name in existing_team_data_map:
-            existing_team_data_map[team_name]["data"].extend(new_team_data)
-        else:
-            existing_team_data_map[team_name] = {"team": team, "data": new_team_data}
-
-    return list(existing_team_data_map.values())
-
-
 def update_s3_object(
     s3_client: boto3.client,
     bucket_name: str,
@@ -304,31 +148,6 @@ def update_s3_object(
         return False
 
 
-def get_team_history(
-    gh: github_api_toolkit.github_interface, team: str, query_params: Optional[dict] = None
-) -> list[dict]:
-    """Gets the team metrics Copilot data through the API.
-    Note - This endpoint will only return results for a given day if the team had
-    five or more members with active Copilot licenses on that day,
-    as evaluated at the end of that day.
-
-    Args:
-        gh (github_api_toolkit.github_interface): An instance of the github_interface class.
-        team (str): Team name.
-        query_params (dict): Additional query parameters for the API request.
-
-    Returns:
-        list[dict]: A team's GitHub Copilot metrics or None if an error occurs.
-    """
-    response = gh.get(f"/orgs/{org}/team/{team}/copilot/metrics", params=query_params)
-
-    if not isinstance(response, Response):
-        # If the response is not a Response object, no copilot data is available for this team
-        # We can return None which is then handled by the calling function
-        return None
-    return response.json()
-
-
 def get_dict_value(dictionary: dict, key: str) -> Any:
     """Gets a value from a dictionary and raises an exception if it is not found.
 
@@ -451,37 +270,6 @@ def handler(event: dict, context) -> str:  # pylint: disable=unused-argument, to
     # Copilot Usage Data (Historic)
     historic_usage, dates_added = get_and_update_historic_usage(s3, gh, write_data_locally)
 
-    # GitHub Teams with Copilot Data
-    copilot_teams = get_and_update_copilot_teams(s3, gh, write_data_locally)
-
-    logger.info("Getting history of each team identified previously")
-
-    # Retrieve existing team history from S3
-    try:
-        response = s3.get_object(Bucket=BUCKET_NAME, Key="teams_history.json")
-        existing_team_history = json.loads(response["Body"].read().decode("utf-8"))
-    except ClientError as e:
-        logger.warning("Error retrieving existing team history: %s", e)
-        existing_team_history = []
-
-    logger.info("Existing team history has %d entries", len(existing_team_history))
-
-    if not write_data_locally:
-        # Convert to dictionary for quick lookup
-        updated_team_history = create_dictionary(gh, copilot_teams, existing_team_history)
-
-        # Write updated team history to S3
-        # This line isn't covered by tests as it's painful to get to.
-        # The function itself is tested though.
-        update_s3_object(s3, BUCKET_NAME, "teams_history.json", updated_team_history)
-    else:
-        local_path = "output/teams_history.json"
-        os.makedirs("output", exist_ok=True)
-        updated_team_history = create_dictionary(gh, copilot_teams, existing_team_history)
-        with open(local_path, "w", encoding="utf-8") as f:
-            json.dump(updated_team_history, f, indent=4)
-        logger.info("Team history written locally to %s (S3 skipped)", local_path)
-
     logger.info(
         "Process complete",
         extra={
@@ -490,14 +278,13 @@ def handler(event: dict, context) -> str:  # pylint: disable=unused-argument, to
             "dates_added": dates_added,
             "no_dates_before": len(historic_usage) - len(dates_added),
             "no_dates_after": len(historic_usage),
-            "no_copilot_teams": len(copilot_teams),
         },
     )
 
     return "Github Data logging is now complete."
 
 
-# # Dev Only
-# # Uncomment the following line to run the script locally
-if __name__ == "__main__":
-    handler(None, None)
+# Dev Only
+# Uncomment the following line to run the script locally
+# if __name__ == "__main__":
+#     handler(None, None)

From 4e02c3827c11fb0edddcbe0cd7d7134ff12e1ac3 Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Thu, 19 Mar 2026 15:27:23 +0000
Subject: [PATCH 06/30] chore: poetry lock

---
 poetry.lock | 25 +++----------------------
 1 file changed, 3 insertions(+), 22 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index 5a33af3..560f3c3 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1082,7 +1082,7 @@ version = "1.0.4"
 description = "Utility library for gitignore style pattern matching of file paths."
 optional = false
 python-versions = ">=3.9"
-groups = ["main", "dev", "docs"]
+groups = ["dev", "docs"]
 files = [
     {file = "pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723"},
     {file = "pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645"},
@@ -1282,7 +1282,7 @@ version = "6.0.2"
 description = "YAML parser and emitter for Python"
 optional = false
 python-versions = ">=3.8"
-groups = ["main", "docs"]
+groups = ["docs"]
 files = [
     {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"},
     {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"},
@@ -1519,26 +1519,7 @@ files = [
 [package.extras]
 watchmedo = ["PyYAML (>=3.10)"]
 
-[[package]]
-name = "yamllint"
-version = "1.38.0"
-description = "A linter for YAML files."
-optional = false
-python-versions = ">=3.10"
-groups = ["main"]
-files = [
-    {file = "yamllint-1.38.0-py3-none-any.whl", hash = "sha256:fc394a5b3be980a4062607b8fdddc0843f4fa394152b6da21722f5d59013c220"},
-    {file = "yamllint-1.38.0.tar.gz", hash = "sha256:09e5f29531daab93366bb061e76019d5e91691ef0a40328f04c927387d1d364d"},
-]
-
-[package.dependencies]
-pathspec = ">=1.0.0"
-pyyaml = "*"
-
-[package.extras]
-dev = ["doc8", "flake8", "flake8-import-order", "rstcheck[sphinx]", "ruff", "sphinx"]
-
 [metadata]
 lock-version = "2.1"
 python-versions = "^3.12"
-content-hash = "8cb6c4a88a27848571078e5ab89f980e1daf60ac2a35842524d155dd3d21c819"
+content-hash = "eedc53a92c67abf43fe87b7f174ed56e885f73181ca849c4c3b9bb1edec773df"

From cca1ac3348d0ba13d80833e2a3c6552ee02302e1 Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Thu, 19 Mar 2026 16:05:40 +0000
Subject: [PATCH 07/30] refactor: improve logging and use requests

---
 src/main.py | 32 +++++++++++++-------------------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/src/main.py b/src/main.py
index f28efe8..0950cf0 100644
--- a/src/main.py
+++ b/src/main.py
@@ -5,17 +5,15 @@
 for an organization. Data is retrieved from the GitHub API and stored in S3.
 """
 
-from itertools import count
 import json
 import logging
 import os
-from typing import Any, Optional
+from typing import Any
 
 import boto3
 import github_api_toolkit
+import requests
 from botocore.exceptions import ClientError
-from requests import Response
-import urllib.request
 
 # GitHub Organisation
 org = os.getenv("GITHUB_ORG")
@@ -74,8 +72,8 @@ def get_and_update_historic_usage(
     """
     # Get the usage data
     api_response = gh.get(f"/orgs/{org}/copilot/metrics/reports/organization-28-day/latest").json()
-    usage_data = json.loads(urllib.request.urlopen(api_response["download_links"][0]).read())["day_totals"]
-    
+    usage_data = requests.get(api_response["download_links"][0], timeout=30).json()["day_totals"]
+
     logger.info("Usage data retrieved")
 
     try:
@@ -86,25 +84,15 @@ def get_and_update_historic_usage(
         historic_usage = []
 
     dates_added = []
-    count = 0
 
     for day in usage_data:
         if not any(d["day"] == day["day"] for d in historic_usage):
             historic_usage.append(day)
             dates_added.append(day["day"])
             logger.info("Added data for day %s", day["day"])
-            count += 1
-    
-    logger.info("Total new days added: %d", count)
 
     sorted_historic_usage = sorted(historic_usage, key=lambda x: x["day"])
 
-    logger.info(
-        "New usage data added to %s",
-        OBJECT_NAME,
-        extra={"no_days_added": len(dates_added), "dates_added": dates_added},
-    )
-
     if not write_data_locally:
         # Write the updated historic_usage to historic_usage_data.json
         update_s3_object(s3, BUCKET_NAME, OBJECT_NAME, sorted_historic_usage)
@@ -115,6 +103,13 @@ def get_and_update_historic_usage(
             json.dump(sorted_historic_usage, f, indent=4)
         logger.info("Historic usage data written locally to %s (S3 skipped)", local_path)
 
+    logger.info(
+        "Usage data written to %s: %d days added (%s)",
+        OBJECT_NAME,
+        len(dates_added),
+        dates_added,
+    )
+
     return sorted_historic_usage, dates_added
 
 
@@ -232,7 +227,6 @@ def handler(event: dict, context) -> str:  # pylint: disable=unused-argument, to
             filename="debug.log",
             filemode="w",
             format="%(asctime)s %(levelname)s %(message)s",
-
         )
     else:
         # Ensure INFO logs show in the terminal when not logging to a file
@@ -286,5 +280,5 @@ def handler(event: dict, context) -> str:  # pylint: disable=unused-argument, to
 
 # Dev Only
 # Uncomment the following line to run the script locally
-# if __name__ == "__main__":
-#     handler(None, None)
+if __name__ == "__main__":
+    handler(None, None)

From 7d209ce6e62001f18e930b42e7245c94a99b8576 Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Thu, 19 Mar 2026 16:06:24 +0000
Subject: [PATCH 08/30] refactor: strip down tests

---
 tests/test_main.py | 374 ++-------------------------------------------
 1 file changed, 9 insertions(+), 365 deletions(-)

diff --git a/tests/test_main.py b/tests/test_main.py
index 487cf98..e1cee62 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -12,11 +12,7 @@
 
 from src.main import (
     BUCKET_NAME,
-    create_dictionary,
-    get_and_update_copilot_teams,
     get_and_update_historic_usage,
-    get_copilot_team_date,
-    get_team_history,
     handler,
     update_s3_object,
     get_dict_value,
@@ -59,115 +55,6 @@ def test_update_s3_object_failure(self, caplog):
         assert any("Failed to update" in record.message for record in caplog.records)
 
 
-class TestGetAndUpdateCopilotTeams:
-    @patch("src.main.update_s3_object")
-    def test_get_and_update_copilot_teams_single_page(self, mock_update_s3_object):
-        s3 = MagicMock()
-        gh = MagicMock()
-        # Mock response for first page
-        mock_response = MagicMock()
-        mock_response.links = {}  # No 'last' link, so only one page
-        gh.get.return_value = mock_response
-
-        # Patch get_copilot_team_date to return a list of teams
-        with patch(
-            "src.main.get_copilot_team_date", return_value=[{"name": "team1"}]
-        ) as mock_get_team_date:
-            result = get_and_update_copilot_teams(s3, gh, False)
-            assert result == [{"name": "team1"}]
-            mock_get_team_date.assert_called_once_with(gh, 1)
-            mock_update_s3_object.assert_called_once()
-            args, kwargs = mock_update_s3_object.call_args
-            assert args[1].endswith("copilot-usage-dashboard")
-            assert args[2] == "copilot_teams.json"
-            assert args[3] == [{"name": "team1"}]
-
-    @patch("src.main.update_s3_object")
-    def test_get_and_update_copilot_teams_multiple_pages(self, mock_update_s3_object):
-        s3 = MagicMock()
-        gh = MagicMock()
-        # Mock response with 'last' link for 3 pages
-        mock_response = MagicMock()
-        mock_response.links = {"last": {"url": "https://api.github.com/orgs/test/teams?page=3"}}
-        gh.get.return_value = mock_response
-
-        # Patch get_copilot_team_date to return different teams per page
-        with patch(
-            "src.main.get_copilot_team_date",
-            side_effect=[[{"name": "team1"}], [{"name": "team2"}], [{"name": "team3"}]],
-        ) as mock_get_team_date:
-            result = get_and_update_copilot_teams(s3, gh, False)
-            assert result == [{"name": "team1"}, {"name": "team2"}, {"name": "team3"}]
-            assert mock_get_team_date.call_count == 3
-            mock_update_s3_object.assert_called_once()
-
-    @patch("src.main.update_s3_object")
-    def test_get_and_update_copilot_teams_no_teams(self, mock_update_s3_object):
-        s3 = MagicMock()
-        gh = MagicMock()
-        mock_response = MagicMock()
-        mock_response.links = {}
-        gh.get.return_value = mock_response
-
-        with patch("src.main.get_copilot_team_date", return_value=[]) as mock_get_team_date:
-            result = get_and_update_copilot_teams(s3, gh, False)
-            assert result == []
-            mock_get_team_date.assert_called_once_with(gh, 1)
-            mock_update_s3_object.assert_called_once()
-            args, kwargs = mock_update_s3_object.call_args
-            assert args[1].endswith("copilot-usage-dashboard")
-            assert args[2] == "copilot_teams.json"
-            assert args[3] == []
-
-    def test_write_data_locally_creates_file(self, tmp_path):
-        s3 = MagicMock()
-        gh = MagicMock()
-        response = MagicMock()
-        response.links = {}
-        gh.get.return_value = response
-
-        with patch("src.main.get_copilot_team_date", return_value=[{"name": "teamA"}]):
-            with patch("src.main.os.makedirs") as mock_makedirs, \
-                    patch("src.main.open", create=True) as mock_open:
-                result = get_and_update_copilot_teams(s3, gh, True)
-                assert result == [{"name": "teamA"}]
-                mock_makedirs.assert_called_once_with("output", exist_ok=True)
-                mock_open.assert_called_once()
-                s3.put_object.assert_not_called()
-
-
-class TestGetTeamHistory:
-    def setup_method(self):
-        self.org_patch = patch("src.main.org", "test-org")
-        self.org_patch.start()
-        self.addCleanup = getattr(self, "addCleanup", lambda f: None)
-
-    def teardown_method(self):
-        self.org_patch.stop()
-
-    def test_get_team_history_success(self):
-        gh = MagicMock()
-        mock_response = MagicMock(spec=Response)
-        mock_response.json.return_value = [{"date": "2024-01-01", "usage": 5}]
-        gh.get.return_value = mock_response
-
-        result = get_team_history(gh, "dev-team", {"since": "2024-01-01"})
-        gh.get.assert_called_once_with(
-            "/orgs/test-org/team/dev-team/copilot/metrics", params={"since": "2024-01-01"}
-        )
-        assert result == [{"date": "2024-01-01", "usage": 5}]
-
-    def test_get_team_history_with_no_query_params(self):
-        gh = MagicMock()
-        mock_response = MagicMock(spec=Response)
-        mock_response.json.return_value = []
-        gh.get.return_value = mock_response
-
-        result = get_team_history(gh, "dev-team")
-        gh.get.assert_called_once_with("/orgs/test-org/team/dev-team/copilot/metrics", params=None)
-        assert result == []
-
-
 class TestHandler:
     @patch("src.main.boto3.Session")
     @patch("src.main.github_api_toolkit.get_token_as_installation")
@@ -179,8 +66,6 @@ class TestHandler:
     def test_handler_success(
         self,
         mock_update_s3_object,
-        mock_create_dictionary,
-        mock_get_and_update_copilot_teams,
         mock_get_and_update_historic_usage,
         mock_github_interface,
         mock_get_token_as_installation,
@@ -200,10 +85,6 @@ def test_handler_success(
         mock_github_interface.return_value = mock_gh
 
         mock_get_and_update_historic_usage.return_value = (["usage1", "usage2"], ["2024-01-01"])
-        mock_get_and_update_copilot_teams.return_value = [{"name": "team1"}]
-        mock_create_dictionary.return_value = [
-            {"team": {"name": "team1"}, "data": [{"date": "2024-01-01"}]}
-        ]
 
         secret_region = "eu-west-1"
         secret_name = "test-secret"
@@ -224,11 +105,6 @@ def test_handler_success(
         mock_get_token_as_installation.assert_called_once()
         mock_github_interface.assert_called_once()
         mock_get_and_update_historic_usage.assert_called_once()
-        mock_get_and_update_copilot_teams.assert_called_once()
-        mock_create_dictionary.assert_called_once()
-        mock_update_s3_object.assert_called_with(
-            mock_s3, BUCKET_NAME, "teams_history.json", mock_create_dictionary.return_value
-        )
 
     @patch("src.main.boto3.Session")
     @patch("src.main.github_api_toolkit.get_token_as_installation")
@@ -247,110 +123,6 @@ def test_handler_access_token_error(
         assert result.startswith("Error getting access token:")
         assert any("Error getting access token" in record.getMessage() for record in caplog.records)
 
-    @patch("src.main.boto3.Session")
-    @patch("src.main.github_api_toolkit.get_token_as_installation")
-    @patch("src.main.github_api_toolkit.github_interface")
-    @patch("src.main.get_and_update_historic_usage")
-    @patch("src.main.get_and_update_copilot_teams")
-    @patch("src.main.create_dictionary")
-    @patch("src.main.update_s3_object")
-    def test_handler_team_history_client_error(
-        self,
-        mock_update_s3_object,
-        mock_create_dictionary,
-        mock_get_and_update_copilot_teams,
-        mock_get_and_update_historic_usage,
-        mock_github_interface,
-        mock_get_token_as_installation,
-        mock_boto3_session,
-        caplog,
-    ):
-        mock_s3 = MagicMock()
-        mock_secret_manager = MagicMock()
-        mock_session = MagicMock()
-        mock_session.client.side_effect = [mock_s3, mock_secret_manager]
-        mock_boto3_session.return_value = mock_session
-
-        mock_secret_manager.get_secret_value.return_value = {"SecretString": "pem-content"}
-        mock_get_token_as_installation.return_value = ("token",)
-        mock_gh = MagicMock()
-        mock_github_interface.return_value = mock_gh
-
-        mock_get_and_update_historic_usage.return_value = (["usage1"], ["2024-01-01"])
-        mock_get_and_update_copilot_teams.return_value = [{"name": "team1"}]
-        mock_create_dictionary.return_value = [
-            {"team": {"name": "team1"}, "data": [{"date": "2024-01-01"}]}
-        ]
-
-        # S3 get_object for teams_history.json raises ClientError
-        mock_s3.get_object.side_effect = ClientError(
-            error_response={"Error": {"Code": "404", "Message": "Not Found"}},
-            operation_name="GetObject",
-        )
-
-        result = handler({}, MagicMock())
-        assert result == "Github Data logging is now complete."
-        assert any(
-            "Error retrieving existing team history" in record.getMessage()
-            for record in caplog.records
-        )
-        mock_update_s3_object.assert_called_with(
-            mock_s3, BUCKET_NAME, "teams_history.json", mock_create_dictionary.return_value
-        )
-
-
-class TestGetCopilotTeamDate:
-    @patch("src.main.org", "test-org")
-    def test_get_copilot_team_date_success(self):
-        gh = MagicMock()
-        # Mock teams response
-        teams_response = MagicMock()
-        teams_response.json.return_value = [
-            {"name": "team1", "slug": "slug1", "description": "desc1", "html_url": "url1"},
-            {"name": "team2", "slug": "slug2", "description": "desc2", "html_url": "url2"},
-        ]
-        gh.get.side_effect = [
-            teams_response,
-            MagicMock(spec=Response),  # usage_data for team1
-            MagicMock(spec=Response),  # usage_data for team2
-        ]
-
-        result = get_copilot_team_date(gh, 1)
-        assert result == [
-            {"name": "team1", "slug": "slug1", "description": "desc1", "url": "url1"},
-            {"name": "team2", "slug": "slug2", "description": "desc2", "url": "url2"},
-        ]
-        gh.get.assert_any_call("/orgs/test-org/teams", params={"per_page": 100, "page": 1})
-        gh.get.assert_any_call("/orgs/test-org/team/team1/copilot/metrics")
-        gh.get.assert_any_call("/orgs/test-org/team/team2/copilot/metrics")
-
-    @patch("src.main.org", "test-org")
-    def test_get_copilot_team_date_unexpected_usage_response(self, caplog):
-        gh = MagicMock()
-        teams_response = MagicMock()
-        teams_response.json.return_value = [
-            {"name": "team1", "slug": "slug1", "description": "desc1", "html_url": "url1"},
-        ]
-        gh.get.side_effect = [
-            teams_response,
-            "not_a_response",  # usage_data for team1
-        ]
-
-        with caplog.at_level("ERROR"):
-            result = get_copilot_team_date(gh, 1)
-            assert result == []
-
-    @patch("src.main.org", "test-org")
-    def test_get_copilot_team_date_empty_teams(self):
-        gh = MagicMock()
-        teams_response = MagicMock()
-        teams_response.json.return_value = []
-        gh.get.return_value = teams_response
-
-        result = get_copilot_team_date(gh, 1)
-        assert result == []
-        gh.get.assert_called_once_with("/orgs/test-org/teams", params={"per_page": 100, "page": 1})
-
 
 class TestGetAndUpdateHistoricUsage:
     def setup_method(self):
@@ -386,11 +158,11 @@ def test_get_and_update_historic_usage_success(self):
             "Body": BytesIO(json.dumps(existing_usage).encode("utf-8"))
         }
 
-        # Mock urllib.request.urlopen returns usage data from download_links
+        # Mock requests.get returns usage data from download_links
         # We always patch dependencies imported inside the function we're testing.
         # Test environment initialisation ends here.
-        with patch("src.main.urllib.request.urlopen") as mock_urlopen:
-            mock_urlopen.return_value = BytesIO(json.dumps(fetched_usage_data).encode("utf-8"))
+        with patch("src.main.requests.get") as mock_requests_get:
+            mock_requests_get.return_value.json.return_value = fetched_usage_data
             result, dates_added = get_and_update_historic_usage(s3, gh, False)
 
         assert result == [
@@ -425,8 +197,8 @@ def test_get_and_update_historic_usage_no_existing_data(self, caplog):
             operation_name="GetObject",
         )
 
-        with patch("src.main.urllib.request.urlopen") as mock_urlopen:
-            mock_urlopen.return_value = BytesIO(json.dumps(fetched_usage_data).encode("utf-8"))
+        with patch("src.main.requests.get") as mock_requests_get:
+            mock_requests_get.return_value.json.return_value = fetched_usage_data
             result, dates_added = get_and_update_historic_usage(s3, gh, False)
 
         assert result == [{"day": "2024-01-01", "usage": 10}]
@@ -456,8 +228,8 @@ def test_get_and_update_historic_usage_no_new_dates(self):
         s3.get_object.return_value = {
             "Body": BytesIO(json.dumps(existing_usage).encode("utf-8"))
         }
-        with patch("src.main.urllib.request.urlopen") as mock_urlopen:
-            mock_urlopen.return_value = BytesIO(json.dumps(fetched_usage_data).encode("utf-8"))
+        with patch("src.main.requests.get") as mock_requests_get:
+            mock_requests_get.return_value.json.return_value = fetched_usage_data
             result, dates_added = get_and_update_historic_usage(s3, gh, False)
 
         assert result == [{"day": "2024-01-01", "usage": 10}]
@@ -487,8 +259,8 @@ def test_write_data_locally_creates_file(self, tmp_path):
         # Patch os.makedirs and open to use tmp_path
         with patch("src.main.os.makedirs") as mock_makedirs, \
                 patch("src.main.open", create=True) as mock_open, \
-                    patch("src.main.urllib.request.urlopen") as mock_urlopen:
-                        mock_urlopen.return_value = BytesIO(json.dumps(fetched_usage_data).encode("utf-8"))
+                    patch("src.main.requests.get") as mock_requests_get:
+                        mock_requests_get.return_value.json.return_value = fetched_usage_data
                         result, dates_added = get_and_update_historic_usage(s3, gh, True)
                         assert result == [{"day": "2024-01-01", "usage": 10}]
                         assert dates_added == ["2024-01-01"]
@@ -497,134 +269,6 @@ def test_write_data_locally_creates_file(self, tmp_path):
                         s3.put_object.assert_not_called()
 
 
-class TestCreateDictionary:
-    def setup_method(self):
-        self.org_patch = patch("src.main.org", "test-org")
-        self.org_patch.start()
-
-    def teardown_method(self):
-        self.org_patch.stop()
-
-    def test_create_dictionary_adds_new_team_history(self):
-        gh = MagicMock()
-        copilot_teams = [{"name": "team1"}, {"name": "team2"}]
-        existing_team_history = []
-
-        # get_team_history returns history for each team
-        with patch(
-            "src.main.get_team_history",
-            side_effect=[
-                [{"date": "2024-01-01", "usage": 5}],
-                [{"date": "2024-01-02", "usage": 10}],
-            ],
-        ) as mock_get_team_history:
-            result = create_dictionary(gh, copilot_teams, existing_team_history)
-            assert len(result) == 2
-            assert result[0]["team"]["name"] == "team1"
-            assert result[0]["data"] == [{"date": "2024-01-01", "usage": 5}]
-            assert result[1]["team"]["name"] == "team2"
-            assert result[1]["data"] == [{"date": "2024-01-02", "usage": 10}]
-            assert mock_get_team_history.call_count == 2
-
-    def test_create_dictionary_extends_existing_team_history(self):
-        gh = MagicMock()
-        copilot_teams = [{"name": "team1"}]
-        existing_team_history = [
-            {"team": {"name": "team1"}, "data": [{"date": "2024-01-01", "usage": 5}]}
-        ]
-
-        # get_team_history returns new history for team1
-        with patch(
-            "src.main.get_team_history", return_value=[{"date": "2024-01-02", "usage": 10}]
-        ) as mock_get_team_history:
-            result = create_dictionary(gh, copilot_teams, existing_team_history)
-            assert len(result) == 1
-            assert result[0]["team"]["name"] == "team1"
-            assert result[0]["data"] == [
-                {"date": "2024-01-01", "usage": 5},
-                {"date": "2024-01-02", "usage": 10},
-            ]
-            mock_get_team_history.assert_called_once()
-
-            args, kwargs = mock_get_team_history.call_args
-            assert args[0] == gh
-            assert args[1] == "team1"
-            assert args[2] == {"since": "2024-01-01"}
-
-    def test_create_dictionary_skips_team_with_no_name(self, caplog):
-        gh = MagicMock()
-        copilot_teams = [{"slug": "slug1"}]  # No 'name'
-        existing_team_history = []
-
-        with patch("src.main.get_team_history") as mock_get_team_history:
-            result = create_dictionary(gh, copilot_teams, existing_team_history)
-            assert result == []
-            assert mock_get_team_history.call_count == 0
-            assert any(
-                "Skipping team with no name" in record.getMessage() for record in caplog.records
-            )
-
-    def test_create_dictionary_no_new_history(self, caplog):
-        gh = MagicMock()
-        copilot_teams = [{"name": "team1"}]
-        existing_team_history = []
-
-        # get_team_history returns empty list
-        with patch("src.main.get_team_history", return_value=[]) as mock_get_team_history:
-            result = create_dictionary(gh, copilot_teams, existing_team_history)
-            assert result == []
-            assert mock_get_team_history.call_count == 1
-
-
-class TestGetTeamHistory:
-    def setup_method(self):
-        self.org_patch = patch("src.main.org", "test-org")
-        self.org_patch.start()
-
-    def teardown_method(self):
-        self.org_patch.stop()
-
-    def test_get_team_history_returns_metrics(self):
-        gh = MagicMock()
-        mock_response = MagicMock(spec=Response)
-        mock_response.json.return_value = [{"date": "2024-01-01", "usage": 5}]
-        gh.get.return_value = mock_response
-
-        result = get_team_history(gh, "dev-team", {"since": "2024-01-01"})
-        gh.get.assert_called_once_with(
-            "/orgs/test-org/team/dev-team/copilot/metrics", params={"since": "2024-01-01"}
-        )
-        assert result == [{"date": "2024-01-01", "usage": 5}]
-
-    def test_get_team_history_returns_empty_list(self):
-        gh = MagicMock()
-        mock_response = MagicMock(spec=Response)
-        mock_response.json.return_value = []
-        gh.get.return_value = mock_response
-
-        result = get_team_history(gh, "dev-team")
-        gh.get.assert_called_once_with("/orgs/test-org/team/dev-team/copilot/metrics", params=None)
-        assert result == []
-
-    def test_get_team_history_non_response_returns_none(self):
-        gh = MagicMock()
-        gh.get.return_value = "not_a_response"
-
-        result = get_team_history(gh, "dev-team")
-        gh.get.assert_called_once_with("/orgs/test-org/team/dev-team/copilot/metrics", params=None)
-        assert result is None
-
-    def test_get_team_history_with_query_params_none(self):
-        gh = MagicMock()
-        mock_response = MagicMock(spec=Response)
-        mock_response.json.return_value = [{"date": "2024-01-01", "usage": 5}]
-        gh.get.return_value = mock_response
-
-        result = get_team_history(gh, "dev-team", None)
-        gh.get.assert_called_once_with("/orgs/test-org/team/dev-team/copilot/metrics", params=None)
-        assert result == [{"date": "2024-01-01", "usage": 5}]
-
-
 class TestGetDictValue:
     def test_get_dict_value_returns_value(self):
         d = {"foo": "bar", "baz": 42}

From 660d1133a7143ec71f92be58a10fb8352aaa6fd8 Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Thu, 19 Mar 2026 16:15:16 +0000
Subject: [PATCH 09/30] fix: missing dependency in tests

---
 tests/test_main.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/test_main.py b/tests/test_main.py
index e1cee62..fa4b6cb 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -60,8 +60,6 @@ class TestHandler:
     @patch("src.main.github_api_toolkit.get_token_as_installation")
     @patch("src.main.github_api_toolkit.github_interface")
     @patch("src.main.get_and_update_historic_usage")
-    @patch("src.main.get_and_update_copilot_teams")
-    @patch("src.main.create_dictionary")
     @patch("src.main.update_s3_object")
     def test_handler_success(
         self,

From a755b3e87698fe7cb6be2a45f620fcdbb5c2e985 Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Thu, 19 Mar 2026 16:21:59 +0000
Subject: [PATCH 10/30] refactor: improve assertions in test_handler_success

---
 tests/test_main.py | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/tests/test_main.py b/tests/test_main.py
index fa4b6cb..5b70772 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -60,10 +60,8 @@ class TestHandler:
     @patch("src.main.github_api_toolkit.get_token_as_installation")
     @patch("src.main.github_api_toolkit.github_interface")
     @patch("src.main.get_and_update_historic_usage")
-    @patch("src.main.update_s3_object")
     def test_handler_success(
         self,
-        mock_update_s3_object,
         mock_get_and_update_historic_usage,
         mock_github_interface,
         mock_get_token_as_installation,
@@ -87,13 +85,6 @@ def test_handler_success(
         secret_region = "eu-west-1"
         secret_name = "test-secret"
 
-        # S3 get_object for teams_history.json returns existing history
-        mock_s3.get_object.return_value = {
-            "Body": MagicMock(
-                read=MagicMock(return_value=b'[{"team": {"name": "team1"}, "data": []}]')
-            )
-        }
-
         result = handler({}, MagicMock())
         assert result == "Github Data logging is now complete."
         mock_boto3_session.assert_called_once()
@@ -101,8 +92,8 @@ def test_handler_success(
         call("secretsmanager", region_name=secret_region) in mock_session.client.call_args_list
         mock_secret_manager.get_secret_value.assert_called_once_with(SecretId=secret_name)
         mock_get_token_as_installation.assert_called_once()
-        mock_github_interface.assert_called_once()
-        mock_get_and_update_historic_usage.assert_called_once()
+        mock_github_interface.assert_called_once_with("token")
+        mock_get_and_update_historic_usage.assert_called_once_with(mock_s3, mock_gh, False)
 
     @patch("src.main.boto3.Session")
     @patch("src.main.github_api_toolkit.get_token_as_installation")

From 501a5be99b7c2766524e9a679b36da115d65868d Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Thu, 19 Mar 2026 16:40:25 +0000
Subject: [PATCH 11/30] fix: bump black to 26.3.1

---
 poetry.lock    | 117 +++++++++++++++++++++++++++++++++++++------------
 pyproject.toml |   2 +-
 2 files changed, 90 insertions(+), 29 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index 560f3c3..78d2ae2 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -49,48 +49,54 @@ extras = ["regex"]
 
 [[package]]
 name = "black"
-version = "24.10.0"
+version = "26.3.1"
 description = "The uncompromising code formatter."
 optional = false
-python-versions = ">=3.9"
+python-versions = ">=3.10"
 groups = ["dev"]
 files = [
-    {file = "black-24.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6668650ea4b685440857138e5fe40cde4d652633b1bdffc62933d0db4ed9812"},
-    {file = "black-24.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1c536fcf674217e87b8cc3657b81809d3c085d7bf3ef262ead700da345bfa6ea"},
-    {file = "black-24.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:649fff99a20bd06c6f727d2a27f401331dc0cc861fb69cde910fe95b01b5928f"},
-    {file = "black-24.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:fe4d6476887de70546212c99ac9bd803d90b42fc4767f058a0baa895013fbb3e"},
-    {file = "black-24.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5a2221696a8224e335c28816a9d331a6c2ae15a2ee34ec857dcf3e45dbfa99ad"},
-    {file = "black-24.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f9da3333530dbcecc1be13e69c250ed8dfa67f43c4005fb537bb426e19200d50"},
-    {file = "black-24.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4007b1393d902b48b36958a216c20c4482f601569d19ed1df294a496eb366392"},
-    {file = "black-24.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:394d4ddc64782e51153eadcaaca95144ac4c35e27ef9b0a42e121ae7e57a9175"},
-    {file = "black-24.10.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b5e39e0fae001df40f95bd8cc36b9165c5e2ea88900167bddf258bacef9bbdc3"},
-    {file = "black-24.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d37d422772111794b26757c5b55a3eade028aa3fde43121ab7b673d050949d65"},
-    {file = "black-24.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14b3502784f09ce2443830e3133dacf2c0110d45191ed470ecb04d0f5f6fcb0f"},
-    {file = "black-24.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:30d2c30dc5139211dda799758559d1b049f7f14c580c409d6ad925b74a4208a8"},
-    {file = "black-24.10.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cbacacb19e922a1d75ef2b6ccaefcd6e93a2c05ede32f06a21386a04cedb981"},
-    {file = "black-24.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1f93102e0c5bb3907451063e08b9876dbeac810e7da5a8bfb7aeb5a9ef89066b"},
-    {file = "black-24.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ddacb691cdcdf77b96f549cf9591701d8db36b2f19519373d60d31746068dbf2"},
-    {file = "black-24.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:680359d932801c76d2e9c9068d05c6b107f2584b2a5b88831c83962eb9984c1b"},
-    {file = "black-24.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:17374989640fbca88b6a448129cd1745c5eb8d9547b464f281b251dd00155ccd"},
-    {file = "black-24.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:63f626344343083322233f175aaf372d326de8436f5928c042639a4afbbf1d3f"},
-    {file = "black-24.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfa1d0cb6200857f1923b602f978386a3a2758a65b52e0950299ea014be6800"},
-    {file = "black-24.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:2cd9c95431d94adc56600710f8813ee27eea544dd118d45896bb734e9d7a0dc7"},
-    {file = "black-24.10.0-py3-none-any.whl", hash = "sha256:3bb2b7a1f7b685f85b11fed1ef10f8a9148bceb49853e47a294a3dd963c1dd7d"},
-    {file = "black-24.10.0.tar.gz", hash = "sha256:846ea64c97afe3bc677b761787993be4991810ecc7a4a937816dd6bddedc4875"},
+    {file = "black-26.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:86a8b5035fce64f5dcd1b794cf8ec4d31fe458cf6ce3986a30deb434df82a1d2"},
+    {file = "black-26.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5602bdb96d52d2d0672f24f6ffe5218795736dd34807fd0fd55ccd6bf206168b"},
+    {file = "black-26.3.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c54a4a82e291a1fee5137371ab488866b7c86a3305af4026bdd4dc78642e1ac"},
+    {file = "black-26.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:6e131579c243c98f35bce64a7e08e87fb2d610544754675d4a0e73a070a5aa3a"},
+    {file = "black-26.3.1-cp310-cp310-win_arm64.whl", hash = "sha256:5ed0ca58586c8d9a487352a96b15272b7fa55d139fc8496b519e78023a8dab0a"},
+    {file = "black-26.3.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28ef38aee69e4b12fda8dba75e21f9b4f979b490c8ac0baa7cb505369ac9e1ff"},
+    {file = "black-26.3.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bf9bf162ed91a26f1adba8efda0b573bc6924ec1408a52cc6f82cb73ec2b142c"},
+    {file = "black-26.3.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:474c27574d6d7037c1bc875a81d9be0a9a4f9ee95e62800dab3cfaadbf75acd5"},
+    {file = "black-26.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:5e9d0d86df21f2e1677cc4bd090cd0e446278bcbbe49bf3659c308c3e402843e"},
+    {file = "black-26.3.1-cp311-cp311-win_arm64.whl", hash = "sha256:9a5e9f45e5d5e1c5b5c29b3bd4265dcc90e8b92cf4534520896ed77f791f4da5"},
+    {file = "black-26.3.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b5e6f89631eb88a7302d416594a32faeee9fb8fb848290da9d0a5f2903519fc1"},
+    {file = "black-26.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:41cd2012d35b47d589cb8a16faf8a32ef7a336f56356babd9fcf70939ad1897f"},
+    {file = "black-26.3.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f76ff19ec5297dd8e66eb64deda23631e642c9393ab592826fd4bdc97a4bce7"},
+    {file = "black-26.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:ddb113db38838eb9f043623ba274cfaf7d51d5b0c22ecb30afe58b1bb8322983"},
+    {file = "black-26.3.1-cp312-cp312-win_arm64.whl", hash = "sha256:dfdd51fc3e64ea4f35873d1b3fb25326773d55d2329ff8449139ebaad7357efb"},
+    {file = "black-26.3.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:855822d90f884905362f602880ed8b5df1b7e3ee7d0db2502d4388a954cc8c54"},
+    {file = "black-26.3.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8a33d657f3276328ce00e4d37fe70361e1ec7614da5d7b6e78de5426cb56332f"},
+    {file = "black-26.3.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f1cd08e99d2f9317292a311dfe578fd2a24b15dbce97792f9c4d752275c1fa56"},
+    {file = "black-26.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:c7e72339f841b5a237ff14f7d3880ddd0fc7f98a1199e8c4327f9a4f478c1839"},
+    {file = "black-26.3.1-cp313-cp313-win_arm64.whl", hash = "sha256:afc622538b430aa4c8c853f7f63bc582b3b8030fd8c80b70fb5fa5b834e575c2"},
+    {file = "black-26.3.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2d6bfaf7fd0993b420bed691f20f9492d53ce9a2bcccea4b797d34e947318a78"},
+    {file = "black-26.3.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f89f2ab047c76a9c03f78d0d66ca519e389519902fa27e7a91117ef7611c0568"},
+    {file = "black-26.3.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b07fc0dab849d24a80a29cfab8d8a19187d1c4685d8a5e6385a5ce323c1f015f"},
+    {file = "black-26.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:0126ae5b7c09957da2bdbd91a9ba1207453feada9e9fe51992848658c6c8e01c"},
+    {file = "black-26.3.1-cp314-cp314-win_arm64.whl", hash = "sha256:92c0ec1f2cc149551a2b7b47efc32c866406b6891b0ee4625e95967c8f4acfb1"},
+    {file = "black-26.3.1-py3-none-any.whl", hash = "sha256:2bd5aa94fc267d38bb21a70d7410a89f1a1d318841855f698746f8e7f51acd1b"},
+    {file = "black-26.3.1.tar.gz", hash = "sha256:2c50f5063a9641c7eed7795014ba37b0f5fa227f3d408b968936e24bc0566b07"},
 ]
 
 [package.dependencies]
 click = ">=8.0.0"
 mypy-extensions = ">=0.4.3"
 packaging = ">=22.0"
-pathspec = ">=0.9.0"
+pathspec = ">=1.0.0"
 platformdirs = ">=2"
+pytokens = ">=0.4.0,<0.5.0"
 
 [package.extras]
 colorama = ["colorama (>=0.4.3)"]
 d = ["aiohttp (>=3.10)"]
 jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"]
-uvloop = ["uvloop (>=0.15.2)"]
+uvloop = ["uvloop (>=0.15.2) ; sys_platform != \"win32\"", "winloop (>=0.5.0) ; sys_platform == \"win32\""]
 
 [[package]]
 name = "boto3"
@@ -369,7 +375,7 @@ files = [
     {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
     {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
 ]
-markers = {dev = "platform_system == \"Windows\" or sys_platform == \"win32\""}
+markers = {dev = "sys_platform == \"win32\" or platform_system == \"Windows\""}
 
 [[package]]
 name = "coverage"
@@ -1276,6 +1282,61 @@ files = [
 [package.dependencies]
 six = ">=1.5"
 
+[[package]]
+name = "pytokens"
+version = "0.4.1"
+description = "A Fast, spec compliant Python 3.14+ tokenizer that runs on older Pythons."
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "pytokens-0.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2a44ed93ea23415c54f3face3b65ef2b844d96aeb3455b8a69b3df6beab6acc5"},
+    {file = "pytokens-0.4.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:add8bf86b71a5d9fb5b89f023a80b791e04fba57960aa790cc6125f7f1d39dfe"},
+    {file = "pytokens-0.4.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:670d286910b531c7b7e3c0b453fd8156f250adb140146d234a82219459b9640c"},
+    {file = "pytokens-0.4.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4e691d7f5186bd2842c14813f79f8884bb03f5995f0575272009982c5ac6c0f7"},
+    {file = "pytokens-0.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:27b83ad28825978742beef057bfe406ad6ed524b2d28c252c5de7b4a6dd48fa2"},
+    {file = "pytokens-0.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d70e77c55ae8380c91c0c18dea05951482e263982911fc7410b1ffd1dadd3440"},
+    {file = "pytokens-0.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a58d057208cb9075c144950d789511220b07636dd2e4708d5645d24de666bdc"},
+    {file = "pytokens-0.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b49750419d300e2b5a3813cf229d4e5a4c728dae470bcc89867a9ad6f25a722d"},
+    {file = "pytokens-0.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d9907d61f15bf7261d7e775bd5d7ee4d2930e04424bab1972591918497623a16"},
+    {file = "pytokens-0.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:ee44d0f85b803321710f9239f335aafe16553b39106384cef8e6de40cb4ef2f6"},
+    {file = "pytokens-0.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:140709331e846b728475786df8aeb27d24f48cbcf7bcd449f8de75cae7a45083"},
+    {file = "pytokens-0.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d6c4268598f762bc8e91f5dbf2ab2f61f7b95bdc07953b602db879b3c8c18e1"},
+    {file = "pytokens-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24afde1f53d95348b5a0eb19488661147285ca4dd7ed752bbc3e1c6242a304d1"},
+    {file = "pytokens-0.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ad948d085ed6c16413eb5fec6b3e02fa00dc29a2534f088d3302c47eb59adf9"},
+    {file = "pytokens-0.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:3f901fe783e06e48e8cbdc82d631fca8f118333798193e026a50ce1b3757ea68"},
+    {file = "pytokens-0.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8bdb9d0ce90cbf99c525e75a2fa415144fd570a1ba987380190e8b786bc6ef9b"},
+    {file = "pytokens-0.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5502408cab1cb18e128570f8d598981c68a50d0cbd7c61312a90507cd3a1276f"},
+    {file = "pytokens-0.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29d1d8fb1030af4d231789959f21821ab6325e463f0503a61d204343c9b355d1"},
+    {file = "pytokens-0.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:970b08dd6b86058b6dc07efe9e98414f5102974716232d10f32ff39701e841c4"},
+    {file = "pytokens-0.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:9bd7d7f544d362576be74f9d5901a22f317efc20046efe2034dced238cbbfe78"},
+    {file = "pytokens-0.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4a14d5f5fc78ce85e426aa159489e2d5961acf0e47575e08f35584009178e321"},
+    {file = "pytokens-0.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f50fd18543be72da51dd505e2ed20d2228c74e0464e4262e4899797803d7fa"},
+    {file = "pytokens-0.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc74c035f9bfca0255c1af77ddd2d6ae8419012805453e4b0e7513e17904545d"},
+    {file = "pytokens-0.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324"},
+    {file = "pytokens-0.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:b35d7e5ad269804f6697727702da3c517bb8a5228afa450ab0fa787732055fc9"},
+    {file = "pytokens-0.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8fcb9ba3709ff77e77f1c7022ff11d13553f3c30299a9fe246a166903e9091eb"},
+    {file = "pytokens-0.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79fc6b8699564e1f9b521582c35435f1bd32dd06822322ec44afdeba666d8cb3"},
+    {file = "pytokens-0.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d31b97b3de0f61571a124a00ffe9a81fb9939146c122c11060725bd5aea79975"},
+    {file = "pytokens-0.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:967cf6e3fd4adf7de8fc73cd3043754ae79c36475c1c11d514fc72cf5490094a"},
+    {file = "pytokens-0.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:584c80c24b078eec1e227079d56dc22ff755e0ba8654d8383b2c549107528918"},
+    {file = "pytokens-0.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:da5baeaf7116dced9c6bb76dc31ba04a2dc3695f3d9f74741d7910122b456edc"},
+    {file = "pytokens-0.4.1-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:11edda0942da80ff58c4408407616a310adecae1ddd22eef8c692fe266fa5009"},
+    {file = "pytokens-0.4.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0fc71786e629cef478cbf29d7ea1923299181d0699dbe7c3c0f4a583811d9fc1"},
+    {file = "pytokens-0.4.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:dcafc12c30dbaf1e2af0490978352e0c4041a7cde31f4f81435c2a5e8b9cabb6"},
+    {file = "pytokens-0.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:42f144f3aafa5d92bad964d471a581651e28b24434d184871bd02e3a0d956037"},
+    {file = "pytokens-0.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:34bcc734bd2f2d5fe3b34e7b3c0116bfb2397f2d9666139988e7a3eb5f7400e3"},
+    {file = "pytokens-0.4.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:941d4343bf27b605e9213b26bfa1c4bf197c9c599a9627eb7305b0defcfe40c1"},
+    {file = "pytokens-0.4.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3ad72b851e781478366288743198101e5eb34a414f1d5627cdd585ca3b25f1db"},
+    {file = "pytokens-0.4.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:682fa37ff4d8e95f7df6fe6fe6a431e8ed8e788023c6bcc0f0880a12eab80ad1"},
+    {file = "pytokens-0.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:30f51edd9bb7f85c748979384165601d028b84f7bd13fe14d3e065304093916a"},
+    {file = "pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de"},
+    {file = "pytokens-0.4.1.tar.gz", hash = "sha256:292052fe80923aae2260c073f822ceba21f3872ced9a68bb7953b348e561179a"},
+]
+
+[package.extras]
+dev = ["black", "build", "mypy", "pytest", "pytest-cov", "setuptools", "tox", "twine", "wheel"]
+
 [[package]]
 name = "pyyaml"
 version = "6.0.2"
@@ -1522,4 +1583,4 @@ watchmedo = ["PyYAML (>=3.10)"]
 [metadata]
 lock-version = "2.1"
 python-versions = "^3.12"
-content-hash = "eedc53a92c67abf43fe87b7f174ed56e885f73181ca849c4c3b9bb1edec773df"
+content-hash = "bbeda2848b9b4b6ee321fe0d887bd8e90c30f386ff1bcf5de394200cb3affcc4"
diff --git a/pyproject.toml b/pyproject.toml
index b336067..e734db3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,13 +25,13 @@ six = "^1.17.0"
 urllib3 = "^2.6.3"
 
 [tool.poetry.group.dev.dependencies]
-black = "^24.8.0"
 ruff = "^0.6.5"
 pylint = "^3.2.7"
 mypy = "^1.11.2"
 pytest = "^8.4.1"
 pytest-cov = "^6.2.1"
 pytest-xdist = "^3.8.0"
+black = "^26.3.1"
 
 [tool.poetry.group.docs.dependencies]
 mkdocs = "^1.6.0"

From e0ab4a60ce24ec11b9a2c69a50f9fa9f0755698b Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Thu, 19 Mar 2026 17:14:16 +0000
Subject: [PATCH 12/30] docs: update overview and team usage

---
 docs/technical_documentation/overview.md   | 4 ++--
 docs/technical_documentation/team_usage.md | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/technical_documentation/overview.md b/docs/technical_documentation/overview.md
index 880040d..b239674 100644
--- a/docs/technical_documentation/overview.md
+++ b/docs/technical_documentation/overview.md
@@ -37,8 +37,8 @@ This component is an imported library which is shared across multiple GitHub too
 
 ### Historic Usage Data
 
-This section gathers data from AWS S3. The Copilot usage endpoints have a limitation where they only return the last 100 days worth of information. To get around this, the project has an AWS Lambda function which runs weekly and stores data within an S3 bucket.
+This section gathers data from AWS S3. The Copilot usage endpoints have a limitation where they only return the last 28 days worth of information. To get around this, the project has an AWS Lambda function which runs weekly and stores data within an S3 bucket.
 
-### Copilot Teams Data
+### Copilot Teams Data (Deprecated - functionality removed but may be restored via alternative methods)
 
 This section gathers a list of teams within the organisation with Copilot data and updates the S3 bucket accordingly. This allows all relevant teams to be displayed within the dashboard.
diff --git a/docs/technical_documentation/team_usage.md b/docs/technical_documentation/team_usage.md
index cc43d8f..b50bf39 100644
--- a/docs/technical_documentation/team_usage.md
+++ b/docs/technical_documentation/team_usage.md
@@ -1,5 +1,7 @@
 # Copilot Team Usage
 
+Note: This functionality has been removed as of 19th March 2026 as the endpoint used to fetch metrics for team usage is being sunsetted. However, it may be restored via alternative methods in the future.
+
 ## Overview
 
 This section of the project leverages GitHub OAuth2 for user authentication, granting access to essential data.

From ae364016ca6144085386790e78e327ecf89d230c Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Thu, 19 Mar 2026 17:26:47 +0000
Subject: [PATCH 13/30] chore: comment main guard

---
 src/main.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main.py b/src/main.py
index 0950cf0..92daa3f 100644
--- a/src/main.py
+++ b/src/main.py
@@ -280,5 +280,5 @@ def handler(event: dict, context) -> str:  # pylint: disable=unused-argument, to
 
 # Dev Only
 # Uncomment the following line to run the script locally
-if __name__ == "__main__":
-    handler(None, None)
+# if __name__ == "__main__":
+#     handler(None, None)

From 29ac72b05830010551f6b0eaa7536d0bc993a90f Mon Sep 17 00:00:00 2001
From: Hadi <76945238+hadiqur@users.noreply.github.com>
Date: Sun, 22 Mar 2026 15:54:18 +0000
Subject: [PATCH 14/30] fix: update pyproject.toml

---
 pyproject.toml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 4fd2978..4026295 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,7 +25,6 @@ six = "^1.17.0"
 urllib3 = "^2.6.3"
 
 [tool.poetry.group.dev.dependencies]
-black = "^26.3.1"
 ruff = "^0.6.5"
 pylint = "^3.2.7"
 mypy = "^1.11.2"
@@ -112,4 +111,4 @@ warn_redundant_casts = "True"
 disallow_untyped_defs = "True"
 disallow_untyped_calls = "True"
 disallow_incomplete_defs = "True"
-strict_equality = "True"
\ No newline at end of file
+strict_equality = "True"

From ec65eb82ad8001062a0ccb933ae3a6f2d6f3b096 Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Mon, 23 Mar 2026 10:26:22 +0000
Subject: [PATCH 15/30] docs: remove everything related to team usage

---
 docs/technical_documentation/overview.md   |  4 --
 docs/technical_documentation/team_usage.md | 81 ----------------------
 2 files changed, 85 deletions(-)
 delete mode 100644 docs/technical_documentation/team_usage.md

diff --git a/docs/technical_documentation/overview.md b/docs/technical_documentation/overview.md
index b239674..1a91246 100644
--- a/docs/technical_documentation/overview.md
+++ b/docs/technical_documentation/overview.md
@@ -38,7 +38,3 @@ This component is an imported library which is shared across multiple GitHub too
 ### Historic Usage Data
 
 This section gathers data from AWS S3. The Copilot usage endpoints have a limitation where they only return the last 28 days worth of information. To get around this, the project has an AWS Lambda function which runs weekly and stores data within an S3 bucket.
-
-### Copilot Teams Data (Deprecated - functionality removed but may be restored via alternative methods)
-
-This section gathers a list of teams within the organisation with Copilot data and updates the S3 bucket accordingly. This allows all relevant teams to be displayed within the dashboard.
diff --git a/docs/technical_documentation/team_usage.md b/docs/technical_documentation/team_usage.md
deleted file mode 100644
index b50bf39..0000000
--- a/docs/technical_documentation/team_usage.md
+++ /dev/null
@@ -1,81 +0,0 @@
-# Copilot Team Usage
-
-Note: This functionality has been removed as of 19th March 2026 as the endpoint used to fetch metrics for team usage is being sunsetted. However, it may be restored via alternative methods in the future.
-
-## Overview
-
-This section of the project leverages GitHub OAuth2 for user authentication, granting access to essential data.
-
-The tech stack and UI is the same as the Organization Usage.
-
-### Requirements for Team Copilot Usage
-
-To retrieve data, a team must meet the following criteria:
-
-- At least 5 members with active Copilot licenses.
-- These 5 users must be active for a minimum of 1 day.
-
-## Access
-
-### Flow & Session
-
-When on the team usage page, a `Login with GitHub` is displayed. Once clicked on, the user is taken to GitHub's authorisation page. The permissions **read-only** personal user data and **read-only** organizations and teams are required for this application. Clicking on the green `Authorize` button takes the user back to the application.
-
-### Types of Access
-
-#### Regular User
-
-A user within ONSDigital. Upon authentication, the app identifies the teams they belong to and populates the UI selection accordingly. If the user is part of a qualifying team, they can view the data. Users not associated with any team cannot select teams.
-
-#### Admin User
-
-An enhanced regular user with the ability to view any team. This user belongs to a specific whitelisted team, enabling them to view metrics for any team that meets the Copilot usage data requirements.
-
-## Metrics
-
-### Team History Metrics
-
-The team history metrics function retrieves historical usage data for each team identified with Copilot usage. This data includes detailed metrics about the team's activity over time. New data for a team is fetched only from the last captured date in the file.
-
-#### Functionality
-
-- **Input**: The function in addition to the GitHub Client takes a team name, organisation and the optional "since" as a query parameter as input.
-- **Process**:
-  - Fetches historical data for the specified team using the GitHub API.
-  - If the since query parameter exist then fetch data only after the specified date.
-  - Filters and organizes the data into a structured format.
-- **Output**: A JSON object containing the team's historical metrics, including:
-  - Team name
-  - Activity data
-  - Copilot usage statistics
-
-#### Usage
-
-The historical metrics are stored in an S3 bucket as a json file (`teams_history.json`).
-
-#### Example
-
-For a team named `kehdev`, the historical metrics might include:
-
-```json
-{
-  "team": {
-    "name": "kehdev",
-    "slug": "kehdev",
-    "description": "Team responsible for CI/CD pipelines",
-    "url": "https://github.com/orgs/<organisation>/teams/kehdev"
-  },
-  "data": [
-    {
-      "date": "2025-07-01",
-      "active_members": 10,
-      "copilot_usage_hours": 50
-    },
-    {
-      "date": "2025-07-02",
-      "active_members": 12,
-      "copilot_usage_hours": 60
-    }
-  ]
-}
-```

From 28c4bea5b44c65ba5feb8b580d4470cc48a01d9c Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Mon, 23 Mar 2026 10:28:48 +0000
Subject: [PATCH 16/30] refactor: rename file in s3

---
 src/main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main.py b/src/main.py
index 92daa3f..d6accdf 100644
--- a/src/main.py
+++ b/src/main.py
@@ -29,7 +29,7 @@
 
 # AWS Bucket Path
 BUCKET_NAME = f"{account}-copilot-usage-dashboard"
-OBJECT_NAME = "org_history.json"
+OBJECT_NAME = "organisation_history.json"
 
 logger = logging.getLogger()
 

From 3edecb9252c5bb8aa9479ca0bfbc18f1e8703922 Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Mon, 23 Mar 2026 10:30:28 +0000
Subject: [PATCH 17/30] refactor: always log locally if logging toggled

---
 src/main.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/src/main.py b/src/main.py
index d6accdf..a63342b 100644
--- a/src/main.py
+++ b/src/main.py
@@ -220,16 +220,6 @@ def handler(event: dict, context) -> str:  # pylint: disable=unused-argument, to
 
     # Toggle local logging
     if show_log_locally:
-        # This is a nightmare to test as it's really hard to get to.
-        # At some point we should look to make a wrapper for logging
-        # so it can be tested more easily.
-        logging.basicConfig(
-            filename="debug.log",
-            filemode="w",
-            format="%(asctime)s %(levelname)s %(message)s",
-        )
-    else:
-        # Ensure INFO logs show in the terminal when not logging to a file
         logging.basicConfig(
             level=logging.INFO,
             format="%(asctime)s %(levelname)s %(message)s",

From cbfff97a4a09e11e15a95c60a5696d8b607cbd07 Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Mon, 23 Mar 2026 10:31:56 +0000
Subject: [PATCH 18/30] refactor: update tests

---
 tests/test_main.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/test_main.py b/tests/test_main.py
index 5b70772..6742263 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -128,7 +128,7 @@ def test_get_and_update_historic_usage_success(self):
         # Mock API response
         api_response = {
             "download_links": [
-                "https://example.com/org_history_api_response.json"
+                "https://example.com/organisation_history_api_response.json"
             ]
             # There are other fields in the API response, but we don't need them for this test
         }
@@ -163,7 +163,7 @@ def test_get_and_update_historic_usage_success(self):
         s3.put_object.assert_called_once()
         args, kwargs = s3.put_object.call_args
         assert kwargs["Bucket"].endswith("copilot-usage-dashboard")
-        assert kwargs["Key"] == "org_history.json"
+        assert kwargs["Key"] == "organisation_history.json"
         assert json.loads(kwargs["Body"].decode("utf-8")) == result
 
     def test_get_and_update_historic_usage_no_existing_data(self, caplog):
@@ -171,7 +171,7 @@ def test_get_and_update_historic_usage_no_existing_data(self, caplog):
         gh = MagicMock()
         api_response = {
             "download_links": [
-                "https://example.com/org_history_api_response.json"
+                "https://example.com/organisation_history_api_response.json"
             ]
         }
         fetched_usage_data = {"day_totals": [
@@ -194,7 +194,7 @@ def test_get_and_update_historic_usage_no_existing_data(self, caplog):
         assert dates_added == ["2024-01-01"]
         s3.put_object.assert_called_once()
         assert any(
-            "Error getting org_history.json" in record.getMessage()
+            "Error getting organisation_history.json" in record.getMessage()
             for record in caplog.records
         )
 
@@ -203,7 +203,7 @@ def test_get_and_update_historic_usage_no_new_dates(self):
         gh = MagicMock()
         api_response = {
             "download_links": [
-                "https://example.com/org_history_api_response.json"
+                "https://example.com/organisation_history_api_response.json"
             ]
         }
         fetched_usage_data = {"day_totals": [
@@ -230,7 +230,7 @@ def test_write_data_locally_creates_file(self, tmp_path):
         gh = MagicMock()
         api_response = {
             "download_links": [
-                "https://example.com/org_history_api_response.json"
+                "https://example.com/organisation_history_api_response.json"
             ]
         }
         fetched_usage_data = {"day_totals": [

From d5579a04a0d5a319ac2b161b484a7091b72b949c Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Mon, 23 Mar 2026 11:31:55 +0000
Subject: [PATCH 19/30] fix: catch AttributeError exception when serializing
 response as json

---
 src/main.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/main.py b/src/main.py
index a63342b..03160f3 100644
--- a/src/main.py
+++ b/src/main.py
@@ -71,11 +71,17 @@ def get_and_update_historic_usage(
         tuple: A tuple containing the updated historic usage data and a list of dates added.
     """
     # Get the usage data
-    api_response = gh.get(f"/orgs/{org}/copilot/metrics/reports/organization-28-day/latest").json()
-    usage_data = requests.get(api_response["download_links"][0], timeout=30).json()["day_totals"]
+    try:
+        api_response = gh.get(f"/orgs/{org}/copilot/metrics/reports/organization-28-day/latest")
+        api_response_json = api_response.json()
+    except AttributeError as e: 
+        logger.error("Error getting usage data: %s", api_response)
+        return [], []
 
+    usage_data = requests.get(api_response_json["download_links"][0], timeout=30).json()["day_totals"]
     logger.info("Usage data retrieved")
 
+    # Get the existing historic usage data from S3
     try:
         response = s3.get_object(Bucket=BUCKET_NAME, Key=OBJECT_NAME)
         historic_usage = json.loads(response["Body"].read().decode("utf-8"))
@@ -83,6 +89,7 @@ def get_and_update_historic_usage(
         logger.error("Error getting %s: %s. Using empty list.", OBJECT_NAME, e)
         historic_usage = []
 
+    # Append the new usage data to the existing historic usage data
     dates_added = []
 
     for day in usage_data:
@@ -94,7 +101,7 @@ def get_and_update_historic_usage(
     sorted_historic_usage = sorted(historic_usage, key=lambda x: x["day"])
 
     if not write_data_locally:
-        # Write the updated historic_usage to historic_usage_data.json
+        # Write the updated historic_usage to organisation_history.json
         update_s3_object(s3, BUCKET_NAME, OBJECT_NAME, sorted_historic_usage)
     else:
         local_path = f"output/{OBJECT_NAME}"
@@ -255,7 +262,7 @@ def handler(event: dict, context) -> str:  # pylint: disable=unused-argument, to
     historic_usage, dates_added = get_and_update_historic_usage(s3, gh, write_data_locally)
 
     logger.info(
-        "Process complete",
+        "Process finished",
         extra={
             "bucket": BUCKET_NAME,
             "no_days_added": len(dates_added),

From 75ea3d25ff9622eba109627f28588782e0216871 Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Mon, 23 Mar 2026 11:53:29 +0000
Subject: [PATCH 20/30] perf: only sort newly added dates rather than whole
 historic usage

---
 src/main.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/main.py b/src/main.py
index 03160f3..cc72f08 100644
--- a/src/main.py
+++ b/src/main.py
@@ -91,23 +91,24 @@ def get_and_update_historic_usage(
 
     # Append the new usage data to the existing historic usage data
     dates_added = []
+    new_usage_data = []
 
     for day in usage_data:
         if not any(d["day"] == day["day"] for d in historic_usage):
-            historic_usage.append(day)
+            new_usage_data.append(day)
             dates_added.append(day["day"])
             logger.info("Added data for day %s", day["day"])
 
-    sorted_historic_usage = sorted(historic_usage, key=lambda x: x["day"])
+    historic_usage.extend(sorted(new_usage_data, key=lambda x: x["day"]))
 
     if not write_data_locally:
         # Write the updated historic_usage to organisation_history.json
-        update_s3_object(s3, BUCKET_NAME, OBJECT_NAME, sorted_historic_usage)
+        update_s3_object(s3, BUCKET_NAME, OBJECT_NAME, historic_usage)
     else:
         local_path = f"output/{OBJECT_NAME}"
         os.makedirs("output", exist_ok=True)
         with open(local_path, "w", encoding="utf-8") as f:
-            json.dump(sorted_historic_usage, f, indent=4)
+            json.dump(historic_usage, f, indent=4)
         logger.info("Historic usage data written locally to %s (S3 skipped)", local_path)
 
     logger.info(
@@ -117,7 +118,7 @@ def get_and_update_historic_usage(
         dates_added,
     )
 
-    return sorted_historic_usage, dates_added
+    return historic_usage, dates_added
 
 
 def update_s3_object(
@@ -277,5 +278,5 @@ def handler(event: dict, context) -> str:  # pylint: disable=unused-argument, to
 
 # Dev Only
 # Uncomment the following line to run the script locally
-# if __name__ == "__main__":
-#     handler(None, None)
+if __name__ == "__main__":
+    handler(None, None)

From 9eff15897a251d882942a06fa0d51b5e54a08b7d Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Mon, 23 Mar 2026 11:57:10 +0000
Subject: [PATCH 21/30] refactor: update files to reflect current logging
 practice

---
 config/config.json | 2 +-
 src/main.py        | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/config/config.json b/config/config.json
index b01f8a1..acba32d 100644
--- a/config/config.json
+++ b/config/config.json
@@ -1,6 +1,6 @@
 {
   "features": {
-    "show_log_locally": false,
+    "show_logs_in_terminal": false,
     "write_data_locally": false
   }
 }
diff --git a/src/main.py b/src/main.py
index cc72f08..6bfe88f 100644
--- a/src/main.py
+++ b/src/main.py
@@ -222,12 +222,12 @@ def handler(event: dict, context) -> str:  # pylint: disable=unused-argument, to
 
     features = get_dict_value(config, "features")
 
-    show_log_locally = get_dict_value(features, "show_log_locally")
+    show_logs_in_terminal = get_dict_value(features, "show_logs_in_terminal")
 
     write_data_locally = get_dict_value(features, "write_data_locally")
 
     # Toggle local logging
-    if show_log_locally:
+    if show_logs_in_terminal:
         logging.basicConfig(
             level=logging.INFO,
             format="%(asctime)s %(levelname)s %(message)s",

From 2e328d46259a34ebd79857aacbb96f79a09f7e94 Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Mon, 23 Mar 2026 12:05:15 +0000
Subject: [PATCH 22/30] perf: only import get from requests library

---
 src/main.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main.py b/src/main.py
index 6bfe88f..f4a2ba9 100644
--- a/src/main.py
+++ b/src/main.py
@@ -12,7 +12,7 @@
 
 import boto3
 import github_api_toolkit
-import requests
+from requests import get
 from botocore.exceptions import ClientError
 
 # GitHub Organisation
@@ -78,7 +78,7 @@ def get_and_update_historic_usage(
         logger.error("Error getting usage data: %s", api_response)
         return [], []
 
-    usage_data = requests.get(api_response_json["download_links"][0], timeout=30).json()["day_totals"]
+    usage_data = get(api_response_json["download_links"][0], timeout=30).json()["day_totals"]
     logger.info("Usage data retrieved")
 
     # Get the existing historic usage data from S3

From 4bfc290382750fd8b5107e5ea9c4de698ac2e0a3 Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Mon, 23 Mar 2026 12:53:12 +0000
Subject: [PATCH 23/30] tests: improve readability by moving mock api response
 and usage data to top

---
 tests/test_main.py | 81 +++++++++++++++++-----------------------------
 1 file changed, 29 insertions(+), 52 deletions(-)

diff --git a/tests/test_main.py b/tests/test_main.py
index 6742263..989c888 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -17,8 +17,23 @@
     update_s3_object,
     get_dict_value,
     get_config_file,
+
 )
 
+# Mock API response
+api_response = {
+    "download_links": [
+        "https://example.com/organisation_history_api_response.json"
+    ]
+    # There are other fields in the API response, but we don't need them for this test
+}
+
+# Mock usage data returned from GitHub API 
+fetched_usage_data = {"day_totals": [
+    {"day": "2024-01-01", "usage": 10},
+    {"day": "2024-01-02", "usage": 20},
+]}
+
 
 class TestUpdateS3Object:
     def test_update_s3_object_success(self, caplog):
@@ -124,20 +139,6 @@ def teardown_method(self):
     def test_get_and_update_historic_usage_success(self):
         s3 = MagicMock()
         gh = MagicMock()
-
-        # Mock API response
-        api_response = {
-            "download_links": [
-                "https://example.com/organisation_history_api_response.json"
-            ]
-            # There are other fields in the API response, but we don't need them for this test
-        }
-
-        # Mock usage data returned from GitHub API 
-        fetched_usage_data = {"day_totals": [
-            {"day": "2024-01-01", "usage": 10},
-            {"day": "2024-01-02", "usage": 20},
-        ]}
         
         gh.get.return_value.json.return_value = api_response
 
@@ -150,8 +151,8 @@ def test_get_and_update_historic_usage_success(self):
         # Mock requests.get returns usage data from download_links
         # We always patch dependencies imported inside the function we're testing.
         # Test environment initialisation ends here.
-        with patch("src.main.requests.get") as mock_requests_get:
-            mock_requests_get.return_value.json.return_value = fetched_usage_data
+        with patch("src.main.get") as mock_get:
+            mock_get.return_value.json.return_value = fetched_usage_data
             result, dates_added = get_and_update_historic_usage(s3, gh, False)
 
         assert result == [
@@ -169,14 +170,6 @@ def test_get_and_update_historic_usage_success(self):
     def test_get_and_update_historic_usage_no_existing_data(self, caplog):
         s3 = MagicMock()
         gh = MagicMock()
-        api_response = {
-            "download_links": [
-                "https://example.com/organisation_history_api_response.json"
-            ]
-        }
-        fetched_usage_data = {"day_totals": [
-            {"day": "2024-01-01", "usage": 10},
-        ]}
 
         gh.get.return_value.json.return_value = api_response
 
@@ -186,12 +179,12 @@ def test_get_and_update_historic_usage_no_existing_data(self, caplog):
             operation_name="GetObject",
         )
 
-        with patch("src.main.requests.get") as mock_requests_get:
-            mock_requests_get.return_value.json.return_value = fetched_usage_data
+        with patch("src.main.get") as mock_get:
+            mock_get.return_value.json.return_value = fetched_usage_data
             result, dates_added = get_and_update_historic_usage(s3, gh, False)
 
-        assert result == [{"day": "2024-01-01", "usage": 10}]
-        assert dates_added == ["2024-01-01"]
+        assert result == [{"day": "2024-01-01", "usage": 10}, {"day": "2024-01-02", "usage": 20}]
+        assert dates_added == ["2024-01-01", "2024-01-02"]
         s3.put_object.assert_called_once()
         assert any(
             "Error getting organisation_history.json" in record.getMessage()
@@ -201,41 +194,25 @@ def test_get_and_update_historic_usage_no_existing_data(self, caplog):
     def test_get_and_update_historic_usage_no_new_dates(self):
         s3 = MagicMock()
         gh = MagicMock()
-        api_response = {
-            "download_links": [
-                "https://example.com/organisation_history_api_response.json"
-            ]
-        }
-        fetched_usage_data = {"day_totals": [
-            {"day": "2024-01-01", "usage": 10},
-        ]}
         
         gh.get.return_value.json.return_value = api_response
 
         # S3 get_object returns same date as usage_data
-        existing_usage = [{"day": "2024-01-01", "usage": 10}]
+        existing_usage = [{"day": "2024-01-01", "usage": 10}, {"day": "2024-01-02", "usage": 20}]
         s3.get_object.return_value = {
             "Body": BytesIO(json.dumps(existing_usage).encode("utf-8"))
         }
-        with patch("src.main.requests.get") as mock_requests_get:
-            mock_requests_get.return_value.json.return_value = fetched_usage_data
+        with patch("src.main.get") as mock_get:
+            mock_get.return_value.json.return_value = fetched_usage_data
             result, dates_added = get_and_update_historic_usage(s3, gh, False)
 
-        assert result == [{"day": "2024-01-01", "usage": 10}]
+        assert result == [{"day": "2024-01-01", "usage": 10}, {"day": "2024-01-02", "usage": 20}]
         assert dates_added == []
         s3.put_object.assert_called_once()
 
     def test_write_data_locally_creates_file(self, tmp_path):
         s3 = MagicMock()
         gh = MagicMock()
-        api_response = {
-            "download_links": [
-                "https://example.com/organisation_history_api_response.json"
-            ]
-        }
-        fetched_usage_data = {"day_totals": [
-            {"day": "2024-01-01", "usage": 10},
-        ]}
         
         gh.get.return_value.json.return_value = api_response
 
@@ -248,11 +225,11 @@ def test_write_data_locally_creates_file(self, tmp_path):
         # Patch os.makedirs and open to use tmp_path
         with patch("src.main.os.makedirs") as mock_makedirs, \
                 patch("src.main.open", create=True) as mock_open, \
-                    patch("src.main.requests.get") as mock_requests_get:
-                        mock_requests_get.return_value.json.return_value = fetched_usage_data
+                    patch("src.main.get") as mock_get:
+                        mock_get.return_value.json.return_value = fetched_usage_data
                         result, dates_added = get_and_update_historic_usage(s3, gh, True)
-                        assert result == [{"day": "2024-01-01", "usage": 10}]
-                        assert dates_added == ["2024-01-01"]
+                        assert result == [{"day": "2024-01-01", "usage": 10}, {"day": "2024-01-02", "usage": 20}]
+                        assert dates_added == ["2024-01-01", "2024-01-02"]
                         mock_makedirs.assert_called_once_with("output", exist_ok=True)
                         mock_open.assert_called_once()
                         s3.put_object.assert_not_called()

From f620eadc4821f5078449e2c2a3a8f71a3ac8f1e3 Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Mon, 23 Mar 2026 12:56:00 +0000
Subject: [PATCH 24/30] chore: improve comments in test file

---
 tests/test_main.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_main.py b/tests/test_main.py
index 989c888..c7e3839 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -20,7 +20,7 @@
 
 )
 
-# Mock API response
+# Mock GitHub API response
 api_response = {
     "download_links": [
         "https://example.com/organisation_history_api_response.json"
@@ -28,7 +28,7 @@
     # There are other fields in the API response, but we don't need them for this test
 }
 
-# Mock usage data returned from GitHub API 
+# Mock usage data fetched from download_links key in the API response
 fetched_usage_data = {"day_totals": [
     {"day": "2024-01-01", "usage": 10},
     {"day": "2024-01-02", "usage": 20},

From 550bc27e4d1a75fd529b3a6172add42e228be3ad Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Mon, 23 Mar 2026 12:56:38 +0000
Subject: [PATCH 25/30] chore: improve comments in test file once more

---
 tests/test_main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_main.py b/tests/test_main.py
index c7e3839..e34b6ba 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -25,7 +25,7 @@
     "download_links": [
         "https://example.com/organisation_history_api_response.json"
     ]
-    # There are other fields in the API response, but we don't need them for this test
+    # There are other fields in the API response, but we don't need them for testing
 }
 
 # Mock usage data fetched from download_links key in the API response

From 105b9c2a9153af205b583968bd94df7c2ee6a155 Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Mon, 23 Mar 2026 14:30:32 +0000
Subject: [PATCH 26/30] perf: implement historic_usage_set to optimise new
 usage data lookup

---
 src/main.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main.py b/src/main.py
index f4a2ba9..34f1ed8 100644
--- a/src/main.py
+++ b/src/main.py
@@ -92,9 +92,10 @@ def get_and_update_historic_usage(
     # Append the new usage data to the existing historic usage data
     dates_added = []
     new_usage_data = []
+    historic_usage_set = {d["day"] for d in historic_usage}
 
     for day in usage_data:
-        if not any(d["day"] == day["day"] for d in historic_usage):
+        if not day["day"] in historic_usage_set:
             new_usage_data.append(day)
             dates_added.append(day["day"])
             logger.info("Added data for day %s", day["day"])

From eaac96457b06a3004d01c5d162a337c4bf1baaea Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Mon, 23 Mar 2026 14:31:27 +0000
Subject: [PATCH 27/30] fix: linting errors

---
 src/main.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main.py b/src/main.py
index 34f1ed8..4a41981 100644
--- a/src/main.py
+++ b/src/main.py
@@ -12,8 +12,8 @@
 
 import boto3
 import github_api_toolkit
-from requests import get
 from botocore.exceptions import ClientError
+from requests import get
 
 # GitHub Organisation
 org = os.getenv("GITHUB_ORG")
@@ -74,7 +74,7 @@ def get_and_update_historic_usage(
     try:
         api_response = gh.get(f"/orgs/{org}/copilot/metrics/reports/organization-28-day/latest")
         api_response_json = api_response.json()
-    except AttributeError as e: 
+    except AttributeError:
         logger.error("Error getting usage data: %s", api_response)
         return [], []
 
@@ -95,7 +95,7 @@ def get_and_update_historic_usage(
     historic_usage_set = {d["day"] for d in historic_usage}
 
     for day in usage_data:
-        if not day["day"] in historic_usage_set:
+        if day["day"] not in historic_usage_set:
             new_usage_data.append(day)
             dates_added.append(day["day"])
             logger.info("Added data for day %s", day["day"])

From 2f18b6e7cf028a11e16aecafd3e590865fc0f385 Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Mon, 23 Mar 2026 14:45:09 +0000
Subject: [PATCH 28/30] chore: comment main guard

---
 src/main.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main.py b/src/main.py
index 4a41981..c9bd8d4 100644
--- a/src/main.py
+++ b/src/main.py
@@ -279,5 +279,5 @@ def handler(event: dict, context) -> str:  # pylint: disable=unused-argument, to
 
 # Dev Only
 # Uncomment the following line to run the script locally
-if __name__ == "__main__":
-    handler(None, None)
+# if __name__ == "__main__":
+#     handler(None, None)

From eac51ff7b5db827daa17b619bcfd2592491641d4 Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Mon, 23 Mar 2026 15:08:46 +0000
Subject: [PATCH 29/30] refactor: logging enabled by default

---
 config/config.json |  1 -
 src/main.py        | 14 +++++---------
 2 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/config/config.json b/config/config.json
index acba32d..caa3775 100644
--- a/config/config.json
+++ b/config/config.json
@@ -1,6 +1,5 @@
 {
   "features": {
-    "show_logs_in_terminal": false,
     "write_data_locally": false
   }
 }
diff --git a/src/main.py b/src/main.py
index c9bd8d4..dca5318 100644
--- a/src/main.py
+++ b/src/main.py
@@ -222,17 +222,13 @@ def handler(event: dict, context) -> str:  # pylint: disable=unused-argument, to
     config = get_config_file("./config/config.json")
 
     features = get_dict_value(config, "features")
-
-    show_logs_in_terminal = get_dict_value(features, "show_logs_in_terminal")
-
+ 
     write_data_locally = get_dict_value(features, "write_data_locally")
 
-    # Toggle local logging
-    if show_logs_in_terminal:
-        logging.basicConfig(
-            level=logging.INFO,
-            format="%(asctime)s %(levelname)s %(message)s",
-        )
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)s %(message)s",
+    )
 
     # Create an S3 client
     session = boto3.Session()

From 298af15d23feb67edc4587c631a06e53d1ee6b3b Mon Sep 17 00:00:00 2001
From: Hadi Qureshi <hadi.qureshi@ons.gov.uk>
Date: Mon, 23 Mar 2026 15:10:53 +0000
Subject: [PATCH 30/30] fix: linting errors

---
 src/main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main.py b/src/main.py
index dca5318..54fce02 100644
--- a/src/main.py
+++ b/src/main.py
@@ -222,7 +222,7 @@ def handler(event: dict, context) -> str:  # pylint: disable=unused-argument, to
     config = get_config_file("./config/config.json")
 
     features = get_dict_value(config, "features")
- 
+
     write_data_locally = get_dict_value(features, "write_data_locally")
 
     logging.basicConfig(