diff --git a/config/config.json b/config/config.json index b01f8a1..caa3775 100644 --- a/config/config.json +++ b/config/config.json @@ -1,6 +1,5 @@ { "features": { - "show_log_locally": false, "write_data_locally": false } } diff --git a/docs/technical_documentation/overview.md b/docs/technical_documentation/overview.md index 880040d..1a91246 100644 --- a/docs/technical_documentation/overview.md +++ b/docs/technical_documentation/overview.md @@ -37,8 +37,4 @@ This component is an imported library which is shared across multiple GitHub too ### Historic Usage Data -This section gathers data from AWS S3. The Copilot usage endpoints have a limitation where they only return the last 100 days worth of information. To get around this, the project has an AWS Lambda function which runs weekly and stores data within an S3 bucket. - -### Copilot Teams Data - -This section gathers a list of teams within the organisation with Copilot data and updates the S3 bucket accordingly. This allows all relevant teams to be displayed within the dashboard. +This section gathers data from AWS S3. The Copilot usage endpoints have a limitation where they only return the last 28 days worth of information. To get around this, the project has an AWS Lambda function which runs weekly and stores data within an S3 bucket. diff --git a/docs/technical_documentation/team_usage.md b/docs/technical_documentation/team_usage.md deleted file mode 100644 index cc43d8f..0000000 --- a/docs/technical_documentation/team_usage.md +++ /dev/null @@ -1,79 +0,0 @@ -# Copilot Team Usage - -## Overview - -This section of the project leverages GitHub OAuth2 for user authentication, granting access to essential data. - -The tech stack and UI is the same as the Organization Usage. - -### Requirements for Team Copilot Usage - -To retrieve data, a team must meet the following criteria: - -- At least 5 members with active Copilot licenses. -- These 5 users must be active for a minimum of 1 day. - -## Access - -### Flow & Session - -When on the team usage page, a `Login with GitHub` is displayed. Once clicked on, the user is taken to GitHub's authorisation page. The permissions **read-only** personal user data and **read-only** organizations and teams are required for this application. Clicking on the green `Authorize` button takes the user back to the application. - -### Types of Access - -#### Regular User - -A user within ONSDigital. Upon authentication, the app identifies the teams they belong to and populates the UI selection accordingly. If the user is part of a qualifying team, they can view the data. Users not associated with any team cannot select teams. - -#### Admin User - -An enhanced regular user with the ability to view any team. This user belongs to a specific whitelisted team, enabling them to view metrics for any team that meets the Copilot usage data requirements. - -## Metrics - -### Team History Metrics - -The team history metrics function retrieves historical usage data for each team identified with Copilot usage. This data includes detailed metrics about the team's activity over time. New data for a team is fetched only from the last captured date in the file. - -#### Functionality - -- **Input**: The function in addition to the GitHub Client takes a team name, organisation and the optional "since" as a query parameter as input. -- **Process**: - - Fetches historical data for the specified team using the GitHub API. - - If the since query parameter exist then fetch data only after the specified date. - - Filters and organizes the data into a structured format. -- **Output**: A JSON object containing the team's historical metrics, including: - - Team name - - Activity data - - Copilot usage statistics - -#### Usage - -The historical metrics are stored in an S3 bucket as a json file (`teams_history.json`). - -#### Example - -For a team named `kehdev`, the historical metrics might include: - -```json -{ - "team": { - "name": "kehdev", - "slug": "kehdev", - "description": "Team responsible for CI/CD pipelines", - "url": "https://github.com/orgs//teams/kehdev" - }, - "data": [ - { - "date": "2025-07-01", - "active_members": 10, - "copilot_usage_hours": 50 - }, - { - "date": "2025-07-02", - "active_members": 12, - "copilot_usage_hours": 60 - } - ] -} -``` diff --git a/poetry.lock b/poetry.lock index 464b7fb..00e4705 100644 --- a/poetry.lock +++ b/poetry.lock @@ -375,7 +375,7 @@ files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -markers = {dev = "platform_system == \"Windows\" or sys_platform == \"win32\""} +markers = {dev = "sys_platform == \"win32\" or platform_system == \"Windows\""} [[package]] name = "coverage" diff --git a/pyproject.toml b/pyproject.toml index 1e87de2..4026295 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,13 +25,13 @@ six = "^1.17.0" urllib3 = "^2.6.3" [tool.poetry.group.dev.dependencies] -black = "^26.3.1" ruff = "^0.6.5" pylint = "^3.2.7" mypy = "^1.11.2" pytest = "^8.4.1" pytest-cov = "^6.2.1" pytest-xdist = "^3.8.0" +black = "^26.3.1" [tool.poetry.group.docs.dependencies] mkdocs = "^1.6.0" @@ -111,4 +111,4 @@ warn_redundant_casts = "True" disallow_untyped_defs = "True" disallow_untyped_calls = "True" disallow_incomplete_defs = "True" -strict_equality = "True" \ No newline at end of file +strict_equality = "True" diff --git a/src/main.py b/src/main.py index d75891a..54fce02 100644 --- a/src/main.py +++ b/src/main.py @@ -8,12 +8,12 @@ import json import logging import os -from typing import Any, Optional +from typing import Any import boto3 import github_api_toolkit from botocore.exceptions import ClientError -from requests import Response +from requests import get # GitHub Organisation org = os.getenv("GITHUB_ORG") @@ -29,7 +29,7 @@ # AWS Bucket Path BUCKET_NAME = f"{account}-copilot-usage-dashboard" -OBJECT_NAME = "historic_usage_data.json" +OBJECT_NAME = "organisation_history.json" logger = logging.getLogger() @@ -57,64 +57,6 @@ # } -def get_copilot_team_date(gh: github_api_toolkit.github_interface, page: int) -> list: - """Gets a list of GitHub Teams with Copilot Data for a given API page. - - Args: - gh (github_api_toolkit.github_interface): An instance of the github_interface class. - page (int): The page number of the API request. - - Returns: - list: A list of GitHub Teams with Copilot Data. - """ - copilot_teams = [] - - response = gh.get(f"/orgs/{org}/teams", params={"per_page": 100, "page": page}) - teams = response.json() - for team in teams: - usage_data = gh.get(f"/orgs/{org}/team/{team['name']}/copilot/metrics") - - if not isinstance(usage_data, Response): - - # If the response is not a Response object, no copilot data is available for this team - # We can then skip this team - - # We don't log this as an error, as it is expected and it'd be too noisy within logs - - continue - - # If the response has data, append the team to the list - # If there is no data, .json() will return an empty list - if usage_data.json(): - - team_name = team.get("name", "") - team_slug = team.get("slug", "") - team_description = team.get("description", "") - team_html_url = team.get("html_url", "") - - logger.info( - "Team %s has Copilot data", - team_name, - extra={ - "team_name": team_name, - "team_slug": team_slug, - "team_description": team_description, - "team_html_url": team_html_url, - }, - ) - - copilot_teams.append( - { - "name": team_name, - "slug": team_slug, - "description": team_description, - "url": team_html_url, - } - ) - - return copilot_teams - - def get_and_update_historic_usage( s3: boto3.client, gh: github_api_toolkit.github_interface, write_data_locally: bool ) -> tuple: @@ -129,36 +71,39 @@ def get_and_update_historic_usage( tuple: A tuple containing the updated historic usage data and a list of dates added. """ # Get the usage data - usage_data = gh.get(f"/orgs/{org}/copilot/metrics") - usage_data = usage_data.json() + try: + api_response = gh.get(f"/orgs/{org}/copilot/metrics/reports/organization-28-day/latest") + api_response_json = api_response.json() + except AttributeError: + logger.error("Error getting usage data: %s", api_response) + return [], [] + usage_data = get(api_response_json["download_links"][0], timeout=30).json()["day_totals"] logger.info("Usage data retrieved") + # Get the existing historic usage data from S3 try: response = s3.get_object(Bucket=BUCKET_NAME, Key=OBJECT_NAME) historic_usage = json.loads(response["Body"].read().decode("utf-8")) except ClientError as e: logger.error("Error getting %s: %s. Using empty list.", OBJECT_NAME, e) - historic_usage = [] + # Append the new usage data to the existing historic usage data dates_added = [] + new_usage_data = [] + historic_usage_set = {d["day"] for d in historic_usage} - # Append the new usage data to the historic_usage_data.json - for date in usage_data: - if not any(d["date"] == date["date"] for d in historic_usage): - historic_usage.append(date) + for day in usage_data: + if day["day"] not in historic_usage_set: + new_usage_data.append(day) + dates_added.append(day["day"]) + logger.info("Added data for day %s", day["day"]) - dates_added.append(date["date"]) - - logger.info( - "New usage data added to %s", - OBJECT_NAME, - extra={"no_days_added": len(dates_added), "dates_added": dates_added}, - ) + historic_usage.extend(sorted(new_usage_data, key=lambda x: x["day"])) if not write_data_locally: - # Write the updated historic_usage to historic_usage_data.json + # Write the updated historic_usage to organisation_history.json update_s3_object(s3, BUCKET_NAME, OBJECT_NAME, historic_usage) else: local_path = f"output/{OBJECT_NAME}" @@ -167,105 +112,14 @@ def get_and_update_historic_usage( json.dump(historic_usage, f, indent=4) logger.info("Historic usage data written locally to %s (S3 skipped)", local_path) - return historic_usage, dates_added - - -def get_and_update_copilot_teams( - s3: boto3.client, gh: github_api_toolkit.github_interface, write_data_locally: bool -) -> list: - """Get and update GitHub Teams with Copilot Data. - - Args: - s3 (boto3.client): An S3 client. - gh (github_api_toolkit.github_interface): An instance of the github_interface class. - write_data_locally (bool): Whether to write data locally instead of to an S3 bucket. - - Returns: - list: A list of GitHub Teams with Copilot Data. - """ - logger.info("Getting GitHub Teams with Copilot Data") - - copilot_teams = [] - - response = gh.get(f"/orgs/{org}/teams", params={"per_page": 100}) - - # Get the last page of teams - try: - last_page = int(response.links["last"]["url"].split("=")[-1]) - except KeyError: - last_page = 1 - - for page in range(1, last_page + 1): - page_teams = get_copilot_team_date(gh, page) - - copilot_teams = copilot_teams + page_teams - logger.info( - "Fetched GitHub Teams with Copilot Data", - extra={"no_teams": len(copilot_teams)}, + "Usage data written to %s: %d days added (%s)", + OBJECT_NAME, + len(dates_added), + dates_added, ) - if not write_data_locally: - update_s3_object(s3, BUCKET_NAME, "copilot_teams.json", copilot_teams) - else: - local_path = "output/copilot_teams.json" - os.makedirs("output", exist_ok=True) - with open(local_path, "w", encoding="utf-8") as f: - json.dump(copilot_teams, f, indent=4) - logger.info("Copilot teams data written locally to %s (S3 skipped)", local_path) - - return copilot_teams - - -def create_dictionary( - gh: github_api_toolkit.github_interface, copilot_teams: list, existing_team_history: list -) -> list: - """Create a dictionary for quick lookup of existing team data using the `name` field. - - Args: - gh (github_api_toolkit.github_interface): An instance of the github_interface class. - copilot_teams (list): List of teams with Copilot data. - existing_team_history (list): List of existing team history data. - - Returns: - list: A list of dictionaries containing team data and their history. - """ - existing_team_data_map = { - single_team["team"]["name"]: single_team for single_team in existing_team_history - } - - # Iterate through identified teams - for team in copilot_teams: - team_name = team.get("name", "") - if not team_name: - logger.warning("Skipping team with no name") - continue - - # Determine the last known date for the team - last_known_date = None - if team_name in existing_team_data_map: - existing_dates = [entry["date"] for entry in existing_team_data_map[team_name]["data"]] - if existing_dates: - last_known_date = max(existing_dates) # Get the most recent date - - # Assign the last known date to the `since` query parameter - query_params = {} - if last_known_date: - query_params["since"] = last_known_date - - single_team_history = get_team_history(gh, team_name, query_params) - if not single_team_history: - logger.info("No new history found for team %s", team_name) - continue - - # Append new data to the existing team history - new_team_data = single_team_history - if team_name in existing_team_data_map: - existing_team_data_map[team_name]["data"].extend(new_team_data) - else: - existing_team_data_map[team_name] = {"team": team, "data": new_team_data} - - return list(existing_team_data_map.values()) + return historic_usage, dates_added def update_s3_object( @@ -298,31 +152,6 @@ def update_s3_object( return False -def get_team_history( - gh: github_api_toolkit.github_interface, team: str, query_params: Optional[dict] = None -) -> list[dict]: - """Gets the team metrics Copilot data through the API. - Note - This endpoint will only return results for a given day if the team had - five or more members with active Copilot licenses on that day, - as evaluated at the end of that day. - - Args: - gh (github_api_toolkit.github_interface): An instance of the github_interface class. - team (str): Team name. - query_params (dict): Additional query parameters for the API request. - - Returns: - list[dict]: A team's GitHub Copilot metrics or None if an error occurs. - """ - response = gh.get(f"/orgs/{org}/team/{team}/copilot/metrics", params=query_params) - - if not isinstance(response, Response): - # If the response is not a Response object, no copilot data is available for this team - # We can return None which is then handled by the calling function - return None - return response.json() - - def get_dict_value(dictionary: dict, key: str) -> Any: """Gets a value from a dictionary and raises an exception if it is not found. @@ -394,19 +223,12 @@ def handler(event: dict, context) -> str: # pylint: disable=unused-argument, to features = get_dict_value(config, "features") - show_log_locally = get_dict_value(features, "show_log_locally") - write_data_locally = get_dict_value(features, "write_data_locally") - # Toggle local logging - if show_log_locally: - # This is a nightmare to test as it's really hard to get to. - # At some point we should look to make a wrapper for logging - # so it can be tested more easily. - logging.basicConfig( - filename="debug.log", - filemode="w", - ) + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + ) # Create an S3 client session = boto3.Session() @@ -437,53 +259,21 @@ def handler(event: dict, context) -> str: # pylint: disable=unused-argument, to # Copilot Usage Data (Historic) historic_usage, dates_added = get_and_update_historic_usage(s3, gh, write_data_locally) - # GitHub Teams with Copilot Data - copilot_teams = get_and_update_copilot_teams(s3, gh, write_data_locally) - - logger.info("Getting history of each team identified previously") - - # Retrieve existing team history from S3 - try: - response = s3.get_object(Bucket=BUCKET_NAME, Key="teams_history.json") - existing_team_history = json.loads(response["Body"].read().decode("utf-8")) - except ClientError as e: - logger.warning("Error retrieving existing team history: %s", e) - existing_team_history = [] - - logger.info("Existing team history has %d entries", len(existing_team_history)) - - if not write_data_locally: - # Convert to dictionary for quick lookup - updated_team_history = create_dictionary(gh, copilot_teams, existing_team_history) - - # Write updated team history to S3 - # This line isn't covered by tests as it's painful to get to. - # The function itself is tested though. - update_s3_object(s3, BUCKET_NAME, "teams_history.json", updated_team_history) - else: - local_path = "output/teams_history.json" - os.makedirs("output", exist_ok=True) - updated_team_history = create_dictionary(gh, copilot_teams, existing_team_history) - with open(local_path, "w", encoding="utf-8") as f: - json.dump(updated_team_history, f, indent=4) - logger.info("Team history written locally to %s (S3 skipped)", local_path) - logger.info( - "Process complete", + "Process finished", extra={ "bucket": BUCKET_NAME, "no_days_added": len(dates_added), "dates_added": dates_added, "no_dates_before": len(historic_usage) - len(dates_added), "no_dates_after": len(historic_usage), - "no_copilot_teams": len(copilot_teams), }, ) return "Github Data logging is now complete." -# # Dev Only -# # Uncomment the following line to run the script locally +# Dev Only +# Uncomment the following line to run the script locally # if __name__ == "__main__": # handler(None, None) diff --git a/tests/test_main.py b/tests/test_main.py index 628f57f..e34b6ba 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,6 +1,7 @@ import json import os from unittest.mock import MagicMock, call, patch +from io import BytesIO from botocore.exceptions import ClientError from requests import Response @@ -11,17 +12,28 @@ from src.main import ( BUCKET_NAME, - create_dictionary, - get_and_update_copilot_teams, get_and_update_historic_usage, - get_copilot_team_date, - get_team_history, handler, update_s3_object, get_dict_value, get_config_file, + ) +# Mock GitHub API response +api_response = { + "download_links": [ + "https://example.com/organisation_history_api_response.json" + ] + # There are other fields in the API response, but we don't need them for testing +} + +# Mock usage data fetched from download_links key in the API response +fetched_usage_data = {"day_totals": [ + {"day": "2024-01-01", "usage": 10}, + {"day": "2024-01-02", "usage": 20}, +]} + class TestUpdateS3Object: def test_update_s3_object_success(self, caplog): @@ -58,128 +70,13 @@ def test_update_s3_object_failure(self, caplog): assert any("Failed to update" in record.message for record in caplog.records) -class TestGetAndUpdateCopilotTeams: - @patch("src.main.update_s3_object") - def test_get_and_update_copilot_teams_single_page(self, mock_update_s3_object): - s3 = MagicMock() - gh = MagicMock() - # Mock response for first page - mock_response = MagicMock() - mock_response.links = {} # No 'last' link, so only one page - gh.get.return_value = mock_response - - # Patch get_copilot_team_date to return a list of teams - with patch( - "src.main.get_copilot_team_date", return_value=[{"name": "team1"}] - ) as mock_get_team_date: - result = get_and_update_copilot_teams(s3, gh, False) - assert result == [{"name": "team1"}] - mock_get_team_date.assert_called_once_with(gh, 1) - mock_update_s3_object.assert_called_once() - args, kwargs = mock_update_s3_object.call_args - assert args[1].endswith("copilot-usage-dashboard") - assert args[2] == "copilot_teams.json" - assert args[3] == [{"name": "team1"}] - - @patch("src.main.update_s3_object") - def test_get_and_update_copilot_teams_multiple_pages(self, mock_update_s3_object): - s3 = MagicMock() - gh = MagicMock() - # Mock response with 'last' link for 3 pages - mock_response = MagicMock() - mock_response.links = {"last": {"url": "https://api.github.com/orgs/test/teams?page=3"}} - gh.get.return_value = mock_response - - # Patch get_copilot_team_date to return different teams per page - with patch( - "src.main.get_copilot_team_date", - side_effect=[[{"name": "team1"}], [{"name": "team2"}], [{"name": "team3"}]], - ) as mock_get_team_date: - result = get_and_update_copilot_teams(s3, gh, False) - assert result == [{"name": "team1"}, {"name": "team2"}, {"name": "team3"}] - assert mock_get_team_date.call_count == 3 - mock_update_s3_object.assert_called_once() - - @patch("src.main.update_s3_object") - def test_get_and_update_copilot_teams_no_teams(self, mock_update_s3_object): - s3 = MagicMock() - gh = MagicMock() - mock_response = MagicMock() - mock_response.links = {} - gh.get.return_value = mock_response - - with patch("src.main.get_copilot_team_date", return_value=[]) as mock_get_team_date: - result = get_and_update_copilot_teams(s3, gh, False) - assert result == [] - mock_get_team_date.assert_called_once_with(gh, 1) - mock_update_s3_object.assert_called_once() - args, kwargs = mock_update_s3_object.call_args - assert args[1].endswith("copilot-usage-dashboard") - assert args[2] == "copilot_teams.json" - assert args[3] == [] - - def test_write_data_locally_creates_file(self, tmp_path): - s3 = MagicMock() - gh = MagicMock() - response = MagicMock() - response.links = {} - gh.get.return_value = response - - with patch("src.main.get_copilot_team_date", return_value=[{"name": "teamA"}]): - with patch("src.main.os.makedirs") as mock_makedirs, \ - patch("src.main.open", create=True) as mock_open: - result = get_and_update_copilot_teams(s3, gh, True) - assert result == [{"name": "teamA"}] - mock_makedirs.assert_called_once_with("output", exist_ok=True) - mock_open.assert_called_once() - s3.put_object.assert_not_called() - - -class TestGetTeamHistory: - def setup_method(self): - self.org_patch = patch("src.main.org", "test-org") - self.org_patch.start() - self.addCleanup = getattr(self, "addCleanup", lambda f: None) - - def teardown_method(self): - self.org_patch.stop() - - def test_get_team_history_success(self): - gh = MagicMock() - mock_response = MagicMock(spec=Response) - mock_response.json.return_value = [{"date": "2024-01-01", "usage": 5}] - gh.get.return_value = mock_response - - result = get_team_history(gh, "dev-team", {"since": "2024-01-01"}) - gh.get.assert_called_once_with( - "/orgs/test-org/team/dev-team/copilot/metrics", params={"since": "2024-01-01"} - ) - assert result == [{"date": "2024-01-01", "usage": 5}] - - def test_get_team_history_with_no_query_params(self): - gh = MagicMock() - mock_response = MagicMock(spec=Response) - mock_response.json.return_value = [] - gh.get.return_value = mock_response - - result = get_team_history(gh, "dev-team") - gh.get.assert_called_once_with("/orgs/test-org/team/dev-team/copilot/metrics", params=None) - assert result == [] - - class TestHandler: @patch("src.main.boto3.Session") @patch("src.main.github_api_toolkit.get_token_as_installation") @patch("src.main.github_api_toolkit.github_interface") @patch("src.main.get_and_update_historic_usage") - @patch("src.main.get_and_update_copilot_teams") - @patch("src.main.create_dictionary") - @patch("src.main.update_s3_object") def test_handler_success( self, - mock_update_s3_object, - mock_create_dictionary, - mock_get_and_update_copilot_teams, mock_get_and_update_historic_usage, mock_github_interface, mock_get_token_as_installation, @@ -199,21 +96,10 @@ def test_handler_success( mock_github_interface.return_value = mock_gh mock_get_and_update_historic_usage.return_value = (["usage1", "usage2"], ["2024-01-01"]) - mock_get_and_update_copilot_teams.return_value = [{"name": "team1"}] - mock_create_dictionary.return_value = [ - {"team": {"name": "team1"}, "data": [{"date": "2024-01-01"}]} - ] secret_region = "eu-west-1" secret_name = "test-secret" - # S3 get_object for teams_history.json returns existing history - mock_s3.get_object.return_value = { - "Body": MagicMock( - read=MagicMock(return_value=b'[{"team": {"name": "team1"}, "data": []}]') - ) - } - result = handler({}, MagicMock()) assert result == "Github Data logging is now complete." mock_boto3_session.assert_called_once() @@ -221,13 +107,8 @@ def test_handler_success( call("secretsmanager", region_name=secret_region) in mock_session.client.call_args_list mock_secret_manager.get_secret_value.assert_called_once_with(SecretId=secret_name) mock_get_token_as_installation.assert_called_once() - mock_github_interface.assert_called_once() - mock_get_and_update_historic_usage.assert_called_once() - mock_get_and_update_copilot_teams.assert_called_once() - mock_create_dictionary.assert_called_once() - mock_update_s3_object.assert_called_with( - mock_s3, BUCKET_NAME, "teams_history.json", mock_create_dictionary.return_value - ) + mock_github_interface.assert_called_once_with("token") + mock_get_and_update_historic_usage.assert_called_once_with(mock_s3, mock_gh, False) @patch("src.main.boto3.Session") @patch("src.main.github_api_toolkit.get_token_as_installation") @@ -246,110 +127,6 @@ def test_handler_access_token_error( assert result.startswith("Error getting access token:") assert any("Error getting access token" in record.getMessage() for record in caplog.records) - @patch("src.main.boto3.Session") - @patch("src.main.github_api_toolkit.get_token_as_installation") - @patch("src.main.github_api_toolkit.github_interface") - @patch("src.main.get_and_update_historic_usage") - @patch("src.main.get_and_update_copilot_teams") - @patch("src.main.create_dictionary") - @patch("src.main.update_s3_object") - def test_handler_team_history_client_error( - self, - mock_update_s3_object, - mock_create_dictionary, - mock_get_and_update_copilot_teams, - mock_get_and_update_historic_usage, - mock_github_interface, - mock_get_token_as_installation, - mock_boto3_session, - caplog, - ): - mock_s3 = MagicMock() - mock_secret_manager = MagicMock() - mock_session = MagicMock() - mock_session.client.side_effect = [mock_s3, mock_secret_manager] - mock_boto3_session.return_value = mock_session - - mock_secret_manager.get_secret_value.return_value = {"SecretString": "pem-content"} - mock_get_token_as_installation.return_value = ("token",) - mock_gh = MagicMock() - mock_github_interface.return_value = mock_gh - - mock_get_and_update_historic_usage.return_value = (["usage1"], ["2024-01-01"]) - mock_get_and_update_copilot_teams.return_value = [{"name": "team1"}] - mock_create_dictionary.return_value = [ - {"team": {"name": "team1"}, "data": [{"date": "2024-01-01"}]} - ] - - # S3 get_object for teams_history.json raises ClientError - mock_s3.get_object.side_effect = ClientError( - error_response={"Error": {"Code": "404", "Message": "Not Found"}}, - operation_name="GetObject", - ) - - result = handler({}, MagicMock()) - assert result == "Github Data logging is now complete." - assert any( - "Error retrieving existing team history" in record.getMessage() - for record in caplog.records - ) - mock_update_s3_object.assert_called_with( - mock_s3, BUCKET_NAME, "teams_history.json", mock_create_dictionary.return_value - ) - - -class TestGetCopilotTeamDate: - @patch("src.main.org", "test-org") - def test_get_copilot_team_date_success(self): - gh = MagicMock() - # Mock teams response - teams_response = MagicMock() - teams_response.json.return_value = [ - {"name": "team1", "slug": "slug1", "description": "desc1", "html_url": "url1"}, - {"name": "team2", "slug": "slug2", "description": "desc2", "html_url": "url2"}, - ] - gh.get.side_effect = [ - teams_response, - MagicMock(spec=Response), # usage_data for team1 - MagicMock(spec=Response), # usage_data for team2 - ] - - result = get_copilot_team_date(gh, 1) - assert result == [ - {"name": "team1", "slug": "slug1", "description": "desc1", "url": "url1"}, - {"name": "team2", "slug": "slug2", "description": "desc2", "url": "url2"}, - ] - gh.get.assert_any_call("/orgs/test-org/teams", params={"per_page": 100, "page": 1}) - gh.get.assert_any_call("/orgs/test-org/team/team1/copilot/metrics") - gh.get.assert_any_call("/orgs/test-org/team/team2/copilot/metrics") - - @patch("src.main.org", "test-org") - def test_get_copilot_team_date_unexpected_usage_response(self, caplog): - gh = MagicMock() - teams_response = MagicMock() - teams_response.json.return_value = [ - {"name": "team1", "slug": "slug1", "description": "desc1", "html_url": "url1"}, - ] - gh.get.side_effect = [ - teams_response, - "not_a_response", # usage_data for team1 - ] - - with caplog.at_level("ERROR"): - result = get_copilot_team_date(gh, 1) - assert result == [] - - @patch("src.main.org", "test-org") - def test_get_copilot_team_date_empty_teams(self): - gh = MagicMock() - teams_response = MagicMock() - teams_response.json.return_value = [] - gh.get.return_value = teams_response - - result = get_copilot_team_date(gh, 1) - assert result == [] - gh.get.assert_called_once_with("/orgs/test-org/teams", params={"per_page": 100, "page": 1}) - class TestGetAndUpdateHistoricUsage: def setup_method(self): @@ -362,39 +139,39 @@ def teardown_method(self): def test_get_and_update_historic_usage_success(self): s3 = MagicMock() gh = MagicMock() - # Mock usage data returned from GitHub API - usage_data = [ - {"date": "2024-01-01", "usage": 10}, - {"date": "2024-01-02", "usage": 20}, - ] - gh.get.return_value.json.return_value = usage_data + + gh.get.return_value.json.return_value = api_response # Mock S3 get_object returns existing historic usage with one date - existing_usage = [{"date": "2024-01-01", "usage": 10}] + existing_usage = [{"day": "2024-01-01", "usage": 10}] s3.get_object.return_value = { - "Body": MagicMock( - read=MagicMock(return_value=json.dumps(existing_usage).encode("utf-8")) - ) + "Body": BytesIO(json.dumps(existing_usage).encode("utf-8")) } - result, dates_added = get_and_update_historic_usage(s3, gh, False) + # Mock requests.get returns usage data from download_links + # We always patch dependencies imported inside the function we're testing. + # Test environment initialisation ends here. + with patch("src.main.get") as mock_get: + mock_get.return_value.json.return_value = fetched_usage_data + result, dates_added = get_and_update_historic_usage(s3, gh, False) + assert result == [ - {"date": "2024-01-01", "usage": 10}, - {"date": "2024-01-02", "usage": 20}, + {"day": "2024-01-01", "usage": 10}, + {"day": "2024-01-02", "usage": 20}, ] assert dates_added == ["2024-01-02"] s3.get_object.assert_called_once() s3.put_object.assert_called_once() args, kwargs = s3.put_object.call_args assert kwargs["Bucket"].endswith("copilot-usage-dashboard") - assert kwargs["Key"] == "historic_usage_data.json" + assert kwargs["Key"] == "organisation_history.json" assert json.loads(kwargs["Body"].decode("utf-8")) == result def test_get_and_update_historic_usage_no_existing_data(self, caplog): s3 = MagicMock() gh = MagicMock() - usage_data = [{"date": "2024-01-01", "usage": 10}] - gh.get.return_value.json.return_value = usage_data + + gh.get.return_value.json.return_value = api_response # S3 get_object raises ClientError s3.get_object.side_effect = ClientError( @@ -402,39 +179,42 @@ def test_get_and_update_historic_usage_no_existing_data(self, caplog): operation_name="GetObject", ) - result, dates_added = get_and_update_historic_usage(s3, gh, False) - assert result == [{"date": "2024-01-01", "usage": 10}] - assert dates_added == ["2024-01-01"] + with patch("src.main.get") as mock_get: + mock_get.return_value.json.return_value = fetched_usage_data + result, dates_added = get_and_update_historic_usage(s3, gh, False) + + assert result == [{"day": "2024-01-01", "usage": 10}, {"day": "2024-01-02", "usage": 20}] + assert dates_added == ["2024-01-01", "2024-01-02"] s3.put_object.assert_called_once() assert any( - "Error getting historic_usage_data.json" in record.getMessage() + "Error getting organisation_history.json" in record.getMessage() for record in caplog.records ) def test_get_and_update_historic_usage_no_new_dates(self): s3 = MagicMock() gh = MagicMock() - usage_data = [{"date": "2024-01-01", "usage": 10}] - gh.get.return_value.json.return_value = usage_data + + gh.get.return_value.json.return_value = api_response # S3 get_object returns same date as usage_data - existing_usage = [{"date": "2024-01-01", "usage": 10}] + existing_usage = [{"day": "2024-01-01", "usage": 10}, {"day": "2024-01-02", "usage": 20}] s3.get_object.return_value = { - "Body": MagicMock( - read=MagicMock(return_value=json.dumps(existing_usage).encode("utf-8")) - ) + "Body": BytesIO(json.dumps(existing_usage).encode("utf-8")) } + with patch("src.main.get") as mock_get: + mock_get.return_value.json.return_value = fetched_usage_data + result, dates_added = get_and_update_historic_usage(s3, gh, False) - result, dates_added = get_and_update_historic_usage(s3, gh, False) - assert result == [{"date": "2024-01-01", "usage": 10}] + assert result == [{"day": "2024-01-01", "usage": 10}, {"day": "2024-01-02", "usage": 20}] assert dates_added == [] s3.put_object.assert_called_once() def test_write_data_locally_creates_file(self, tmp_path): s3 = MagicMock() gh = MagicMock() - usage_data = [{"date": "2024-01-01", "usage": 10}] - gh.get.return_value.json.return_value = usage_data + + gh.get.return_value.json.return_value = api_response # S3 get_object raises ClientError s3.get_object.side_effect = ClientError( @@ -444,141 +224,15 @@ def test_write_data_locally_creates_file(self, tmp_path): # Patch os.makedirs and open to use tmp_path with patch("src.main.os.makedirs") as mock_makedirs, \ - patch("src.main.open", create=True) as mock_open: - result, dates_added = get_and_update_historic_usage(s3, gh, True) - assert result == [{"date": "2024-01-01", "usage": 10}] - assert dates_added == ["2024-01-01"] - mock_makedirs.assert_called_once_with("output", exist_ok=True) - mock_open.assert_called_once() - s3.put_object.assert_not_called() - - -class TestCreateDictionary: - def setup_method(self): - self.org_patch = patch("src.main.org", "test-org") - self.org_patch.start() - - def teardown_method(self): - self.org_patch.stop() - - def test_create_dictionary_adds_new_team_history(self): - gh = MagicMock() - copilot_teams = [{"name": "team1"}, {"name": "team2"}] - existing_team_history = [] - - # get_team_history returns history for each team - with patch( - "src.main.get_team_history", - side_effect=[ - [{"date": "2024-01-01", "usage": 5}], - [{"date": "2024-01-02", "usage": 10}], - ], - ) as mock_get_team_history: - result = create_dictionary(gh, copilot_teams, existing_team_history) - assert len(result) == 2 - assert result[0]["team"]["name"] == "team1" - assert result[0]["data"] == [{"date": "2024-01-01", "usage": 5}] - assert result[1]["team"]["name"] == "team2" - assert result[1]["data"] == [{"date": "2024-01-02", "usage": 10}] - assert mock_get_team_history.call_count == 2 - - def test_create_dictionary_extends_existing_team_history(self): - gh = MagicMock() - copilot_teams = [{"name": "team1"}] - existing_team_history = [ - {"team": {"name": "team1"}, "data": [{"date": "2024-01-01", "usage": 5}]} - ] - - # get_team_history returns new history for team1 - with patch( - "src.main.get_team_history", return_value=[{"date": "2024-01-02", "usage": 10}] - ) as mock_get_team_history: - result = create_dictionary(gh, copilot_teams, existing_team_history) - assert len(result) == 1 - assert result[0]["team"]["name"] == "team1" - assert result[0]["data"] == [ - {"date": "2024-01-01", "usage": 5}, - {"date": "2024-01-02", "usage": 10}, - ] - mock_get_team_history.assert_called_once() - - args, kwargs = mock_get_team_history.call_args - assert args[0] == gh - assert args[1] == "team1" - assert args[2] == {"since": "2024-01-01"} - - def test_create_dictionary_skips_team_with_no_name(self, caplog): - gh = MagicMock() - copilot_teams = [{"slug": "slug1"}] # No 'name' - existing_team_history = [] - - with patch("src.main.get_team_history") as mock_get_team_history: - result = create_dictionary(gh, copilot_teams, existing_team_history) - assert result == [] - assert mock_get_team_history.call_count == 0 - assert any( - "Skipping team with no name" in record.getMessage() for record in caplog.records - ) - - def test_create_dictionary_no_new_history(self, caplog): - gh = MagicMock() - copilot_teams = [{"name": "team1"}] - existing_team_history = [] - - # get_team_history returns empty list - with patch("src.main.get_team_history", return_value=[]) as mock_get_team_history: - result = create_dictionary(gh, copilot_teams, existing_team_history) - assert result == [] - assert mock_get_team_history.call_count == 1 - - -class TestGetTeamHistory: - def setup_method(self): - self.org_patch = patch("src.main.org", "test-org") - self.org_patch.start() - - def teardown_method(self): - self.org_patch.stop() - - def test_get_team_history_returns_metrics(self): - gh = MagicMock() - mock_response = MagicMock(spec=Response) - mock_response.json.return_value = [{"date": "2024-01-01", "usage": 5}] - gh.get.return_value = mock_response - - result = get_team_history(gh, "dev-team", {"since": "2024-01-01"}) - gh.get.assert_called_once_with( - "/orgs/test-org/team/dev-team/copilot/metrics", params={"since": "2024-01-01"} - ) - assert result == [{"date": "2024-01-01", "usage": 5}] - - def test_get_team_history_returns_empty_list(self): - gh = MagicMock() - mock_response = MagicMock(spec=Response) - mock_response.json.return_value = [] - gh.get.return_value = mock_response - - result = get_team_history(gh, "dev-team") - gh.get.assert_called_once_with("/orgs/test-org/team/dev-team/copilot/metrics", params=None) - assert result == [] - - def test_get_team_history_non_response_returns_none(self): - gh = MagicMock() - gh.get.return_value = "not_a_response" - - result = get_team_history(gh, "dev-team") - gh.get.assert_called_once_with("/orgs/test-org/team/dev-team/copilot/metrics", params=None) - assert result is None - - def test_get_team_history_with_query_params_none(self): - gh = MagicMock() - mock_response = MagicMock(spec=Response) - mock_response.json.return_value = [{"date": "2024-01-01", "usage": 5}] - gh.get.return_value = mock_response - - result = get_team_history(gh, "dev-team", None) - gh.get.assert_called_once_with("/orgs/test-org/team/dev-team/copilot/metrics", params=None) - assert result == [{"date": "2024-01-01", "usage": 5}] + patch("src.main.open", create=True) as mock_open, \ + patch("src.main.get") as mock_get: + mock_get.return_value.json.return_value = fetched_usage_data + result, dates_added = get_and_update_historic_usage(s3, gh, True) + assert result == [{"day": "2024-01-01", "usage": 10}, {"day": "2024-01-02", "usage": 20}] + assert dates_added == ["2024-01-01", "2024-01-02"] + mock_makedirs.assert_called_once_with("output", exist_ok=True) + mock_open.assert_called_once() + s3.put_object.assert_not_called() class TestGetDictValue: