Keh 735 add team info (#54)

* Update to remove nan rows in the dataset
* Add functionality to get team history from the Github API
* add documentation of the team historic metrics
* add support for query params - since
* update docs

Author: mwirikia

docs/team_usage.md

@@ -23,3 +23,46 @@ A user within ONSDigital. Upon authentication, the app identifies the teams they
 
 #### Admin User
 An enhanced regular user with the ability to search for any team. This user belongs to a specific whitelisted team, enabling them to view metrics for any team that meets the CoPilot usage data requirements.
+
+## Metrics
+
+### Team History Metrics
+The team history metrics function retrieves historical usage data for each team identified with CoPilot usage. This data includes detailed metrics about the team's activity over time. New data for a team is fetched only from the last captured date in the file.
+
+#### Functionality
+- **Input**: The function in addition to the GitHub Client takes a team name, organisation and the optional "since" as a query parameter as input.
+- **Process**: 
+  - Fetches historical data for the specified team using the GitHub API.
+  - If the since query parameter exist then fetch data only after the specified date.
+  - Filters and organizes the data into a structured format.
+- **Output**: A JSON object containing the team's historical metrics, including:
+  - Team name
+  - Activity data
+  - CoPilot usage statistics
+
+#### Usage
+The historical metrics are stored in an S3 bucket as a json file (`teams_history.json`).
+
+#### Example
+For a team named `kehdev`, the historical metrics might include:
+```json
+{
+  "team": {
+    "name": "kehdev",
+    "slug": "kehdev",
+    "description": "Team responsible for CI/CD pipelines",
+    "url": "https://github.com/orgs/<organisation>/teams/kehdev"
+  },
+  "data": [
+    {
+      "date": "2025-07-01",
+      "active_members": 10,
+      "copilot_usage_hours": 50
+    },
+    {
+      "date": "2025-07-02",
+      "active_members": 12,
+      "copilot_usage_hours": 60
+    }
+  ]
+}

lambda_data_logger/main.py

@@ -194,3 +194,91 @@ def handler(event, context):
             "no_copilot_teams": len(copilot_teams),
         },
     )
+
+    # Get teams history
+    team_history = []
+    
+    logger.info("Getting history of each team identified previously")
+
+    # Retrieve existing team history from S3
+    try:
+        response = s3.get_object(Bucket=bucket_name, Key="teams_history.json")
+        existing_team_history = json.loads(response["Body"].read().decode("utf-8"))
+    except ClientError as e:
+        logger.warning(f"Error retrieving existing team history: {e}")
+        existing_team_history = []
+
+    logger.info(f"Existing team history has {len(existing_team_history)} entries")
+
+    # Create a dictionary for quick lookup of existing team data using the `name` field
+    existing_team_data_map = {single_team["team"]["name"]: single_team for single_team in existing_team_history}
+
+    # Iterate through identified teams
+    for team in copilot_teams:
+        team_name = team.get("name", "")
+        if not team_name:
+            logger.warning("Skipping team with no name")
+            continue
+
+        # Determine the last known date for the team
+        last_known_date = None
+        if team_name in existing_team_data_map:
+            existing_dates = [entry["date"] for entry in existing_team_data_map[team_name]["data"]]
+            if existing_dates:
+                last_known_date = max(existing_dates)  # Get the most recent date
+
+        # Assign the last know date to the `since` query parameter
+        query_params = {}
+        if last_known_date:
+            query_params["since"] = last_known_date
+
+        single_team_history = get_team_history(gh, org, team_name, query_params)
+        if not single_team_history:
+            logger.info(f"No new history found for team {team_name}")
+            continue
+
+        # Append new data to the existing team history
+        new_team_data = single_team_history
+        if team_name in existing_team_data_map:
+            existing_team_data_map[team_name]["data"].extend(new_team_data)
+        else:
+            existing_team_data_map[team_name] = {"team": team, "data": new_team_data}
+
+    # Convert the updated team data map back to a list
+    updated_team_history = list(existing_team_data_map.values())
+
+    # Write updated team history to S3
+    s3.put_object(
+        Bucket=bucket_name,
+        Key="teams_history.json",
+        Body=json.dumps(updated_team_history, indent=4).encode("utf-8"),
+    )
+
+    logger.info("Uploaded updated teams_history.json to S3")
+
+    return "Github Data logging is now complete."
+
+
+def get_team_history(gh: github_api_toolkit.github_interface, org: str, team: str, query_params: dict = None):
+    """
+    Gets the team metrics Copilot data through the API.
+    Note - This endpoint will only return results for a given day if the team had
+    five or more members with active Copilot licenses on that day,
+    as evaluated at the end of that day.
+
+    Args:
+        gh (github_api_toolkit.github_interface): An instance of the github_interface class.
+        org (str): Organisation name.
+        team (str): Team name.
+        query_params (dict): Additional query parameters for the API request.
+
+    Returns:
+        json: A json of team's GitHub team metrics or None if an error occurs.
+    """
+    try:
+        response = gh.get(f"/orgs/{org}/team/{team}/copilot/metrics", params=query_params)
+        return response.json()
+    except Exception as e:
+        logger.error(f"Error getting history for team {team} due to {e} with Github API")
+        return None
+    

lambda_data_logger/requirements.txt

@@ -4,7 +4,7 @@ certifi==2024.7.4
 cffi==1.16.0
 charset-normalizer==3.3.2
 cryptography==42.0.8
-github-api-toolkit @ git+https://github.com/ONS-Innovation/github-api-package.git@v1.0.0
+github-api-toolkit @ git+https://github.com/ONS-Innovation/github-api-package@v2.0.3
 idna==3.7
 jmespath==1.0.1
 jwt==1.3.1

src/pages/interim_page.py

@@ -97,7 +97,7 @@ def generate_datasets(date_range: tuple):
     # Create a subset of data based on slider selection
     df_usage_data_subset = df_usage_data.loc[
         (df_usage_data["date"] >= date_range[0]) & (df_usage_data["date"] <= date_range[1])
-    ].reset_index(drop=True)
+    ].reset_index(drop=True).dropna()
 
     copilot_chat = pd.DataFrame()
     ide_completions = pd.DataFrame()
@@ -106,7 +106,6 @@ def generate_datasets(date_range: tuple):
 
         # Get copilot chat data
         editors_data = df_usage_data_subset.iloc[i]["copilot_ide_chat.editors"]
-
         for editor in editors_data:
             editor_name = editor.get("name", "")
             models_data = editor.get("models", [])
@@ -122,7 +121,6 @@ def generate_datasets(date_range: tuple):
                 }])
                 copilot_chat = pd.concat([copilot_chat, data], ignore_index=True)
 
-
         # Get IDE completions data
         editors_data = df_usage_data_subset.iloc[i]["copilot_ide_code_completions.editors"]
 
@@ -364,12 +362,24 @@ def generate_datasets(date_range: tuple):
         # Format into a year format (i.e 2022)
         df_historic_data["date"] = df_historic_data["date"].dt.strftime("%Y")
 
+    # Remove any rows where the data is NAN
+    initial_historical_data = historic_data
+    indexes = []
+    for day in initial_historical_data:
+        try:
+            editor = day["copilot_ide_chat"]["editors"]
+        except KeyError:
+            indexes.append(initial_historical_data.index(day))
+
+    for index in indexes:
+        initial_historical_data.pop(index)
+
     # Extract IDE chat data
     df_chat = pd.json_normalize(
-        historic_data,
+        initial_historical_data,
         record_path=["copilot_ide_chat", "editors", "models"],
         meta=["date"],
-        errors="ignore"
+        errors="ignore" 
     )
     df_chat["date"] = pd.to_datetime(df_chat["date"]).dt.strftime(
     "%Y-%m-%d" if date_grouping == "Day" else