From 31003d363491aa24cc1c5de51b01bd353ca5659b Mon Sep 17 00:00:00 2001 From: Pankaj Chaudhari Date: Wed, 29 Jul 2026 11:11:37 +0000 Subject: [PATCH 1/3] feat(gh-copilot): add AI credit usage billing metrics Adds collection and extraction of GitHub Copilot AI credit usage billing metrics at org, user, and enterprise levels, including new models, tasks (ai_credit_collector/extractor), a migration for the new billing tables, and e2e snapshot fixtures. Ports the work from apache/devlake#8980. Co-authored-by: Andrei Savu <54935810+AndreiS-gh@users.noreply.github.com> Signed-off-by: Pankaj Chaudhari --- ...ool_copilot_enterprise_ai_credit_usage.csv | 4 + .../_tool_copilot_org_ai_credit_usage.csv | 4 + .../_tool_copilot_user_ai_credit_usage.csv | 4 + .../models/enterprise_ai_credit_usage.go | 56 +++++ .../20260708_add_ai_credit_usage_metrics.go | 45 ++++ .../models/migrationscripts/register.go | 1 + backend/plugins/gh-copilot/models/models.go | 4 + .../plugins/gh-copilot/models/models_test.go | 3 + .../gh-copilot/models/org_ai_credit_usage.go | 53 +++++ .../gh-copilot/models/user_ai_credit_usage.go | 52 +++++ .../plugins/gh-copilot/models/user_metrics.go | 1 + .../gh-copilot/tasks/ai_credit_collector.go | 123 +++++++++++ .../gh-copilot/tasks/ai_credit_extractor.go | 194 ++++++++++++++++++ backend/plugins/gh-copilot/tasks/register.go | 2 + backend/plugins/gh-copilot/tasks/subtasks.go | 17 ++ .../tasks/user_metrics_extractor.go | 2 + 16 files changed, 565 insertions(+) create mode 100644 backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_enterprise_ai_credit_usage.csv create mode 100644 backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_org_ai_credit_usage.csv create mode 100644 backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_user_ai_credit_usage.csv create mode 100644 backend/plugins/gh-copilot/models/enterprise_ai_credit_usage.go create mode 100644 backend/plugins/gh-copilot/models/migrationscripts/20260708_add_ai_credit_usage_metrics.go create mode 100644 backend/plugins/gh-copilot/models/org_ai_credit_usage.go create mode 100644 backend/plugins/gh-copilot/models/user_ai_credit_usage.go create mode 100644 backend/plugins/gh-copilot/tasks/ai_credit_collector.go create mode 100644 backend/plugins/gh-copilot/tasks/ai_credit_extractor.go diff --git a/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_enterprise_ai_credit_usage.csv b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_enterprise_ai_credit_usage.csv new file mode 100644 index 00000000000..d9e0e72c229 --- /dev/null +++ b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_enterprise_ai_credit_usage.csv @@ -0,0 +1,4 @@ +connection_id,scope_id,year,month,day,enterprise,model,organization,user,product,cost_center_id,cost_center_name,gross_quantity,discount_quantity,net_quantity,price_per_unit,gross_amount,discount_amount,net_amount +1,octodemo,2025,12,10,octodemo,gpt-4.1,,,copilot,,,100.5,10.0,90.5,1.0,100.5,10.0,90.5 +1,octodemo,2025,12,10,octodemo,gpt-4o,,,copilot,cc-eng,Engineering,50.0,5.0,45.0,1.0,50.0,5.0,45.0 +1,octodemo,2025,12,10,octodemo,claude-3,,,copilot,cc-ml,Machine Learning,30.0,2.0,28.0,1.0,30.0,2.0,28.0 diff --git a/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_org_ai_credit_usage.csv b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_org_ai_credit_usage.csv new file mode 100644 index 00000000000..caf4f77cfed --- /dev/null +++ b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_org_ai_credit_usage.csv @@ -0,0 +1,4 @@ +connection_id,scope_id,year,month,day,organization,model,user,product,gross_quantity,discount_quantity,net_quantity,price_per_unit,gross_amount,discount_amount,net_amount +1,octodemo,2025,12,10,octodemo,gpt-4.1,alice,copilot,50.0,5.0,45.0,1.0,50.0,5.0,45.0 +1,octodemo,2025,12,10,octodemo,gpt-4o,bob,copilot,30.0,3.0,27.0,1.0,30.0,3.0,27.0 +1,octodemo,2025,12,10,octodemo,claude-3,charlie,copilot,20.0,0.0,20.0,1.0,20.0,0.0,20.0 diff --git a/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_user_ai_credit_usage.csv b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_user_ai_credit_usage.csv new file mode 100644 index 00000000000..7e098273481 --- /dev/null +++ b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_user_ai_credit_usage.csv @@ -0,0 +1,4 @@ +connection_id,scope_id,year,month,day,user,model,product,gross_quantity,discount_quantity,net_quantity,price_per_unit,gross_amount,discount_amount,net_amount +1,octodemo,2025,12,10,alice,gpt-4.1,copilot,50.0,5.0,45.0,1.0,50.0,5.0,45.0 +1,octodemo,2025,12,10,alice,gpt-4o,copilot,30.0,3.0,27.0,1.0,30.0,3.0,27.0 +1,octodemo,2025,12,10,bob,claude-3,copilot,20.0,2.0,18.0,1.0,20.0,2.0,18.0 diff --git a/backend/plugins/gh-copilot/models/enterprise_ai_credit_usage.go b/backend/plugins/gh-copilot/models/enterprise_ai_credit_usage.go new file mode 100644 index 00000000000..c10814000ce --- /dev/null +++ b/backend/plugins/gh-copilot/models/enterprise_ai_credit_usage.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// GhCopilotEnterpriseAiCreditUsage tracks AI credit consumption at the enterprise level. +// One row per time period per model per entity (user, org, or cost center). +type GhCopilotEnterpriseAiCreditUsage struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(191)" json:"scopeId"` + Year int `gorm:"primaryKey" json:"year"` + Month int `gorm:"primaryKey" json:"month"` + Day int `gorm:"primaryKey" json:"day"` + + Enterprise string `gorm:"primaryKey;type:varchar(191);comment:Enterprise slug" json:"enterprise"` + Model string `gorm:"primaryKey;type:varchar(191);comment:AI model name (e.g., gpt-4.1)" json:"model"` + Organization string `gorm:"index;type:varchar(255);comment:Organization within enterprise, if specified" json:"organization"` + User string `gorm:"index;type:varchar(255);comment:Username, if specified" json:"user"` + + Product string `gorm:"type:varchar(32);comment:Product name (e.g., copilot)" json:"product"` + CostCenterId string `gorm:"index;type:varchar(255);comment:Cost center identifier" json:"costCenterId"` + CostCenterName string `gorm:"type:varchar(255);comment:Cost center display name" json:"costCenterName"` + + // Credit usage breakdown + GrossQuantity float64 `json:"grossQuantity" gorm:"comment:Raw credits consumed"` + DiscountQuantity float64 `json:"discountQuantity" gorm:"comment:Credits discounted"` + NetQuantity float64 `json:"netQuantity" gorm:"comment:Credits after discount"` + PricePerUnit float64 `json:"pricePerUnit" gorm:"comment:Price per credit unit"` + GrossAmount float64 `json:"grossAmount" gorm:"comment:Gross cost before discount"` + DiscountAmount float64 `json:"discountAmount" gorm:"comment:Discount amount"` + NetAmount float64 `json:"netAmount" gorm:"comment:Net cost after discount"` + + common.NoPKModel +} + +func (GhCopilotEnterpriseAiCreditUsage) TableName() string { + return "_tool_copilot_enterprise_ai_credit_usage" +} diff --git a/backend/plugins/gh-copilot/models/migrationscripts/20260708_add_ai_credit_usage_metrics.go b/backend/plugins/gh-copilot/models/migrationscripts/20260708_add_ai_credit_usage_metrics.go new file mode 100644 index 00000000000..6b52dd0940a --- /dev/null +++ b/backend/plugins/gh-copilot/models/migrationscripts/20260708_add_ai_credit_usage_metrics.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" +) + +type addAiCreditUsageMetrics struct{} + +func (u *addAiCreditUsageMetrics) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &models.GhCopilotEnterpriseAiCreditUsage{}, + &models.GhCopilotOrgAiCreditUsage{}, + &models.GhCopilotUserAiCreditUsage{}, + &models.GhCopilotUserDailyMetrics{}, + ) +} + +func (u *addAiCreditUsageMetrics) Version() uint64 { + return 20260708000000 +} + +func (u *addAiCreditUsageMetrics) Name() string { + return "add AI credit usage billing tables" +} diff --git a/backend/plugins/gh-copilot/models/migrationscripts/register.go b/backend/plugins/gh-copilot/models/migrationscripts/register.go index 399735695e0..6bf50a37193 100644 --- a/backend/plugins/gh-copilot/models/migrationscripts/register.go +++ b/backend/plugins/gh-copilot/models/migrationscripts/register.go @@ -31,5 +31,6 @@ func All() []plugin.MigrationScript { new(addPRFieldsToEnterpriseMetrics), new(addOrganizationIdToUserMetrics), new(addCopilotMetricsGaps), + new(addAiCreditUsageMetrics), } } diff --git a/backend/plugins/gh-copilot/models/models.go b/backend/plugins/gh-copilot/models/models.go index 5143ce5f8b7..814e45f4a51 100644 --- a/backend/plugins/gh-copilot/models/models.go +++ b/backend/plugins/gh-copilot/models/models.go @@ -47,5 +47,9 @@ func GetTablesInfo() []dal.Tabler { &GhCopilotSeat{}, // User-team mappings &GhCopilotUserTeam{}, + // AI credit usage billing (enterprise, org, user levels) + &GhCopilotEnterpriseAiCreditUsage{}, + &GhCopilotOrgAiCreditUsage{}, + &GhCopilotUserAiCreditUsage{}, } } diff --git a/backend/plugins/gh-copilot/models/models_test.go b/backend/plugins/gh-copilot/models/models_test.go index ef5b3eff6f5..7e9a2f2e979 100644 --- a/backend/plugins/gh-copilot/models/models_test.go +++ b/backend/plugins/gh-copilot/models/models_test.go @@ -41,6 +41,9 @@ func TestGetTablesInfo(t *testing.T) { (&GhCopilotUserMetricsByModelFeature{}).TableName(): false, (&GhCopilotSeat{}).TableName(): false, (&GhCopilotUserTeam{}).TableName(): false, + (&GhCopilotEnterpriseAiCreditUsage{}).TableName(): false, + (&GhCopilotOrgAiCreditUsage{}).TableName(): false, + (&GhCopilotUserAiCreditUsage{}).TableName(): false, } if len(tables) != len(expected) { diff --git a/backend/plugins/gh-copilot/models/org_ai_credit_usage.go b/backend/plugins/gh-copilot/models/org_ai_credit_usage.go new file mode 100644 index 00000000000..2e079f88f4d --- /dev/null +++ b/backend/plugins/gh-copilot/models/org_ai_credit_usage.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// GhCopilotOrgAiCreditUsage tracks AI credit consumption at the organization level. +// One row per time period per model per user (within the organization). +type GhCopilotOrgAiCreditUsage struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(191)" json:"scopeId"` + Year int `gorm:"primaryKey" json:"year"` + Month int `gorm:"primaryKey" json:"month"` + Day int `gorm:"primaryKey" json:"day"` + + Organization string `gorm:"primaryKey;type:varchar(191);comment:Organization name" json:"organization"` + Model string `gorm:"primaryKey;type:varchar(191);comment:AI model name (e.g., gpt-4.1)" json:"model"` + User string `gorm:"index;type:varchar(255);comment:Username consuming the credits" json:"user"` + + Product string `gorm:"type:varchar(32);comment:Product name (e.g., copilot)" json:"product"` + + // Credit usage breakdown + GrossQuantity float64 `json:"grossQuantity" gorm:"comment:Raw credits consumed"` + DiscountQuantity float64 `json:"discountQuantity" gorm:"comment:Credits discounted"` + NetQuantity float64 `json:"netQuantity" gorm:"comment:Credits after discount"` + PricePerUnit float64 `json:"pricePerUnit" gorm:"comment:Price per credit unit"` + GrossAmount float64 `json:"grossAmount" gorm:"comment:Gross cost before discount"` + DiscountAmount float64 `json:"discountAmount" gorm:"comment:Discount amount"` + NetAmount float64 `json:"netAmount" gorm:"comment:Net cost after discount"` + + common.NoPKModel +} + +func (GhCopilotOrgAiCreditUsage) TableName() string { + return "_tool_copilot_org_ai_credit_usage" +} diff --git a/backend/plugins/gh-copilot/models/user_ai_credit_usage.go b/backend/plugins/gh-copilot/models/user_ai_credit_usage.go new file mode 100644 index 00000000000..cb1ef2a6089 --- /dev/null +++ b/backend/plugins/gh-copilot/models/user_ai_credit_usage.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// GhCopilotUserAiCreditUsage tracks AI credit consumption at the individual user level. +// One row per time period per model per authenticated user. +type GhCopilotUserAiCreditUsage struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(191)" json:"scopeId"` + Year int `gorm:"primaryKey" json:"year"` + Month int `gorm:"primaryKey" json:"month"` + Day int `gorm:"primaryKey" json:"day"` + + User string `gorm:"primaryKey;type:varchar(191);comment:GitHub username" json:"user"` + Model string `gorm:"primaryKey;type:varchar(191);comment:AI model name (e.g., gpt-4.1)" json:"model"` + + Product string `gorm:"type:varchar(32);comment:Product name (e.g., copilot)" json:"product"` + + // Credit usage breakdown + GrossQuantity float64 `json:"grossQuantity" gorm:"comment:Raw credits consumed"` + DiscountQuantity float64 `json:"discountQuantity" gorm:"comment:Credits discounted"` + NetQuantity float64 `json:"netQuantity" gorm:"comment:Credits after discount"` + PricePerUnit float64 `json:"pricePerUnit" gorm:"comment:Price per credit unit"` + GrossAmount float64 `json:"grossAmount" gorm:"comment:Gross cost before discount"` + DiscountAmount float64 `json:"discountAmount" gorm:"comment:Discount amount"` + NetAmount float64 `json:"netAmount" gorm:"comment:Net cost after discount"` + + common.NoPKModel +} + +func (GhCopilotUserAiCreditUsage) TableName() string { + return "_tool_copilot_user_ai_credit_usage" +} diff --git a/backend/plugins/gh-copilot/models/user_metrics.go b/backend/plugins/gh-copilot/models/user_metrics.go index 18e9134c226..ad4ce83551d 100644 --- a/backend/plugins/gh-copilot/models/user_metrics.go +++ b/backend/plugins/gh-copilot/models/user_metrics.go @@ -38,6 +38,7 @@ type GhCopilotUserDailyMetrics struct { UsedCli bool `json:"usedCli" gorm:"comment:Whether user used Copilot CLI"` UsedCopilotCodeReviewActive bool `json:"usedCopilotCodeReviewActive" gorm:"comment:Whether user actively used code review"` UsedCopilotCodeReviewPassive bool `json:"usedCopilotCodeReviewPassive" gorm:"comment:Whether user passively used code review"` + AiCreditsUsed float64 `json:"aiCreditsUsed" gorm:"comment:AI credits consumed on this day"` CopilotActivityMetrics `mapstructure:",squash"` CopilotCliMetrics `mapstructure:",squash"` diff --git a/backend/plugins/gh-copilot/tasks/ai_credit_collector.go b/backend/plugins/gh-copilot/tasks/ai_credit_collector.go new file mode 100644 index 00000000000..ae9d5fdf0bb --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/ai_credit_collector.go @@ -0,0 +1,123 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +the ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const rawAiCreditUsageTable = "copilot_ai_credit_usage" + +func CollectAiCreditUsage(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*GhCopilotTaskData) + if !ok { + return errors.Default.New("task data is not GhCopilotTaskData") + } + connection := data.Connection + connection.Normalize() + + apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + if err != nil { + return err + } + + var urlTemplate string + var scope string + + if connection.HasEnterprise() { + urlTemplate = fmt.Sprintf("enterprises/%s/settings/billing/ai_credit/usage", connection.Enterprise) + scope = connection.Enterprise + } else if connection.Organization != "" { + urlTemplate = fmt.Sprintf("organizations/%s/settings/billing/ai_credit/usage", connection.Organization) + scope = connection.Organization + } else { + urlTemplate = "user/settings/billing/ai_credit/usage" + scope = "user" + } + + rawArgs := helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawAiCreditUsageTable, + Options: copilotRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: connection.Endpoint, + }, + } + + collector, err := helper.NewStatefulApiCollector(rawArgs) + if err != nil { + return err + } + + now := time.Now().UTC() + start, until := computeReportDateRange(now, collector.GetSince()) + start = clampDailyMetricsStartForBackfill(start, until) + + dayIter := newDayIterator(start, until) + + err = collector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: apiClient, + Input: dayIter, + UrlTemplate: urlTemplate, + Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { + input := reqData.Input.(*dayInput) + day, parseErr := time.Parse("2006-01-02", input.Day) + if parseErr != nil { + return nil, errors.Convert(parseErr) + } + q := url.Values{} + q.Set("year", strconv.Itoa(day.Year())) + q.Set("month", strconv.Itoa(int(day.Month()))) + q.Set("day", strconv.Itoa(day.Day())) + return q, nil + }, + Incremental: true, + Concurrency: 1, + AfterResponse: ignoreNoContent, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + if res.StatusCode != http.StatusOK { + return nil, errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("failed to collect AI credit usage for %s", scope)) + } + + var response struct { + UsageItems []json.RawMessage `json:"usageItems"` + } + if unmErr := helper.UnmarshalResponse(res, &response); unmErr != nil { + return nil, unmErr + } + + return response.UsageItems, nil + }, + }) + if err != nil { + return err + } + + return collector.Execute() +} diff --git a/backend/plugins/gh-copilot/tasks/ai_credit_extractor.go b/backend/plugins/gh-copilot/tasks/ai_credit_extractor.go new file mode 100644 index 00000000000..5cca80a5dcd --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/ai_credit_extractor.go @@ -0,0 +1,194 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +the ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" +) + +// aiCreditUsageRecord represents a single usage item from the AI credit usage API. +type aiCreditUsageRecord struct { + Product string `json:"product"` + Sku string `json:"sku"` + Model string `json:"model"` + UnitType string `json:"unitType"` + PricePerUnit float64 `json:"pricePerUnit"` + GrossQuantity float64 `json:"grossQuantity"` + DiscountQuantity float64 `json:"discountQuantity"` + NetQuantity float64 `json:"netQuantity"` + GrossAmount float64 `json:"grossAmount"` + DiscountAmount float64 `json:"discountAmount"` + NetAmount float64 `json:"netAmount"` +} + +// aiCreditResponseWrapper represents the wrapper around the API response containing time period and usage items. +type aiCreditResponseWrapper struct { + TimePeriod struct { + Year int `json:"year"` + Month int `json:"month"` + Day int `json:"day"` + } `json:"timePeriod"` + Enterprise string `json:"enterprise"` + Organization string `json:"organization"` + User string `json:"user"` + Product string `json:"product"` + Model string `json:"model"` + CostCenter struct { + Id string `json:"id"` + Name string `json:"name"` + } `json:"costCenter"` +} + +// ExtractAiCreditUsage parses AI credit usage records into the appropriate model tables. +func ExtractAiCreditUsage(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*GhCopilotTaskData) + if !ok { + return errors.Default.New("task data is not GhCopilotTaskData") + } + connection := data.Connection + connection.Normalize() + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawAiCreditUsageTable, + Options: copilotRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: connection.Endpoint, + }, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + // Parse raw data + var record aiCreditUsageRecord + err := json.Unmarshal(row.Data, &record) + if err != nil { + return nil, errors.Convert(err) + } + + // Extract wrapper info from row context + var wrapper aiCreditResponseWrapper + if connection.HasEnterprise() { + wrapper.Enterprise = connection.Enterprise + } else if connection.Organization != "" { + wrapper.Organization = connection.Organization + } + wrapper.Product = record.Product + wrapper.Model = record.Model + + // Derive the time period from the collector's day input so records are + // deterministic and aligned with the requested billing day, rather than + // depending on the extraction-time clock. + var input dayInput + if len(row.Input) > 0 { + if unmErr := json.Unmarshal(row.Input, &input); unmErr != nil { + return nil, errors.Convert(unmErr) + } + } + day, parseErr := time.Parse("2006-01-02", input.Day) + if parseErr != nil { + return nil, errors.Convert(parseErr) + } + wrapper.TimePeriod.Year = day.Year() + wrapper.TimePeriod.Month = int(day.Month()) + wrapper.TimePeriod.Day = day.Day() + + var results []interface{} + + // Route to appropriate table based on connection type + if connection.HasEnterprise() { + toolRecord := &models.GhCopilotEnterpriseAiCreditUsage{ + ConnectionId: data.Connection.ID, + ScopeId: data.Options.ScopeId, + Year: wrapper.TimePeriod.Year, + Month: wrapper.TimePeriod.Month, + Day: wrapper.TimePeriod.Day, + Enterprise: wrapper.Enterprise, + Model: record.Model, + Organization: wrapper.Organization, + User: wrapper.User, + Product: record.Product, + CostCenterId: wrapper.CostCenter.Id, + CostCenterName: wrapper.CostCenter.Name, + GrossQuantity: record.GrossQuantity, + DiscountQuantity: record.DiscountQuantity, + NetQuantity: record.NetQuantity, + PricePerUnit: record.PricePerUnit, + GrossAmount: record.GrossAmount, + DiscountAmount: record.DiscountAmount, + NetAmount: record.NetAmount, + } + results = append(results, toolRecord) + } else if connection.Organization != "" { + toolRecord := &models.GhCopilotOrgAiCreditUsage{ + ConnectionId: data.Connection.ID, + ScopeId: data.Options.ScopeId, + Year: wrapper.TimePeriod.Year, + Month: wrapper.TimePeriod.Month, + Day: wrapper.TimePeriod.Day, + Organization: wrapper.Organization, + Model: record.Model, + User: wrapper.User, + Product: record.Product, + GrossQuantity: record.GrossQuantity, + DiscountQuantity: record.DiscountQuantity, + NetQuantity: record.NetQuantity, + PricePerUnit: record.PricePerUnit, + GrossAmount: record.GrossAmount, + DiscountAmount: record.DiscountAmount, + NetAmount: record.NetAmount, + } + results = append(results, toolRecord) + } else { + // User-level credits + toolRecord := &models.GhCopilotUserAiCreditUsage{ + ConnectionId: data.Connection.ID, + ScopeId: data.Options.ScopeId, + Year: wrapper.TimePeriod.Year, + Month: wrapper.TimePeriod.Month, + Day: wrapper.TimePeriod.Day, + User: connection.Name, // Use connection display name + Model: record.Model, + Product: record.Product, + GrossQuantity: record.GrossQuantity, + DiscountQuantity: record.DiscountQuantity, + NetQuantity: record.NetQuantity, + PricePerUnit: record.PricePerUnit, + GrossAmount: record.GrossAmount, + DiscountAmount: record.DiscountAmount, + NetAmount: record.NetAmount, + } + results = append(results, toolRecord) + } + + return results, nil + }, + }) + if err != nil { + return err + } + + return extractor.Execute() +} diff --git a/backend/plugins/gh-copilot/tasks/register.go b/backend/plugins/gh-copilot/tasks/register.go index 3c7e5b1eeb9..23d592304e9 100644 --- a/backend/plugins/gh-copilot/tasks/register.go +++ b/backend/plugins/gh-copilot/tasks/register.go @@ -28,11 +28,13 @@ func GetSubTaskMetas() []plugin.SubTaskMeta { CollectEnterpriseMetricsMeta, CollectUserMetricsMeta, CollectUserTeamsMeta, + CollectAiCreditUsageMeta, // Extractors ExtractSeatsMeta, ExtractOrgMetricsMeta, ExtractEnterpriseMetricsMeta, ExtractUserMetricsMeta, ExtractUserTeamsMeta, + ExtractAiCreditUsageMeta, } } diff --git a/backend/plugins/gh-copilot/tasks/subtasks.go b/backend/plugins/gh-copilot/tasks/subtasks.go index 61ed5799525..69255287026 100644 --- a/backend/plugins/gh-copilot/tasks/subtasks.go +++ b/backend/plugins/gh-copilot/tasks/subtasks.go @@ -105,3 +105,20 @@ var ExtractUserTeamsMeta = plugin.SubTaskMeta{ Description: "Extract Copilot user-team mappings into tool-layer table", Dependencies: []*plugin.SubTaskMeta{&CollectUserTeamsMeta}, } + +var CollectAiCreditUsageMeta = plugin.SubTaskMeta{ + Name: "collectAiCreditUsage", + EntryPoint: CollectAiCreditUsage, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Collect GitHub Copilot AI credit usage billing data from billing API", +} + +var ExtractAiCreditUsageMeta = plugin.SubTaskMeta{ + Name: "extractAiCreditUsage", + EntryPoint: ExtractAiCreditUsage, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Extract Copilot AI credit usage into tool-layer billing tables", + Dependencies: []*plugin.SubTaskMeta{&CollectAiCreditUsageMeta}, +} diff --git a/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go b/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go index 72992194063..7671fab86a9 100644 --- a/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go +++ b/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go @@ -49,6 +49,7 @@ type userDailyReport struct { UsedCli bool `json:"used_cli"` UsedCopilotCodeReviewActive bool `json:"used_copilot_code_review_active"` UsedCopilotCodeReviewPassive bool `json:"used_copilot_code_review_passive"` + AiCreditsUsed float64 `json:"ai_credits_used"` TotalsByIde []userTotalsByIde `json:"totals_by_ide"` TotalsByFeature []totalsByFeature `json:"totals_by_feature"` TotalsByLanguageFeature []totalsByLangFeature `json:"totals_by_language_feature"` @@ -123,6 +124,7 @@ func ExtractUserMetrics(taskCtx plugin.SubTaskContext) errors.Error { UsedCli: u.UsedCli, UsedCopilotCodeReviewActive: u.UsedCopilotCodeReviewActive, UsedCopilotCodeReviewPassive: u.UsedCopilotCodeReviewPassive, + AiCreditsUsed: u.AiCreditsUsed, CopilotActivityMetrics: models.CopilotActivityMetrics{ UserInitiatedInteractionCount: u.UserInitiatedInteractionCount, CodeGenerationActivityCount: u.CodeGenerationActivityCount, From c4f3d7a123f79056752c680d2e84e64525395a54 Mon Sep 17 00:00:00 2001 From: Pankaj Chaudhari Date: Tue, 28 Jul 2026 17:33:04 +0000 Subject: [PATCH 2/3] fix(gh-copilot): parse report metadata from already-read body The user and enterprise Copilot metrics collectors read the report metadata HTTP response body with io.ReadAll, then called parseReportMetadataResponse(res, ...) which read res.Body a second time. Since the body was already consumed, the second read returned empty, parseReportMetadata logged "Report metadata response was empty, skipping" and returned nil, so the collectors produced zero records. Parse the metadata from the body already read instead. The organization collector was unaffected because it reads the body only once inline. Fixes empty _raw_copilot_user_metrics and _raw_copilot_enterprise_metrics (and downstream _tool_copilot_user_daily_metrics) while org metrics work. Signed-off-by: Pankaj Chaudhari --- .../gh-copilot/tasks/report_download_helper.go | 16 +++++----------- .../gh-copilot/tasks/user_metrics_collector.go | 11 +++++------ 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/backend/plugins/gh-copilot/tasks/report_download_helper.go b/backend/plugins/gh-copilot/tasks/report_download_helper.go index 236b039889b..0e0aec770fa 100644 --- a/backend/plugins/gh-copilot/tasks/report_download_helper.go +++ b/backend/plugins/gh-copilot/tasks/report_download_helper.go @@ -212,17 +212,11 @@ func parseRawReportResponse(res *http.Response, logger log.Logger) ([]json.RawMe return nil, nil } - var meta *reportMetadataResponse - if jsonErr := json.Unmarshal(body, &meta); jsonErr != nil { - snippet := string(body) - if len(snippet) > 200 { - snippet = snippet[:200] - } - logger.Error(jsonErr, "failed to parse report metadata, body=%s", snippet) - return nil, errors.Default.Wrap(jsonErr, "failed to parse report metadata") - } - - meta, err := parseReportMetadataResponse(res, logger) + // Parse the metadata from the body we already read above. Previously this + // re-read res.Body via parseReportMetadataResponse, but the body had already + // been consumed by io.ReadAll, so the second read returned empty and the + // collector silently produced zero records (affecting enterprise metrics). + meta, err := parseReportMetadata(body, logger) if err != nil || meta == nil { return nil, err } diff --git a/backend/plugins/gh-copilot/tasks/user_metrics_collector.go b/backend/plugins/gh-copilot/tasks/user_metrics_collector.go index 8198b48a624..d24db69f411 100644 --- a/backend/plugins/gh-copilot/tasks/user_metrics_collector.go +++ b/backend/plugins/gh-copilot/tasks/user_metrics_collector.go @@ -62,12 +62,11 @@ func parseUserMetricsReportResponse(res *http.Response, logger log.Logger) ([]js return nil, nil } - var meta *reportMetadataResponse - if jsonErr := json.Unmarshal(body, &meta); jsonErr != nil { - return nil, errors.Default.Wrap(jsonErr, "failed to parse report metadata") - } - - meta, err := parseReportMetadataResponse(res, logger) + // Parse the metadata from the body we already read above. Previously this + // re-read res.Body via parseReportMetadataResponse, but the body had already + // been consumed by io.ReadAll, so the second read returned empty and the + // collector silently produced zero user-metrics records. + meta, err := parseReportMetadata(body, logger) if err != nil || meta == nil { return nil, err } From 829f17f221940ead41236b7e4311f706780b680f Mon Sep 17 00:00:00 2001 From: Pankaj Chaudhari Date: Thu, 30 Jul 2026 08:44:22 +0000 Subject: [PATCH 3/3] fix(gh-copilot): resolve migration-linter and golangci-lint failures - migrationscripts: inline snapshot structs instead of importing models pkg - user_metrics: fix gofmt struct-tag alignment - ai_credit_collector/extractor: fix ASF license header (capital The) - report_download_helper: remove unused readReportMetadataBody/parseReportMetadataResponse - metrics_collector_test: retarget tests to parseReportMetadata/ignoreNoContent --- .../20260708_add_ai_credit_usage_metrics.go | 127 +++++++++++++++++- .../plugins/gh-copilot/models/user_metrics.go | 16 +-- .../gh-copilot/tasks/ai_credit_collector.go | 2 +- .../gh-copilot/tasks/ai_credit_extractor.go | 2 +- .../tasks/metrics_collector_test.go | 34 ++--- .../tasks/report_download_helper.go | 26 ---- 6 files changed, 141 insertions(+), 66 deletions(-) diff --git a/backend/plugins/gh-copilot/models/migrationscripts/20260708_add_ai_credit_usage_metrics.go b/backend/plugins/gh-copilot/models/migrationscripts/20260708_add_ai_credit_usage_metrics.go index 6b52dd0940a..e968fb0dcfd 100644 --- a/backend/plugins/gh-copilot/models/migrationscripts/20260708_add_ai_credit_usage_metrics.go +++ b/backend/plugins/gh-copilot/models/migrationscripts/20260708_add_ai_credit_usage_metrics.go @@ -18,21 +18,138 @@ limitations under the License. package migrationscripts import ( + "time" + "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" "github.com/apache/incubator-devlake/helpers/migrationhelper" - "github.com/apache/incubator-devlake/plugins/gh-copilot/models" ) type addAiCreditUsageMetrics struct{} +// --- Snapshot structs for migration (avoid importing models package to prevent drift) --- + +type creditUsageBreakdown20260708 struct { + GrossQuantity float64 + DiscountQuantity float64 + NetQuantity float64 + PricePerUnit float64 + GrossAmount float64 + DiscountAmount float64 + NetAmount float64 +} + +type enterpriseAiCreditUsage20260708 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(191)"` + Year int `gorm:"primaryKey"` + Month int `gorm:"primaryKey"` + Day int `gorm:"primaryKey"` + + Enterprise string `gorm:"primaryKey;type:varchar(191)"` + Model string `gorm:"primaryKey;type:varchar(191)"` + Organization string `gorm:"index;type:varchar(255)"` + User string `gorm:"index;type:varchar(255)"` + + Product string `gorm:"type:varchar(32)"` + CostCenterId string `gorm:"index;type:varchar(255)"` + CostCenterName string `gorm:"type:varchar(255)"` + + creditUsageBreakdown20260708 `gorm:"embedded"` + archived.NoPKModel +} + +func (enterpriseAiCreditUsage20260708) TableName() string { + return "_tool_copilot_enterprise_ai_credit_usage" +} + +type orgAiCreditUsage20260708 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(191)"` + Year int `gorm:"primaryKey"` + Month int `gorm:"primaryKey"` + Day int `gorm:"primaryKey"` + + Organization string `gorm:"primaryKey;type:varchar(191)"` + Model string `gorm:"primaryKey;type:varchar(191)"` + User string `gorm:"index;type:varchar(255)"` + + Product string `gorm:"type:varchar(32)"` + + creditUsageBreakdown20260708 `gorm:"embedded"` + archived.NoPKModel +} + +func (orgAiCreditUsage20260708) TableName() string { + return "_tool_copilot_org_ai_credit_usage" +} + +type userAiCreditUsage20260708 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(191)"` + Year int `gorm:"primaryKey"` + Month int `gorm:"primaryKey"` + Day int `gorm:"primaryKey"` + + User string `gorm:"primaryKey;type:varchar(191)"` + Model string `gorm:"primaryKey;type:varchar(191)"` + + Product string `gorm:"type:varchar(32)"` + + creditUsageBreakdown20260708 `gorm:"embedded"` + archived.NoPKModel +} + +func (userAiCreditUsage20260708) TableName() string { + return "_tool_copilot_user_ai_credit_usage" +} + +// userDailyMetrics20260708 adds AI credit, CLI and code-review columns to the existing table. +type userDailyMetrics20260708 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + UserId int64 `gorm:"primaryKey"` + + OrganizationId string `gorm:"type:varchar(100)"` + EnterpriseId string `gorm:"type:varchar(100)"` + UserLogin string `gorm:"type:varchar(255);index"` + UsedAgent bool + UsedChat bool + UsedCli bool `gorm:"comment:Whether user used Copilot CLI"` + UsedCopilotCodeReviewActive bool `gorm:"comment:Whether user actively used code review"` + UsedCopilotCodeReviewPassive bool `gorm:"comment:Whether user passively used code review"` + AiCreditsUsed float64 `gorm:"comment:AI credits consumed on this day"` + + UserInitiatedInteractionCount int + CodeGenerationActivityCount int + CodeAcceptanceActivityCount int + LocSuggestedToAddSum int + LocSuggestedToDeleteSum int + LocAddedSum int + LocDeletedSum int + + CliSessionCount int + CliRequestCount int + CliPromptCount int + CliOutputTokenSum int + CliPromptTokenSum int + + archived.NoPKModel +} + +func (userDailyMetrics20260708) TableName() string { + return "_tool_copilot_user_daily_metrics" +} + func (u *addAiCreditUsageMetrics) Up(basicRes context.BasicRes) errors.Error { return migrationhelper.AutoMigrateTables( basicRes, - &models.GhCopilotEnterpriseAiCreditUsage{}, - &models.GhCopilotOrgAiCreditUsage{}, - &models.GhCopilotUserAiCreditUsage{}, - &models.GhCopilotUserDailyMetrics{}, + &enterpriseAiCreditUsage20260708{}, + &orgAiCreditUsage20260708{}, + &userAiCreditUsage20260708{}, + &userDailyMetrics20260708{}, ) } diff --git a/backend/plugins/gh-copilot/models/user_metrics.go b/backend/plugins/gh-copilot/models/user_metrics.go index ad4ce83551d..0a16b3bce82 100644 --- a/backend/plugins/gh-copilot/models/user_metrics.go +++ b/backend/plugins/gh-copilot/models/user_metrics.go @@ -30,14 +30,14 @@ type GhCopilotUserDailyMetrics struct { Day time.Time `gorm:"primaryKey;type:date" json:"day"` UserId int64 `gorm:"primaryKey" json:"userId"` - OrganizationId string `json:"organizationId" gorm:"type:varchar(100)"` - EnterpriseId string `json:"enterpriseId" gorm:"type:varchar(100)"` - UserLogin string `json:"userLogin" gorm:"type:varchar(255);index"` - UsedAgent bool `json:"usedAgent"` - UsedChat bool `json:"usedChat"` - UsedCli bool `json:"usedCli" gorm:"comment:Whether user used Copilot CLI"` - UsedCopilotCodeReviewActive bool `json:"usedCopilotCodeReviewActive" gorm:"comment:Whether user actively used code review"` - UsedCopilotCodeReviewPassive bool `json:"usedCopilotCodeReviewPassive" gorm:"comment:Whether user passively used code review"` + OrganizationId string `json:"organizationId" gorm:"type:varchar(100)"` + EnterpriseId string `json:"enterpriseId" gorm:"type:varchar(100)"` + UserLogin string `json:"userLogin" gorm:"type:varchar(255);index"` + UsedAgent bool `json:"usedAgent"` + UsedChat bool `json:"usedChat"` + UsedCli bool `json:"usedCli" gorm:"comment:Whether user used Copilot CLI"` + UsedCopilotCodeReviewActive bool `json:"usedCopilotCodeReviewActive" gorm:"comment:Whether user actively used code review"` + UsedCopilotCodeReviewPassive bool `json:"usedCopilotCodeReviewPassive" gorm:"comment:Whether user passively used code review"` AiCreditsUsed float64 `json:"aiCreditsUsed" gorm:"comment:AI credits consumed on this day"` CopilotActivityMetrics `mapstructure:",squash"` diff --git a/backend/plugins/gh-copilot/tasks/ai_credit_collector.go b/backend/plugins/gh-copilot/tasks/ai_credit_collector.go index ae9d5fdf0bb..f9dd4d75b3a 100644 --- a/backend/plugins/gh-copilot/tasks/ai_credit_collector.go +++ b/backend/plugins/gh-copilot/tasks/ai_credit_collector.go @@ -2,7 +2,7 @@ Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. -the ASF licenses this file to You under the Apache License, Version 2.0 +The ASF licenses this file to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at diff --git a/backend/plugins/gh-copilot/tasks/ai_credit_extractor.go b/backend/plugins/gh-copilot/tasks/ai_credit_extractor.go index 5cca80a5dcd..7d372db2629 100644 --- a/backend/plugins/gh-copilot/tasks/ai_credit_extractor.go +++ b/backend/plugins/gh-copilot/tasks/ai_credit_extractor.go @@ -2,7 +2,7 @@ Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. -the ASF licenses this file to You under the Apache License, Version 2.0 +The ASF licenses this file to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at diff --git a/backend/plugins/gh-copilot/tasks/metrics_collector_test.go b/backend/plugins/gh-copilot/tasks/metrics_collector_test.go index d71d20d4440..88214af6db6 100644 --- a/backend/plugins/gh-copilot/tasks/metrics_collector_test.go +++ b/backend/plugins/gh-copilot/tasks/metrics_collector_test.go @@ -18,12 +18,11 @@ limitations under the License. package tasks import ( - "bytes" - "io" "net/http" "testing" "time" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/stretchr/testify/require" ) @@ -109,35 +108,20 @@ func TestUserMetricsDateRangeAppliesFourDayBackfillWindow(t *testing.T) { require.Equal(t, time.Date(2025, 1, 6, 0, 0, 0, 0, time.UTC), start) } -func TestParseReportMetadataResponseNoContent(t *testing.T) { - res := &http.Response{ - StatusCode: http.StatusNoContent, - Body: io.NopCloser(bytes.NewReader(nil)), - } - - meta, err := parseReportMetadataResponse(res, nil) - require.NoError(t, err) - require.Nil(t, meta) +func TestIgnoreNoContentSkips204And404(t *testing.T) { + require.Equal(t, helper.ErrIgnoreAndContinue, ignoreNoContent(&http.Response{StatusCode: http.StatusNoContent})) + require.Equal(t, helper.ErrIgnoreAndContinue, ignoreNoContent(&http.Response{StatusCode: http.StatusNotFound})) + require.NoError(t, ignoreNoContent(&http.Response{StatusCode: http.StatusOK})) } -func TestParseReportMetadataResponseEmptyBody(t *testing.T) { - res := &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewReader(nil)), - } - - meta, err := parseReportMetadataResponse(res, nil) +func TestParseReportMetadataEmptyBody(t *testing.T) { + meta, err := parseReportMetadata(nil, nil) require.NoError(t, err) require.Nil(t, meta) } -func TestParseReportMetadataResponseEmptyString(t *testing.T) { - res := &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewReader([]byte(`""`))), - } - - meta, err := parseReportMetadataResponse(res, nil) +func TestParseReportMetadataEmptyString(t *testing.T) { + meta, err := parseReportMetadata([]byte(`""`), nil) require.NoError(t, err) require.Nil(t, meta) } diff --git a/backend/plugins/gh-copilot/tasks/report_download_helper.go b/backend/plugins/gh-copilot/tasks/report_download_helper.go index 0e0aec770fa..280e418e30c 100644 --- a/backend/plugins/gh-copilot/tasks/report_download_helper.go +++ b/backend/plugins/gh-copilot/tasks/report_download_helper.go @@ -95,15 +95,6 @@ type reportMetadataResponse struct { ReportEndDay string `json:"report_end_day"` } -func readReportMetadataBody(res *http.Response) ([]byte, errors.Error) { - body, readErr := io.ReadAll(res.Body) - res.Body.Close() - if readErr != nil { - return nil, errors.Default.Wrap(readErr, "failed to read report metadata") - } - return body, nil -} - func logReportMetadataParseError(body []byte, err error, logger log.Logger) { if logger == nil { return @@ -165,23 +156,6 @@ func parseReportMetadata(body []byte, logger log.Logger) (*reportMetadataRespons return &meta, nil } -func parseReportMetadataResponse(res *http.Response, logger log.Logger) (*reportMetadataResponse, errors.Error) { - if res.StatusCode == http.StatusNoContent { - if logger != nil { - logger.Info("Report metadata not ready yet (204), skipping for now") - } - res.Body.Close() - return nil, nil - } - - body, readErr := readReportMetadataBody(res) - if readErr != nil { - return nil, readErr - } - - return parseReportMetadata(body, logger) -} - func collectRawReportRecords(meta *reportMetadataResponse, logger log.Logger) ([]json.RawMessage, errors.Error) { if len(meta.DownloadLinks) == 0 { logger.Info("No download links for report day=%s, skipping", meta.ReportDay)