From 2d5a8105dd1080c92a75a07b21b3e55dd4e4c0f2 Mon Sep 17 00:00:00 2001 From: Roger Abelenda <3179183+rabelenda@users.noreply.github.com> Date: Fri, 30 Oct 2026 11:17:30 -0300 Subject: [PATCH] feat(azuredevops): collect work items and test plans Collect Azure Boards work items, iterations, comments, and changelogs, and Azure Test Plans cases and executions, into the ticket and QA domains. Scope config can map Azure types and states, and comment and changelog collection stay optional. Work items and tests belong to the project. Collecting them on every selected repository would store the same rows once per repository, so the the plugin only collects the one board and QA project for first scope that requests them. Now the plugin does not show repositories that are disabled in Azure Devops. Additionally, a project with no repository now gets one " (project)" scope, which the scope picker can select. Graph users are served by vssps.dev.azure.com. With recent changes in API client, it now rejects an absolute UrlTemplate, so the plugin now uses a second client for account collection with that endpoint and a relative path. The connection form now lists the Read scopes this plugin needs, so a user can easily create a token with necessary scopes. Additionally, username can now be left empty for Cloud (as was stated on the description of the field), instead of having to set a username with a whitespace to be able to save the connection. Refs #5996 Signed-off-by: Roger Abelenda <3179183+rabelenda@users.noreply.github.com> --- backend/plugins/azuredevops_go/README.md | 6 + .../azuredevops_go/api/azuredevops/client.go | 78 ++ .../api/azuredevops/feature_state_test.go | 32 + .../azuredevops_go/api/blueprint_v200.go | 137 +-- .../azuredevops_go/api/blueprint_v200_test.go | 51 +- .../azuredevops_go/api/remote_helper.go | 49 +- backend/plugins/azuredevops_go/impl/impl.go | 8 + .../azuredevops_go/impl/subtasks_test.go | 45 + backend/plugins/azuredevops_go/models/base.go | 5 +- .../azuredevops_go/models/iteration.go | 45 + .../20261028_add_boards_and_tests.go | 63 ++ .../models/migrationscripts/register.go | 1 + backend/plugins/azuredevops_go/models/repo.go | 47 + .../azuredevops_go/models/repo_test.go | 76 ++ .../azuredevops_go/models/scope_config.go | 5 + .../azuredevops_go/models/scope_entities.go | 95 ++ .../azuredevops_go/models/test_plan.go | 107 +++ .../azuredevops_go/models/work_item.go | 103 +++ .../azuredevops_go/tasks/account_collector.go | 24 +- .../azuredevops_go/tasks/boards_common.go | 149 ++++ .../plugins/azuredevops_go/tasks/iteration.go | 296 +++++++ .../azuredevops_go/tasks/repo_converter.go | 8 +- .../azuredevops_go/tasks/test_plans.go | 606 +++++++++++++ .../azuredevops_go/tasks/test_plans_test.go | 45 + .../tasks/work_item_children.go | 305 +++++++ .../tasks/work_item_collector.go | 115 +++ .../azuredevops_go/tasks/work_item_convert.go | 407 +++++++++ .../azuredevops_go/tasks/work_item_mapping.go | 378 ++++++++ .../tasks/work_item_mapping_test.go | 98 +++ .../azuredevops_go/tasks/work_item_state.go | 108 +++ .../azuredevops_go/tasks/work_item_updates.go | 244 ++++++ .../connection-form/fields/username.tsx | 8 +- .../src/plugins/register/azure/config.tsx | 20 +- .../register/azure/ticket-transformation.tsx | 272 ++++++ .../plugins/register/azure/transformation.tsx | 8 + grafana/dashboards/mysql/azure-dev-ops.json | 824 ++++++++++++++++- .../dashboards/postgresql/azure-dev-ops.json | 826 +++++++++++++++++- 37 files changed, 5579 insertions(+), 115 deletions(-) create mode 100644 backend/plugins/azuredevops_go/api/azuredevops/feature_state_test.go create mode 100644 backend/plugins/azuredevops_go/impl/subtasks_test.go create mode 100644 backend/plugins/azuredevops_go/models/iteration.go create mode 100644 backend/plugins/azuredevops_go/models/migrationscripts/20261028_add_boards_and_tests.go create mode 100644 backend/plugins/azuredevops_go/models/repo_test.go create mode 100644 backend/plugins/azuredevops_go/models/scope_entities.go create mode 100644 backend/plugins/azuredevops_go/models/test_plan.go create mode 100644 backend/plugins/azuredevops_go/models/work_item.go create mode 100644 backend/plugins/azuredevops_go/tasks/boards_common.go create mode 100644 backend/plugins/azuredevops_go/tasks/iteration.go create mode 100644 backend/plugins/azuredevops_go/tasks/test_plans.go create mode 100644 backend/plugins/azuredevops_go/tasks/test_plans_test.go create mode 100644 backend/plugins/azuredevops_go/tasks/work_item_children.go create mode 100644 backend/plugins/azuredevops_go/tasks/work_item_collector.go create mode 100644 backend/plugins/azuredevops_go/tasks/work_item_convert.go create mode 100644 backend/plugins/azuredevops_go/tasks/work_item_mapping.go create mode 100644 backend/plugins/azuredevops_go/tasks/work_item_mapping_test.go create mode 100644 backend/plugins/azuredevops_go/tasks/work_item_state.go create mode 100644 backend/plugins/azuredevops_go/tasks/work_item_updates.go create mode 100644 config-ui/src/plugins/register/azure/ticket-transformation.tsx diff --git a/backend/plugins/azuredevops_go/README.md b/backend/plugins/azuredevops_go/README.md index 1ea487025df..1a0d8072517 100644 --- a/backend/plugins/azuredevops_go/README.md +++ b/backend/plugins/azuredevops_go/README.md @@ -26,5 +26,11 @@ The plugin is able to coexist with the Python version as both implementations co - Code - Graph (collectAccounts task) - Release +- Work Items, to collect Azure Boards work items, iterations, comments, and changelogs +- Test Management, to collect test plans, test cases, test runs, and test results + +Selecting a repository with work item or test entities collects that repository's project once. Further repositories of the same project in a blueprint do not collect the project again, and the work items and tests are stored on one board. Repository scope names stay the repository name. A project with no enabled repository lists one scope named after the project with a "(project)" suffix, and that scope collects its work items and tests. + +Work items are stored in the ticket domain. The board is named after the project, uses the project URL, and has type Scrum. Custom fields are stored under both the Azure reference name and the display name. Test plans, cases, and executions are stored in the QA domain, including each test case's creation time and creator. The scope config can map work item types and states, and can turn comment and changelog collection on. Access to Service Connections has been removed as they usually contain sensitive security information. \ No newline at end of file diff --git a/backend/plugins/azuredevops_go/api/azuredevops/client.go b/backend/plugins/azuredevops_go/api/azuredevops/client.go index 355f5b997d3..824e29bdf98 100644 --- a/backend/plugins/azuredevops_go/api/azuredevops/client.go +++ b/backend/plugins/azuredevops_go/api/azuredevops/client.go @@ -311,3 +311,81 @@ func (c *Client) GetRemoteRepositories(args GetRemoteRepositoriesArgs) ([]Remote query.Set("continuationToken", contToken) } } + +const reposFeatureID = "ms.vss-code.version-control" + +func (c *Client) GitReposEnabled(orgId, projectName string) (bool, errors.Error) { + project, err := c.getProject(orgId, projectName) + if err != nil { + return false, err + } + query := url.Values{} + query.Set("api-version", "4.1-preview.1") + path := fmt.Sprintf("_apis/FeatureManagement/FeatureStatesQuery/host/project/%s", url.PathEscape(project.Id)) + if orgId != "" { + path = url.PathEscape(orgId) + "/" + path + } + res, err := c.apiClient.Post(path, query, map[string]interface{}{ + "featureIds": []string{reposFeatureID}, + "scopeValues": map[string]string{"project": project.Id}, + }, nil) + if err != nil { + return false, err + } + if res.StatusCode == http.StatusNonAuthoritativeInfo || res.StatusCode == http.StatusUnauthorized || res.StatusCode >= 400 { + _, _ = io.Copy(io.Discard, res.Body) + res.Body.Close() + return false, errors.Default.New(fmt.Sprintf("failed to read Azure Repos service state (%d)", res.StatusCode)) + } + var body struct { + FeatureStates map[string]struct { + State json.RawMessage `json:"state"` + } `json:"featureStates"` + } + if err = api.UnmarshalResponse(res, &body); err != nil { + return false, err + } + feature, ok := body.FeatureStates[reposFeatureID] + if !ok { + return true, nil + } + return !featureStateDisabled(feature.State), nil +} + +func (c *Client) getProject(orgId, projectName string) (Project, errors.Error) { + query := url.Values{} + query.Set("api-version", apiVersion) + path := fmt.Sprintf("_apis/projects/%s", url.PathEscape(projectName)) + if orgId != "" { + path = url.PathEscape(orgId) + "/" + path + } + res, err := c.apiClient.Get(path, query, nil) + if err != nil { + return Project{}, err + } + if res.StatusCode == http.StatusNonAuthoritativeInfo || res.StatusCode == http.StatusUnauthorized || res.StatusCode >= 400 { + _, _ = io.Copy(io.Discard, res.Body) + res.Body.Close() + return Project{}, errors.Default.New(fmt.Sprintf("failed to read Azure DevOps project %s (%d)", projectName, res.StatusCode)) + } + var project Project + if err = api.UnmarshalResponse(res, &project); err != nil { + return Project{}, err + } + if project.Id == "" { + return Project{}, errors.Default.New(fmt.Sprintf("Azure DevOps project %s has no id", projectName)) + } + return project, nil +} + +func featureStateDisabled(raw json.RawMessage) bool { + var text string + if json.Unmarshal(raw, &text) == nil { + return strings.EqualFold(text, "disabled") || text == "0" + } + var number float64 + if json.Unmarshal(raw, &number) == nil { + return number == 0 + } + return false +} diff --git a/backend/plugins/azuredevops_go/api/azuredevops/feature_state_test.go b/backend/plugins/azuredevops_go/api/azuredevops/feature_state_test.go new file mode 100644 index 00000000000..1a55e286fa6 --- /dev/null +++ b/backend/plugins/azuredevops_go/api/azuredevops/feature_state_test.go @@ -0,0 +1,32 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azuredevops + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFeatureStateDisabled(t *testing.T) { + assert.True(t, featureStateDisabled(json.RawMessage(`"disabled"`))) + assert.True(t, featureStateDisabled(json.RawMessage(`0`))) + assert.False(t, featureStateDisabled(json.RawMessage(`"enabled"`))) + assert.False(t, featureStateDisabled(json.RawMessage(`1`))) +} diff --git a/backend/plugins/azuredevops_go/api/blueprint_v200.go b/backend/plugins/azuredevops_go/api/blueprint_v200.go index 22880acb768..483228214b6 100644 --- a/backend/plugins/azuredevops_go/api/blueprint_v200.go +++ b/backend/plugins/azuredevops_go/api/blueprint_v200.go @@ -20,15 +20,16 @@ package api import ( "net/url" - "golang.org/x/exp/slices" - "github.com/apache/devlake/core/errors" "github.com/apache/devlake/core/utils" coreModels "github.com/apache/devlake/core/models" + "github.com/apache/devlake/core/models/common" + "github.com/apache/devlake/core/models/domainlayer" "github.com/apache/devlake/core/models/domainlayer/code" "github.com/apache/devlake/core/models/domainlayer/devops" "github.com/apache/devlake/core/models/domainlayer/didgen" + "github.com/apache/devlake/core/models/domainlayer/qa" "github.com/apache/devlake/core/models/domainlayer/ticket" "github.com/apache/devlake/core/plugin" helper "github.com/apache/devlake/helpers/pluginhelper/api" @@ -51,7 +52,7 @@ func MakePipelinePlanV200( return nil, nil, err } - sc, err := makeScopeV200(connectionId, scopeDetails) + sc, err := makeScopeV200(connection, scopeDetails) if err != nil { return nil, nil, err } @@ -65,73 +66,82 @@ func MakePipelinePlanV200( } func makeScopeV200( - connectionId uint64, + connection *models.AzuredevopsConnection, scopeDetails []*srvhelper.ScopeDetail[models.AzuredevopsRepo, models.AzuredevopsScopeConfig], ) ([]plugin.Scope, errors.Error) { sc := make([]plugin.Scope, 0, 3*len(scopeDetails)) - - for _, scope := range scopeDetails { - azuredevopsRepo, scopeConfig := scope.Scope, scope.ScopeConfig - if azuredevopsRepo.Type != models.RepositoryTypeADO { - continue - } - id := didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connectionId, azuredevopsRepo.Id) - - // if no entities specified, use all entities enabled by default - if len(scopeConfig.Entities) == 0 { - scopeConfig.Entities = plugin.DOMAIN_TYPES - } - - if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE_REVIEW) || - utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) || - utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CROSS) { - // if we don't need to collect gitex, we need to add repo to scopes here - scopeRepo := code.NewRepo(id, azuredevopsRepo.Name) - sc = append(sc, scopeRepo) - } - - // add cicd_scope to scopes - if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CICD) { - scopeCICD := devops.NewCicdScope(id, azuredevopsRepo.Name) - sc = append(sc, scopeCICD) - } - - // add board to scopes - if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) { - scopeTicket := ticket.NewBoard(id, azuredevopsRepo.Name) - sc = append(sc, scopeTicket) - } + connectionId := uint64(0) + endpoint := "" + if connection != nil { + connectionId = connection.ID + endpoint = connection.GetEndpoint() } + claimedProjects := map[string]struct{}{} for _, scope := range scopeDetails { azuredevopsRepo, scopeConfig := scope.Scope, scope.ScopeConfig - if azuredevopsRepo.Type == models.RepositoryTypeADO { - continue - } - id := didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connectionId, azuredevopsRepo.Id) + entities := models.ClaimProjectEntities( + models.ScopeEntities(azuredevopsRepo.Type, scopeConfig.Entities), + azuredevopsRepo.ProjectKey(), + claimedProjects, + ) + repoID := didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connectionId, azuredevopsRepo.Id) + name := azuredevopsRepo.DomainName() - // Azure DevOps Pipeline can be used with remote repositories such as GitHub and Bitbucket - if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CICD) { - scopeCICD := devops.NewCicdScope(id, azuredevopsRepo.Name) - sc = append(sc, scopeCICD) - } - - // DOMAIN_TYPE_CODE (i.e. gitextractor, rediff) only works if the repository is public - if !azuredevopsRepo.IsPrivate && utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) { - scopeRepo := code.NewRepo(id, azuredevopsRepo.Name) - sc = append(sc, scopeRepo) + if azuredevopsRepo.Type == models.RepositoryTypeADO { + if utils.StringsContains(entities, plugin.DOMAIN_TYPE_CODE_REVIEW) || + utils.StringsContains(entities, plugin.DOMAIN_TYPE_CODE) || + utils.StringsContains(entities, plugin.DOMAIN_TYPE_CROSS) { + sc = append(sc, code.NewRepo(repoID, name)) + } + if utils.StringsContains(entities, plugin.DOMAIN_TYPE_CICD) { + sc = append(sc, devops.NewCicdScope(repoID, name)) + } + } else if azuredevopsRepo.Type != models.RepositoryTypeProject { + if utils.StringsContains(entities, plugin.DOMAIN_TYPE_CICD) { + sc = append(sc, devops.NewCicdScope(repoID, name)) + } + if !azuredevopsRepo.IsPrivate && utils.StringsContains(entities, plugin.DOMAIN_TYPE_CODE) { + sc = append(sc, code.NewRepo(repoID, name)) + } } + sc = appendProjectScopes(sc, connectionId, endpoint, azuredevopsRepo, entities) } return sc, nil } +func appendProjectScopes(sc []plugin.Scope, connectionId uint64, endpoint string, repo models.AzuredevopsRepo, entities []string) []plugin.Scope { + if !models.HasProjectEntity(entities) { + return sc + } + id := didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connectionId, repo.ProjectDomainScopeID()) + projectName := repo.ProjectName() + if utils.StringsContains(entities, plugin.DOMAIN_TYPE_TICKET) { + board := ticket.NewBoard(id, projectName) + board.Type = "Scrum" + board.Url = models.ProjectBoardURL(endpoint, repo.OrganizationId, projectName) + sc = append(sc, board) + } + if utils.StringsContains(entities, plugin.DOMAIN_TYPE_CODE_QUALITY) { + sc = append(sc, &qa.QaProject{ + DomainEntityExtended: domainlayer.DomainEntityExtended{ + Id: id, + NoPKModel: common.NewNoPKModel(), + }, + Name: projectName, + }) + } + return sc +} + func makePipelinePlanV200( subtaskMetas []plugin.SubTaskMeta, connection *models.AzuredevopsConnection, scopeDetails []*srvhelper.ScopeDetail[models.AzuredevopsRepo, models.AzuredevopsScopeConfig], ) (coreModels.PipelinePlan, errors.Error) { plans := make(coreModels.PipelinePlan, 0, 3*len(scopeDetails)) + claimedProjects := map[string]struct{}{} for _, scope := range scopeDetails { azuredevopsRepo, scopeConfig := scope.Scope, scope.ScopeConfig var stage coreModels.PipelineStage @@ -147,18 +157,19 @@ func makePipelinePlanV200( options["repositoryId"] = azuredevopsRepo.Id options["repositoryType"] = azuredevopsRepo.Type - // construct subtasks - var entities []string - if scope.Scope.Type == models.RepositoryTypeADO { - entities = append(entities, scopeConfig.Entities...) - } else { - if i := slices.Index(scopeConfig.Entities, plugin.DOMAIN_TYPE_CICD); i >= 0 { - entities = append(entities, scopeConfig.Entities[i]) - } - - if i := slices.Index(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE); i >= 0 && !scope.Scope.IsPrivate { - entities = append(entities, scopeConfig.Entities[i]) + entities := models.ClaimProjectEntities( + models.ScopeEntities(azuredevopsRepo.Type, scopeConfig.Entities), + azuredevopsRepo.ProjectKey(), + claimedProjects, + ) + if azuredevopsRepo.Type != models.RepositoryTypeADO && azuredevopsRepo.Type != models.RepositoryTypeProject && azuredevopsRepo.IsPrivate { + filtered := make([]string, 0, len(entities)) + for _, entity := range entities { + if entity != plugin.DOMAIN_TYPE_CODE { + filtered = append(filtered, entity) + } } + entities = filtered } subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, entities) @@ -173,7 +184,7 @@ func makePipelinePlanV200( }) // collect git data by gitextractor if CODE was requested - if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) && !scope.Scope.IsPrivate || len(scopeConfig.Entities) == 0 { + if azuredevopsRepo.Type != models.RepositoryTypeProject && (utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) && !scope.Scope.IsPrivate || len(scopeConfig.Entities) == 0) { cloneUrl, err := errors.Convert01(url.Parse(azuredevopsRepo.RemoteUrl)) if err != nil { return nil, err @@ -201,7 +212,7 @@ func makePipelinePlanV200( plans = append(plans, stage) // refdiff part - if scopeConfig.Refdiff != nil { + if azuredevopsRepo.Type != models.RepositoryTypeProject && scopeConfig.Refdiff != nil { task := &coreModels.PipelineTask{ Plugin: "refdiff", Options: scopeConfig.Refdiff, diff --git a/backend/plugins/azuredevops_go/api/blueprint_v200_test.go b/backend/plugins/azuredevops_go/api/blueprint_v200_test.go index ce45e380881..96c3046a82a 100644 --- a/backend/plugins/azuredevops_go/api/blueprint_v200_test.go +++ b/backend/plugins/azuredevops_go/api/blueprint_v200_test.go @@ -53,7 +53,9 @@ func TestMakeScopes(t *testing.T) { mockAzuredevopsPlugin(t) actualScopes, err := makeScopeV200( - connectionID, + &models.AzuredevopsConnection{ + BaseConnection: api.BaseConnection{Model: common.Model{ID: connectionID}}, + }, []*srvhelper.ScopeDetail[models.AzuredevopsRepo, models.AzuredevopsScopeConfig]{ { Scope: models.AzuredevopsRepo{ @@ -62,6 +64,10 @@ func TestMakeScopes(t *testing.T) { }, Id: azuredevopsRepoId, Type: models.RepositoryTypeADO, + AzureDevOpsPK: models.AzureDevOpsPK{ + OrganizationId: "org", + ProjectId: "proj", + }, }, ScopeConfig: &models.AzuredevopsScopeConfig{ ScopeConfig: common.ScopeConfig{ @@ -75,14 +81,19 @@ func TestMakeScopes(t *testing.T) { assert.Equal(t, 3, len(actualScopes)) assert.Equal(t, actualScopes[0].ScopeId(), expectDomainScopeId) assert.Equal(t, actualScopes[1].ScopeId(), expectDomainScopeId) - assert.Equal(t, actualScopes[2].ScopeId(), expectDomainScopeId) + assert.Equal(t, "repos", actualScopes[0].TableName()) + assert.Equal(t, "cicd_scopes", actualScopes[1].TableName()) + assert.Equal(t, "boards", actualScopes[2].TableName()) + assert.Equal(t, "azuredevops_go:AzuredevopsRepo:1:org/proj/__project__", actualScopes[2].ScopeId()) } func TestMakeScopesWithEmptyEntities(t *testing.T) { mockAzuredevopsPlugin(t) actualScopes, err := makeScopeV200( - connectionID, + &models.AzuredevopsConnection{ + BaseConnection: api.BaseConnection{Model: common.Model{ID: connectionID}}, + }, []*srvhelper.ScopeDetail[models.AzuredevopsRepo, models.AzuredevopsScopeConfig]{ { Scope: models.AzuredevopsRepo{ @@ -91,6 +102,10 @@ func TestMakeScopesWithEmptyEntities(t *testing.T) { }, Id: azuredevopsRepoId, Type: models.RepositoryTypeADO, + AzureDevOpsPK: models.AzureDevOpsPK{ + OrganizationId: "org", + ProjectId: "proj", + }, }, ScopeConfig: &models.AzuredevopsScopeConfig{ ScopeConfig: common.ScopeConfig{ @@ -101,8 +116,7 @@ func TestMakeScopesWithEmptyEntities(t *testing.T) { }, ) assert.Nil(t, err) - // empty entities should default to all domain types, producing repo + cicd + board scopes - assert.Equal(t, 3, len(actualScopes)) + assert.Equal(t, 4, len(actualScopes)) assert.Equal(t, actualScopes[0].ScopeId(), expectDomainScopeId) } @@ -110,7 +124,9 @@ func TestMakeScopesWithCrossEntity(t *testing.T) { mockAzuredevopsPlugin(t) actualScopes, err := makeScopeV200( - connectionID, + &models.AzuredevopsConnection{ + BaseConnection: api.BaseConnection{Model: common.Model{ID: connectionID}}, + }, []*srvhelper.ScopeDetail[models.AzuredevopsRepo, models.AzuredevopsScopeConfig]{ { Scope: models.AzuredevopsRepo{ @@ -119,6 +135,10 @@ func TestMakeScopesWithCrossEntity(t *testing.T) { }, Id: azuredevopsRepoId, Type: models.RepositoryTypeADO, + AzureDevOpsPK: models.AzureDevOpsPK{ + OrganizationId: "org", + ProjectId: "proj", + }, }, ScopeConfig: &models.AzuredevopsScopeConfig{ ScopeConfig: common.ScopeConfig{ @@ -129,7 +149,6 @@ func TestMakeScopesWithCrossEntity(t *testing.T) { }, ) assert.Nil(t, err) - // CROSS entity should trigger repo scope creation, plus ticket = board scope assert.Equal(t, 2, len(actualScopes)) assert.Equal(t, actualScopes[0].ScopeId(), expectDomainScopeId) assert.Equal(t, "repos", actualScopes[0].TableName()) @@ -250,11 +269,13 @@ func TestMakeRemoteRepoScopes(t *testing.T) { Private bool ExpectedScopes []string }{ - {Name: "Azure DevOps Repository", Type: models.RepositoryTypeADO, Private: false, ExpectedScopes: []string{"*code.Repo", "*ticket.Board", "*devops.CicdScope"}}, + {Name: "Azure DevOps Repository", Type: models.RepositoryTypeADO, Private: false, ExpectedScopes: []string{"*code.Repo", "*devops.CicdScope", "*ticket.Board", "*qa.QaProject"}}, + + {Name: "Public GitHub Repository", Type: models.RepositoryTypeGithub, Private: false, ExpectedScopes: []string{"*code.Repo", "*devops.CicdScope", "*ticket.Board", "*qa.QaProject"}}, - {Name: "Public GitHub Repository", Type: models.RepositoryTypeGithub, Private: false, ExpectedScopes: []string{"*code.Repo", "*devops.CicdScope"}}, + {Name: "Private GitHub Repository", Type: models.RepositoryTypeGithub, Private: true, ExpectedScopes: []string{"*devops.CicdScope", "*ticket.Board", "*qa.QaProject"}}, - {Name: "Private GitHub Repository", Type: models.RepositoryTypeGithub, Private: true, ExpectedScopes: []string{"*devops.CicdScope"}}, + {Name: "Azure DevOps Project", Type: models.RepositoryTypeProject, Private: false, ExpectedScopes: []string{"*ticket.Board", "*qa.QaProject"}}, } for _, d := range data { @@ -263,7 +284,9 @@ func TestMakeRemoteRepoScopes(t *testing.T) { id := strings.ToLower(d.Name) id = strings.ReplaceAll(id, " ", "-") actualScopes, err := makeScopeV200( - connectionID, + &models.AzuredevopsConnection{ + BaseConnection: api.BaseConnection{Model: common.Model{ID: connectionID}}, + }, []*srvhelper.ScopeDetail[models.AzuredevopsRepo, models.AzuredevopsScopeConfig]{ { Scope: models.AzuredevopsRepo{ @@ -274,11 +297,15 @@ func TestMakeRemoteRepoScopes(t *testing.T) { Type: d.Type, Name: d.Name, IsPrivate: d.Private, + AzureDevOpsPK: models.AzureDevOpsPK{ + OrganizationId: "org", + ProjectId: "proj", + }, }, ScopeConfig: &models.AzuredevopsScopeConfig{ ScopeConfig: common.ScopeConfig{ Entities: []string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_TICKET, - plugin.DOMAIN_TYPE_CICD, plugin.DOMAIN_TYPE_CODE_REVIEW}, + plugin.DOMAIN_TYPE_CICD, plugin.DOMAIN_TYPE_CODE_REVIEW, plugin.DOMAIN_TYPE_CODE_QUALITY}, }, }, }, diff --git a/backend/plugins/azuredevops_go/api/remote_helper.go b/backend/plugins/azuredevops_go/api/remote_helper.go index 6da68905e3c..e872f55d61b 100644 --- a/backend/plugins/azuredevops_go/api/remote_helper.go +++ b/backend/plugins/azuredevops_go/api/remote_helper.go @@ -86,8 +86,15 @@ func listAzuredevopsRemoteScopes( children = append(children, remote...) } - if remote, err := listAzuredevopsRepos(vsc, orgId, projectId); err == nil { - children = append(children, remote...) + // Repos turned off in project settings still leaves the default Git repo in the Git API. + reposEnabled, reposErr := vsc.GitReposEnabled(orgId, projectId) + if reposErr != nil || reposEnabled { + if remote, err := listAzuredevopsRepos(vsc, orgId, projectId); err == nil { + children = append(children, remote...) + } + } + if len(children) == 0 { + children = append(children, projectScopeEntry(connection, orgId, projectId)) } } return children, nextPage, nil @@ -137,9 +144,10 @@ func listAzuredevopsProjects(connection *models.AzuredevopsConnection, vsc azure groupId = accountName + idSeparator + vv.Name } tmp = append(tmp, dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo]{ - Id: groupId, - Type: api.RAS_ENTRY_TYPE_GROUP, - Name: vv.Name, + Id: groupId, + Type: api.RAS_ENTRY_TYPE_GROUP, + Name: vv.Name, + FullName: vv.Name, }) } mu.Lock() @@ -193,14 +201,41 @@ func listAzuredevopsRepos( Type: api.RAS_ENTRY_TYPE_SCOPE, ParentId: &pID, Id: v.Id, - Name: v.Name, - FullName: v.Name, + Name: repo.Name, + FullName: repo.Name, Data: &repo, }) } return } +func projectScopeEntry(connection *models.AzuredevopsConnection, orgId, projectId string) dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo] { + parent := projectId + if orgId != "" { + parent = orgId + idSeparator + projectId + } + endpoint := "" + if connection != nil { + endpoint = connection.GetEndpoint() + } + repo := models.AzuredevopsRepo{ + Id: models.ProjectScopeID(orgId, projectId), + Type: models.RepositoryTypeProject, + Name: projectId + " (project)", + Url: models.ProjectBoardURL(endpoint, orgId, projectId), + } + repo.ProjectId = projectId + repo.OrganizationId = orgId + return dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + ParentId: &parent, + Id: repo.Id, + Name: repo.Name, + FullName: repo.Name, + Data: &repo, + } +} + func listRemoteRepos( vsc azuredevops.Client, orgId, projectId string, diff --git a/backend/plugins/azuredevops_go/impl/impl.go b/backend/plugins/azuredevops_go/impl/impl.go index 0d1b862c85d..83ba65619a7 100644 --- a/backend/plugins/azuredevops_go/impl/impl.go +++ b/backend/plugins/azuredevops_go/impl/impl.go @@ -93,6 +93,14 @@ func (p Azuredevops) GetTablesInfo() []dal.Tabler { &models.AzuredevopsScopeConfig{}, &models.AzuredevopsTimelineRecord{}, &models.AzuredevopsUser{}, + &models.AzuredevopsIteration{}, + &models.AzuredevopsWorkItem{}, + &models.AzuredevopsWorkItemComment{}, + &models.AzuredevopsWorkItemUpdate{}, + &models.AzuredevopsTestPlan{}, + &models.AzuredevopsTestCase{}, + &models.AzuredevopsTestRun{}, + &models.AzuredevopsTestResult{}, } } diff --git a/backend/plugins/azuredevops_go/impl/subtasks_test.go b/backend/plugins/azuredevops_go/impl/subtasks_test.go new file mode 100644 index 00000000000..1aef4b755b4 --- /dev/null +++ b/backend/plugins/azuredevops_go/impl/subtasks_test.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import "testing" + +func TestBoardSubtasksAreOrdered(t *testing.T) { + order := map[string]int{} + for i, meta := range sortedSubtaskMetas { + order[meta.Name] = i + } + assertBefore := func(earlier, later string) { + t.Helper() + if _, ok := order[earlier]; !ok { + t.Fatalf("missing subtask %s", earlier) + } + if _, ok := order[later]; !ok { + t.Fatalf("missing subtask %s", later) + } + if order[earlier] > order[later] { + t.Fatalf("%s must run before %s", earlier, later) + } + } + assertBefore("extractApiIterations", "convertWorkItems") + assertBefore("extractApiWorkItems", "convertWorkItems") + assertBefore("extractApiWorkItems", "collectApiWorkItemComments") + assertBefore("extractApiTestPlans", "collectApiTestCases") + assertBefore("extractApiTestCases", "convertTestResults") + assertBefore("extractApiTestRuns", "collectApiTestResults") +} diff --git a/backend/plugins/azuredevops_go/models/base.go b/backend/plugins/azuredevops_go/models/base.go index e30369a7d08..af7e3844c2e 100644 --- a/backend/plugins/azuredevops_go/models/base.go +++ b/backend/plugins/azuredevops_go/models/base.go @@ -23,6 +23,7 @@ type AzureDevOpsPK struct { } const ( - RepositoryTypeADO = "TfsGit" - RepositoryTypeGithub = "github" + RepositoryTypeADO = "TfsGit" + RepositoryTypeGithub = "github" + RepositoryTypeProject = "project" ) diff --git a/backend/plugins/azuredevops_go/models/iteration.go b/backend/plugins/azuredevops_go/models/iteration.go new file mode 100644 index 00000000000..ec1df0dde5f --- /dev/null +++ b/backend/plugins/azuredevops_go/models/iteration.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/devlake/core/models/common" +) + +type AzuredevopsIteration struct { + common.NoPKModel + + ConnectionId uint64 `gorm:"primaryKey"` + Identifier string `gorm:"primaryKey;type:varchar(255)"` + NodeId int + OrgId string `gorm:"type:varchar(255)"` + ProjectId string `gorm:"type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Path string `gorm:"type:varchar(500)"` + PathKey string `gorm:"type:varchar(500)"` + HasChildren bool + StartedDate *time.Time + EndedDate *time.Time + TimeFrame string `gorm:"type:varchar(50)"` +} + +func (AzuredevopsIteration) TableName() string { + return "_tool_azuredevops_go_iterations" +} diff --git a/backend/plugins/azuredevops_go/models/migrationscripts/20261028_add_boards_and_tests.go b/backend/plugins/azuredevops_go/models/migrationscripts/20261028_add_boards_and_tests.go new file mode 100644 index 00000000000..31b10f74ecc --- /dev/null +++ b/backend/plugins/azuredevops_go/models/migrationscripts/20261028_add_boards_and_tests.go @@ -0,0 +1,63 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/devlake/core/context" + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/helpers/migrationhelper" + "github.com/apache/devlake/plugins/azuredevops_go/models" + "gorm.io/datatypes" +) + +type addAzuredevopsBoardsAndTests struct{} + +type azuredevopsScopeConfig20261028 struct { + ProcessTemplate string `gorm:"type:varchar(255)"` + TypeMappings datatypes.JSON + StatusMappings datatypes.JSON + CollectChangelogs bool + CollectComments bool +} + +func (azuredevopsScopeConfig20261028) TableName() string { + return "_tool_azuredevops_go_scope_configs" +} + +func (script *addAzuredevopsBoardsAndTests) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &azuredevopsScopeConfig20261028{}, + &models.AzuredevopsIteration{}, + &models.AzuredevopsWorkItem{}, + &models.AzuredevopsWorkItemComment{}, + &models.AzuredevopsWorkItemUpdate{}, + &models.AzuredevopsTestPlan{}, + &models.AzuredevopsTestCase{}, + &models.AzuredevopsTestRun{}, + &models.AzuredevopsTestResult{}, + ) +} + +func (*addAzuredevopsBoardsAndTests) Version() uint64 { + return 20261028100001 +} + +func (*addAzuredevopsBoardsAndTests) Name() string { + return "add Azure DevOps work item and test plan tables" +} diff --git a/backend/plugins/azuredevops_go/models/migrationscripts/register.go b/backend/plugins/azuredevops_go/models/migrationscripts/register.go index e71885f3583..d53a8aff5a4 100644 --- a/backend/plugins/azuredevops_go/models/migrationscripts/register.go +++ b/backend/plugins/azuredevops_go/models/migrationscripts/register.go @@ -29,5 +29,6 @@ func All() []plugin.MigrationScript { new(addEndpointToAzuredevops), new(addUsernameToAzuredevops), new(addRunNameToAzuredevopsBuild), + new(addAzuredevopsBoardsAndTests), } } diff --git a/backend/plugins/azuredevops_go/models/repo.go b/backend/plugins/azuredevops_go/models/repo.go index 25d6d6f2001..07ec3864f78 100644 --- a/backend/plugins/azuredevops_go/models/repo.go +++ b/backend/plugins/azuredevops_go/models/repo.go @@ -18,6 +18,8 @@ limitations under the License. package models import ( + "strings" + "github.com/apache/devlake/core/models/common" "github.com/apache/devlake/core/plugin" ) @@ -50,6 +52,51 @@ func (repo AzuredevopsRepo) ScopeFullName() string { return repo.Name } +func (repo AzuredevopsRepo) ProjectName() string { + name := repo.ProjectId + if name == "" { + name = repo.Name + } + return strings.TrimSuffix(name, " (project)") +} + +func (repo AzuredevopsRepo) DomainName() string { + if repo.Type == RepositoryTypeADO && repo.ProjectId != "" && repo.Name != "" && !strings.HasPrefix(repo.Name, repo.ProjectId+"/") { + return repo.ProjectId + "/" + repo.Name + } + return repo.Name +} + +func (repo AzuredevopsRepo) ProjectKey() string { + project := repo.ProjectName() + if repo.Type != RepositoryTypeProject && repo.ProjectId == "" { + return "" + } + return repo.OrganizationId + "/" + project +} + +func (repo AzuredevopsRepo) ProjectDomainScopeID() string { + return ProjectScopeID(repo.OrganizationId, repo.ProjectName()) +} + +func ProjectScopeID(org, project string) string { + if org != "" { + return org + "/" + project + "/__project__" + } + return project + "/__project__" +} + +func ProjectBoardURL(endpoint, org, project string) string { + base := strings.TrimRight(endpoint, "/") + if base == "" { + base = "https://dev.azure.com" + } + if org != "" { + return base + "/" + org + "/" + project + } + return base + "/" + project +} + func (repo AzuredevopsRepo) ScopeParams() interface{} { return &AzuredevopsApiParams{ ConnectionId: repo.ConnectionId, diff --git a/backend/plugins/azuredevops_go/models/repo_test.go b/backend/plugins/azuredevops_go/models/repo_test.go new file mode 100644 index 00000000000..6b4c200262e --- /dev/null +++ b/backend/plugins/azuredevops_go/models/repo_test.go @@ -0,0 +1,76 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "testing" + + "github.com/apache/devlake/core/plugin" + "github.com/stretchr/testify/assert" +) + +func TestScopeNames(t *testing.T) { + project := AzuredevopsRepo{ + Type: RepositoryTypeProject, Name: "proj1 (project)", ProjectId: "proj1", + OrganizationId: "org", Id: ProjectScopeID("org", "proj1"), + } + assert.Equal(t, "proj1 (project)", project.ScopeName()) + assert.Equal(t, "proj1", project.ProjectName()) + assert.Equal(t, project.Id, project.ProjectDomainScopeID()) + + repo := AzuredevopsRepo{Type: RepositoryTypeADO, Name: "repo1", ProjectId: "proj1", OrganizationId: "org"} + assert.Equal(t, "repo1", repo.ScopeName()) + assert.Equal(t, "proj1/repo1", repo.DomainName()) + assert.Equal(t, "org/proj1/__project__", repo.ProjectDomainScopeID()) + remote := AzuredevopsRepo{Type: RepositoryTypeGithub, Name: "github/org/repo1", ProjectId: "proj1"} + assert.Equal(t, "github/org/repo1", remote.DomainName()) + assert.Equal(t, "org/proj2/__project__", ProjectScopeID("org", "proj2")) + assert.Equal(t, "https://dev.azure.com/org/proj1", ProjectBoardURL("https://dev.azure.com/", "org", "proj1")) + assert.Equal(t, "https://ado.example/proj2", ProjectBoardURL("https://ado.example", "", "proj2")) +} + +func TestScopeEntities(t *testing.T) { + assert.Equal(t, []string{plugin.DOMAIN_TYPE_TICKET, plugin.DOMAIN_TYPE_CODE_QUALITY}, ScopeEntities(RepositoryTypeProject, nil)) + assert.Equal(t, []string{ + plugin.DOMAIN_TYPE_CODE, + plugin.DOMAIN_TYPE_TICKET, + plugin.DOMAIN_TYPE_CICD, + plugin.DOMAIN_TYPE_CODE_QUALITY, + }, ScopeEntities(RepositoryTypeADO, []string{ + plugin.DOMAIN_TYPE_CODE, + plugin.DOMAIN_TYPE_TICKET, + plugin.DOMAIN_TYPE_CICD, + plugin.DOMAIN_TYPE_CODE_QUALITY, + })) + assert.Equal(t, []string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_CICD}, ScopeEntities(RepositoryTypeGithub, []string{ + plugin.DOMAIN_TYPE_CODE, + plugin.DOMAIN_TYPE_CICD, + })) +} + +func TestClaimProjectEntitiesOnce(t *testing.T) { + claimed := map[string]struct{}{} + entities := []string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_TICKET, plugin.DOMAIN_TYPE_CODE_QUALITY} + first := ClaimProjectEntities(entities, "org/proj", claimed) + second := ClaimProjectEntities(entities, "org/proj", claimed) + other := ClaimProjectEntities(entities, "org/other", claimed) + assert.Equal(t, entities, first) + assert.Equal(t, []string{plugin.DOMAIN_TYPE_CODE}, second) + assert.Equal(t, entities, other) + assert.Equal(t, []string{plugin.DOMAIN_TYPE_CODE}, ClaimProjectEntities(entities, "", claimed)) +} diff --git a/backend/plugins/azuredevops_go/models/scope_config.go b/backend/plugins/azuredevops_go/models/scope_config.go index 94115df98e1..5f43e7978c2 100644 --- a/backend/plugins/azuredevops_go/models/scope_config.go +++ b/backend/plugins/azuredevops_go/models/scope_config.go @@ -31,6 +31,11 @@ type AzuredevopsScopeConfig struct { DeploymentPattern string `mapstructure:"deploymentPattern,omitempty" json:"deploymentPattern"` ProductionPattern string `mapstructure:"productionPattern,omitempty" json:"productionPattern"` Refdiff datatypes.JSONMap `mapstructure:"refdiff,omitempty" json:"refdiff" swaggertype:"object" format:"json"` + ProcessTemplate string `mapstructure:"processTemplate,omitempty" json:"processTemplate"` + TypeMappings datatypes.JSONMap `mapstructure:"typeMappings,omitempty" json:"typeMappings" swaggertype:"object" format:"json"` + StatusMappings datatypes.JSONMap `mapstructure:"statusMappings,omitempty" json:"statusMappings" swaggertype:"object" format:"json"` + CollectChangelogs bool `mapstructure:"collectChangelogs" json:"collectChangelogs"` + CollectComments bool `mapstructure:"collectComments" json:"collectComments"` } // GetConnectionId implements plugin.ToolLayerScopeConfig. diff --git a/backend/plugins/azuredevops_go/models/scope_entities.go b/backend/plugins/azuredevops_go/models/scope_entities.go new file mode 100644 index 00000000000..06f22b9f42f --- /dev/null +++ b/backend/plugins/azuredevops_go/models/scope_entities.go @@ -0,0 +1,95 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "github.com/apache/devlake/core/plugin" + +func ScopeEntities(scopeType string, configured []string) []string { + if len(configured) == 0 { + configured = plugin.DOMAIN_TYPES + } + allowed := map[string]struct{}{} + for _, entity := range allowedScopeEntities(scopeType) { + allowed[entity] = struct{}{} + } + entities := make([]string, 0, len(configured)) + for _, entity := range configured { + if _, ok := allowed[entity]; ok { + entities = append(entities, entity) + } + } + return entities +} + +func allowedScopeEntities(scopeType string) []string { + switch scopeType { + case RepositoryTypeProject: + return []string{plugin.DOMAIN_TYPE_TICKET, plugin.DOMAIN_TYPE_CODE_QUALITY} + case RepositoryTypeADO: + return []string{ + plugin.DOMAIN_TYPE_CODE, + plugin.DOMAIN_TYPE_TICKET, + plugin.DOMAIN_TYPE_CODE_REVIEW, + plugin.DOMAIN_TYPE_CROSS, + plugin.DOMAIN_TYPE_CICD, + plugin.DOMAIN_TYPE_CODE_QUALITY, + } + default: + return []string{ + plugin.DOMAIN_TYPE_CODE, + plugin.DOMAIN_TYPE_TICKET, + plugin.DOMAIN_TYPE_CICD, + plugin.DOMAIN_TYPE_CODE_QUALITY, + } + } +} + +func HasProjectEntity(entities []string) bool { + for _, entity := range entities { + if isProjectEntity(entity) { + return true + } + } + return false +} + +func DropProjectEntities(entities []string) []string { + kept := make([]string, 0, len(entities)) + for _, entity := range entities { + if !isProjectEntity(entity) { + kept = append(kept, entity) + } + } + return kept +} + +// ClaimProjectEntities keeps ticket and QA entities for the first scope of a project. +func ClaimProjectEntities(entities []string, projectKey string, claimed map[string]struct{}) []string { + if !HasProjectEntity(entities) || projectKey == "" { + return DropProjectEntities(entities) + } + if _, ok := claimed[projectKey]; ok { + return DropProjectEntities(entities) + } + claimed[projectKey] = struct{}{} + return entities +} + +func isProjectEntity(entity string) bool { + return entity == plugin.DOMAIN_TYPE_TICKET || entity == plugin.DOMAIN_TYPE_CODE_QUALITY +} diff --git a/backend/plugins/azuredevops_go/models/test_plan.go b/backend/plugins/azuredevops_go/models/test_plan.go new file mode 100644 index 00000000000..e88b948d29d --- /dev/null +++ b/backend/plugins/azuredevops_go/models/test_plan.go @@ -0,0 +1,107 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/devlake/core/models/common" +) + +type AzuredevopsTestPlan struct { + common.NoPKModel + + ConnectionId uint64 `gorm:"primaryKey"` + AzuredevopsId int `gorm:"primaryKey"` + OrgId string `gorm:"type:varchar(255)"` + ProjectId string `gorm:"type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + State string `gorm:"type:varchar(255)"` + RootSuiteId int + StartedDate *time.Time + EndedDate *time.Time + Url string `gorm:"type:varchar(500)"` +} + +func (AzuredevopsTestPlan) TableName() string { + return "_tool_azuredevops_go_test_plans" +} + +type AzuredevopsTestCase struct { + common.NoPKModel + + ConnectionId uint64 `gorm:"primaryKey"` + AzuredevopsId int `gorm:"primaryKey"` + PlanId int `gorm:"primaryKey"` + OrgId string `gorm:"type:varchar(255)"` + ProjectId string `gorm:"type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + WorkItemId int + Url string `gorm:"type:varchar(500)"` + CreatedDate *time.Time + CreatorId string `gorm:"type:varchar(255)"` +} + +func (AzuredevopsTestCase) TableName() string { + return "_tool_azuredevops_go_test_cases" +} + +type AzuredevopsTestRun struct { + common.NoPKModel + + ConnectionId uint64 `gorm:"primaryKey"` + AzuredevopsId int `gorm:"primaryKey"` + OrgId string `gorm:"type:varchar(255)"` + ProjectId string `gorm:"type:varchar(255)"` + PlanId int + Name string `gorm:"type:varchar(255)"` + State string `gorm:"type:varchar(255)"` + StartedDate *time.Time + CompletedDate *time.Time + TotalTests int + PassedTests int + Url string `gorm:"type:varchar(500)"` +} + +func (AzuredevopsTestRun) TableName() string { + return "_tool_azuredevops_go_test_runs" +} + +type AzuredevopsTestResult struct { + common.NoPKModel + + ConnectionId uint64 `gorm:"primaryKey"` + AzuredevopsId int `gorm:"primaryKey"` + RunId int `gorm:"primaryKey"` + OrgId string `gorm:"type:varchar(255)"` + ProjectId string `gorm:"type:varchar(255)"` + PlanId int + TestCaseId int + TestCaseTitle string `gorm:"type:varchar(255)"` + Outcome string `gorm:"type:varchar(255)"` + State string `gorm:"type:varchar(255)"` + StartedDate *time.Time + CompletedDate *time.Time + DurationMs int + Url string `gorm:"type:varchar(500)"` + CreatorId string `gorm:"type:varchar(255)"` +} + +func (AzuredevopsTestResult) TableName() string { + return "_tool_azuredevops_go_test_results" +} diff --git a/backend/plugins/azuredevops_go/models/work_item.go b/backend/plugins/azuredevops_go/models/work_item.go new file mode 100644 index 00000000000..ae22fa9b1a1 --- /dev/null +++ b/backend/plugins/azuredevops_go/models/work_item.go @@ -0,0 +1,103 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/devlake/core/models/common" +) + +// Description, relations, custom fields, comments, and changelog values use a size past 16MiB +// so MySQL creates LONGTEXT. TEXT is 64KiB there, and Azure payloads exceed it. Postgres maps this size to text. +type AzuredevopsWorkItem struct { + common.NoPKModel + + ConnectionId uint64 `gorm:"primaryKey"` + AzuredevopsId int `gorm:"primaryKey"` + OrgId string `gorm:"type:varchar(255)"` + ProjectId string `gorm:"type:varchar(255)"` + Title string + WorkItemType string `gorm:"type:varchar(255)"` + State string `gorm:"type:varchar(255)"` + Reason string `gorm:"type:varchar(255)"` + Priority string `gorm:"type:varchar(255)"` + Severity string `gorm:"type:varchar(255)"` + Description string `gorm:"size:16777217"` + Url string `gorm:"type:varchar(500)"` + CreatedDate *time.Time + UpdatedDate *time.Time + ClosedDate *time.Time + DueDate *time.Time + OriginalEstimateMinutes *int64 + TimeSpentMinutes *int64 + TimeRemainingMinutes *int64 + CreatorId string `gorm:"type:varchar(255)"` + CreatorName string `gorm:"type:varchar(255)"` + AssigneeId string `gorm:"type:varchar(255)"` + AssigneeName string `gorm:"type:varchar(255)"` + ParentId *int + StoryPoints *float64 + IterationPath string `gorm:"type:varchar(500)"` + AreaPath string `gorm:"type:varchar(500)"` + Tags string `gorm:"type:text"` + RelationsJson string `gorm:"size:16777217"` + CustomFieldsJson string `gorm:"size:16777217"` +} + +func (AzuredevopsWorkItem) TableName() string { + return "_tool_azuredevops_go_work_items" +} + +type AzuredevopsWorkItemComment struct { + common.NoPKModel + + ConnectionId uint64 `gorm:"primaryKey"` + WorkItemId int `gorm:"primaryKey"` + AzuredevopsId int `gorm:"primaryKey"` + ProjectId string `gorm:"type:varchar(255)"` + Body string `gorm:"size:16777217"` + AuthorId string `gorm:"type:varchar(255)"` + AuthorName string `gorm:"type:varchar(255)"` + CreatedDate *time.Time + UpdatedDate *time.Time +} + +func (AzuredevopsWorkItemComment) TableName() string { + return "_tool_azuredevops_go_work_item_comments" +} + +type AzuredevopsWorkItemUpdate struct { + common.NoPKModel + + ConnectionId uint64 `gorm:"primaryKey"` + WorkItemId int `gorm:"primaryKey"` + Revision int `gorm:"primaryKey"` + FieldRef string `gorm:"primaryKey;type:varchar(255)"` + ProjectId string `gorm:"type:varchar(255)"` + WorkItemType string `gorm:"type:varchar(255)"` + AuthorId string `gorm:"type:varchar(255)"` + AuthorName string `gorm:"type:varchar(255)"` + OriginalFrom string `gorm:"size:16777217"` + OriginalTo string `gorm:"size:16777217"` + CreatedDate *time.Time +} + +func (AzuredevopsWorkItemUpdate) TableName() string { + return "_tool_azuredevops_go_work_item_updates" +} diff --git a/backend/plugins/azuredevops_go/tasks/account_collector.go b/backend/plugins/azuredevops_go/tasks/account_collector.go index 0436c888657..6c80277f07d 100644 --- a/backend/plugins/azuredevops_go/tasks/account_collector.go +++ b/backend/plugins/azuredevops_go/tasks/account_collector.go @@ -21,6 +21,7 @@ import ( "github.com/apache/devlake/core/errors" "github.com/apache/devlake/core/plugin" "github.com/apache/devlake/helpers/pluginhelper/api" + "github.com/apache/devlake/plugins/azuredevops_go/models" ) func init() { @@ -48,12 +49,25 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { return nil } + connectionHelper := api.NewConnectionHelper(taskCtx, nil, "azuredevops_go") + connection := &models.AzuredevopsConnection{} + err := connectionHelper.FirstById(connection, data.Options.ConnectionId) + if err != nil { + return err + } + // Graph users are served by vssps, not the dev.azure.com client used for the rest of the collection. + graphConnection := *connection + graphConnection.Endpoint = "https://vssps.dev.azure.com/" + graphClient, err := CreateApiClient(taskCtx.TaskContext(), &graphConnection) + if err != nil { + return err + } + defer graphClient.Release() + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, - ApiClient: data.ApiClient, - // The Graph API can only be accessed through vssps.dev.azure.com. - // Since we cannot configure the connection host, we need to hardcode it - UrlTemplate: "https://vssps.dev.azure.com/{{ .Params.OrganizationId }}/_apis/graph/users?api-version=7.0-preview.1", + RawDataSubTaskArgs: *rawDataSubTaskArgs, + ApiClient: graphClient, + UrlTemplate: "{{ .Params.OrganizationId }}/_apis/graph/users?api-version=7.0-preview.1", PageSize: 500, GetNextPageCustomData: ExtractContToken, Query: BuildPaginator(true), diff --git a/backend/plugins/azuredevops_go/tasks/boards_common.go b/backend/plugins/azuredevops_go/tasks/boards_common.go new file mode 100644 index 00000000000..48e51ebde09 --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/boards_common.go @@ -0,0 +1,149 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/models/domainlayer/didgen" + "github.com/apache/devlake/helpers/pluginhelper/api" + "github.com/apache/devlake/plugins/azuredevops_go/models" +) + +func projectBoardID(data *AzuredevopsTaskData) string { + repo := models.AzuredevopsRepo{Id: data.Options.RepositoryId, Type: data.Options.RepositoryType} + repo.OrganizationId = data.Options.OrganizationId + repo.ProjectId = data.Options.ProjectId + return didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(data.Options.ConnectionId, repo.ProjectDomainScopeID()) +} + +func adoScopePrefix(data *AzuredevopsTaskData) string { + org := url.PathEscape(data.Options.OrganizationId) + project := url.PathEscape(data.Options.ProjectId) + if org != "" { + return org + "/" + project + } + return project +} + +func adoOrgPrefix(data *AzuredevopsTaskData) string { + if data.Options.OrganizationId == "" { + return "" + } + return url.PathEscape(data.Options.OrganizationId) + "/" +} + +func chunkInts(ids []int, size int) [][]int { + if size <= 0 || len(ids) == 0 { + return nil + } + chunks := make([][]int, 0, (len(ids)+size-1)/size) + for start := 0; start < len(ids); start += size { + end := start + size + if end > len(ids) { + end = len(ids) + } + chunks = append(chunks, ids[start:end]) + } + return chunks +} + +func joinInts(ids []int) string { + parts := make([]string, len(ids)) + for i, id := range ids { + parts[i] = fmt.Sprint(id) + } + return strings.Join(parts, ",") +} + +func readJSON(res *http.Response, target interface{}) errors.Error { + if res == nil { + return errors.Default.New("empty Azure DevOps response") + } + if res.StatusCode == http.StatusNonAuthoritativeInfo || res.StatusCode == http.StatusUnauthorized { + return errors.Unauthorized.New("authentication failed, please check your AccessToken") + } + if res.StatusCode >= 400 { + body, _ := io.ReadAll(res.Body) + return errors.Default.New(fmt.Sprintf("Azure DevOps API returned %d for %s: %s", res.StatusCode, res.Request.URL.String(), string(body))) + } + return api.UnmarshalResponse(res, target) +} + +type continuationBody struct { + Value []json.RawMessage `json:"value"` + ContinuationToken string `json:"continuationToken"` +} + +func collectContinuation(rawArgs *api.RawDataSubTaskArgs, client api.RateLimitedApiClient, urlTemplate string, query func(reqData *api.RequestData) (url.Values, errors.Error)) errors.Error { + var token string + collector, err := api.NewStatefulApiCollector(*rawArgs) + if err != nil { + return err + } + err = collector.InitCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: *rawArgs, + ApiClient: client, + PageSize: 1, + UrlTemplate: urlTemplate, + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + values, queryErr := query(reqData) + if queryErr != nil { + return nil, queryErr + } + if values == nil { + values = url.Values{} + } + if reqData.CustomData != nil { + values.Set("continuationToken", fmt.Sprint(reqData.CustomData)) + } + return values, nil + }, + GetNextPageCustomData: func(prevReqData *api.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) { + if token == "" { + return nil, api.ErrFinishCollect + } + next := token + token = "" + return next, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + body := &continuationBody{} + parseErr := api.UnmarshalResponse(res, body) + if parseErr != nil { + return nil, parseErr + } + token = body.ContinuationToken + if token == "" && res != nil { + token = res.Header.Get("X-Ms-Continuationtoken") + } + return body.Value, nil + }, + AfterResponse: change203To401, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/azuredevops_go/tasks/iteration.go b/backend/plugins/azuredevops_go/tasks/iteration.go new file mode 100644 index 00000000000..1ba5d7b629f --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/iteration.go @@ -0,0 +1,296 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/models/domainlayer" + "github.com/apache/devlake/core/models/domainlayer/didgen" + "github.com/apache/devlake/core/models/domainlayer/ticket" + "github.com/apache/devlake/core/plugin" + "github.com/apache/devlake/helpers/pluginhelper/api" + "github.com/apache/devlake/plugins/azuredevops_go/models" +) + +func init() { + RegisterSubtaskMeta(&CollectApiIterationsMeta) + RegisterSubtaskMeta(&ExtractApiIterationsMeta) + RegisterSubtaskMeta(&ConvertIterationsMeta) +} + +const RawIterationTable = "azuredevops_go_api_iterations" + +var CollectApiIterationsMeta = plugin.SubTaskMeta{ + Name: "collectApiIterations", + EntryPoint: CollectApiIterations, + EnabledByDefault: true, + Description: "Collect Azure Boards iterations and team time frames.", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + ProductTables: []string{RawIterationTable}, +} + +var ExtractApiIterationsMeta = plugin.SubTaskMeta{ + Name: "extractApiIterations", + EntryPoint: ExtractApiIterations, + EnabledByDefault: true, + Description: "Extract Azure Boards iterations into the tool layer.", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{RawIterationTable}, + ProductTables: []string{models.AzuredevopsIteration{}.TableName()}, +} + +var ConvertIterationsMeta = plugin.SubTaskMeta{ + Name: "convertIterations", + EntryPoint: ConvertIterations, + EnabledByDefault: true, + Description: "Convert Azure Boards iterations into sprints.", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.AzuredevopsIteration{}.TableName()}, + ProductTables: []string{ticket.Sprint{}.TableName(), ticket.BoardSprint{}.TableName()}, +} + +func CollectApiIterations(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + timeFrames := loadIterationTimeFrames(taskCtx, data) + rawArgs := &api.RawDataSubTaskArgs{Ctx: taskCtx, Table: RawIterationTable, Options: data.Options} + collector, err := api.NewStatefulApiCollector(*rawArgs) + if err != nil { + return err + } + err = collector.InitCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: *rawArgs, + ApiClient: data.ApiClient, + UrlTemplate: fmt.Sprintf("%s/_apis/wit/classificationnodes/Iterations", adoScopePrefix(data)), + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("$depth", "10") + query.Set("api-version", "7.1") + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var root map[string]interface{} + if parseErr := readJSON(res, &root); parseErr != nil { + return nil, parseErr + } + flat := flattenIterationNodes(root) + messages := make([]json.RawMessage, 0, len(flat)) + for _, node := range flat { + identifier := asString(node["identifier"]) + if identifier == "" { + continue + } + if frame := timeFrames[identifier]; frame != "" { + node["_timeFrame"] = frame + } + encoded, encodeErr := json.Marshal(node) + if encodeErr != nil { + return nil, errors.Convert(encodeErr) + } + messages = append(messages, encoded) + } + return messages, nil + }, + AfterResponse: change203To401, + }) + if err != nil { + return err + } + return collector.Execute() +} + +func flattenIterationNodes(node map[string]interface{}) []map[string]interface{} { + if node == nil { + return nil + } + children, _ := node["children"].([]interface{}) + delete(node, "children") + nodes := []map[string]interface{}{node} + for _, child := range children { + childNode, ok := child.(map[string]interface{}) + if !ok { + continue + } + nodes = append(nodes, flattenIterationNodes(childNode)...) + } + return nodes +} + +func loadIterationTimeFrames(taskCtx plugin.SubTaskContext, data *AzuredevopsTaskData) map[string]string { + frames := map[string]string{} + path := fmt.Sprintf("%s_apis/projects/%s/teams?api-version=7.1", adoOrgPrefix(data), url.PathEscape(data.Options.ProjectId)) + res, err := data.ApiClient.Get(path, nil, nil) + if err != nil { + taskCtx.GetLogger().Warn(err, "failed to list Azure DevOps teams for iteration time frames") + return frames + } + var teams struct { + Value []struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"value"` + } + if err = readJSON(res, &teams); err != nil { + taskCtx.GetLogger().Warn(err, "failed to parse Azure DevOps teams for iteration time frames") + return frames + } + rank := map[string]int{"current": 3, "future": 2, "past": 1} + for _, team := range teams.Value { + teamRef := team.ID + if teamRef == "" { + teamRef = team.Name + } + if teamRef == "" { + continue + } + teamPath := fmt.Sprintf("%s/%s/_apis/work/teamsettings/iterations?api-version=7.1", adoScopePrefix(data), url.PathEscape(teamRef)) + teamRes, teamErr := data.ApiClient.Get(teamPath, nil, nil) + if teamErr != nil { + taskCtx.GetLogger().Warn(teamErr, "failed to list iterations for team %s", teamRef) + continue + } + var body struct { + Value []struct { + ID string `json:"id"` + Path string `json:"path"` + Attributes struct { + TimeFrame string `json:"timeFrame"` + } `json:"attributes"` + } `json:"value"` + } + if teamErr = readJSON(teamRes, &body); teamErr != nil { + continue + } + for _, item := range body.Value { + frame := strings.ToLower(item.Attributes.TimeFrame) + if item.ID == "" || rank[frame] == 0 { + continue + } + if rank[frame] > rank[frames[item.ID]] { + frames[item.ID] = frame + } + } + } + return frames +} + +func ExtractApiIterations(taskCtx plugin.SubTaskContext) errors.Error { + rawArgs, data := CreateRawDataSubTaskArgs(taskCtx, RawIterationTable) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: *rawArgs, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var raw struct { + ID int `json:"id"` + Identifier string `json:"identifier"` + Name string `json:"name"` + Path string `json:"path"` + HasChildren bool `json:"hasChildren"` + Attributes map[string]interface{} `json:"attributes"` + TimeFrame string `json:"_timeFrame"` + } + if err := errors.Convert(json.Unmarshal(row.Data, &raw)); err != nil { + return nil, err + } + if raw.Identifier == "" { + return nil, nil + } + attributes := raw.Attributes + if attributes == nil { + attributes = map[string]interface{}{} + } + return []interface{}{&models.AzuredevopsIteration{ + ConnectionId: data.Options.ConnectionId, + Identifier: raw.Identifier, + NodeId: raw.ID, + OrgId: data.Options.OrganizationId, + ProjectId: data.Options.ProjectId, + Name: raw.Name, + Path: raw.Path, + PathKey: iterationPathKey(raw.Path, data.Options.ProjectId), + HasChildren: raw.HasChildren, + StartedDate: parseAdoTime(attributes["startDate"]), + EndedDate: parseAdoTime(attributes["finishDate"]), + TimeFrame: raw.TimeFrame, + }}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + +func ConvertIterations(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + iterationIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsIteration{}) + boardID := projectBoardID(data) + return convertProjectRows(taskCtx, models.AzuredevopsIteration{}.TableName(), &models.AzuredevopsIteration{}, func(row interface{}) ([]interface{}, errors.Error) { + iteration := row.(*models.AzuredevopsIteration) + if iteration.HasChildren || iteration.PathKey == "" { + return nil, nil + } + status := sprintStatus(iteration.TimeFrame, iteration.StartedDate, iteration.EndedDate) + var completed *time.Time + if status == "CLOSED" { + completed = iteration.EndedDate + } + sprintID := iterationIdGen.Generate(data.Options.ConnectionId, iteration.PathKey) + sprint := &ticket.Sprint{ + DomainEntity: domainlayer.NewDomainEntity(sprintID), + Name: iteration.Name, + Url: fmt.Sprintf("https://dev.azure.com/%s/%s/_settings/work-team?_a=iterations", iteration.OrgId, iteration.ProjectId), + Status: status, + StartedDate: iteration.StartedDate, + EndedDate: iteration.EndedDate, + CompletedDate: completed, + OriginalBoardID: boardID, + } + return []interface{}{ + sprint, + &ticket.BoardSprint{BoardId: boardID, SprintId: sprintID}, + }, nil + }) +} + +func sprintStatus(timeFrame string, started, ended *time.Time) string { + switch strings.ToLower(timeFrame) { + case "past": + return "CLOSED" + case "current": + return "ACTIVE" + case "future": + return "FUTURE" + } + now := time.Now().UTC() + if ended != nil && ended.Before(now) { + return "CLOSED" + } + if started != nil && started.After(now) { + return "FUTURE" + } + if started != nil { + return "ACTIVE" + } + return "" +} diff --git a/backend/plugins/azuredevops_go/tasks/repo_converter.go b/backend/plugins/azuredevops_go/tasks/repo_converter.go index 970ef5b0870..0878f70b005 100644 --- a/backend/plugins/azuredevops_go/tasks/repo_converter.go +++ b/backend/plugins/azuredevops_go/tasks/repo_converter.go @@ -41,7 +41,6 @@ var ConvertRepoMeta = plugin.SubTaskMeta{ Description: "Convert tool layer table _tool_azuredevops_go_repos into domain layer table repos and cicd scope", DomainTypes: []string{ plugin.DOMAIN_TYPE_CODE, - plugin.DOMAIN_TYPE_TICKET, plugin.DOMAIN_TYPE_CICD, plugin.DOMAIN_TYPE_CODE_REVIEW, plugin.DOMAIN_TYPE_CROSS}, @@ -55,6 +54,9 @@ var ConvertRepoMeta = plugin.SubTaskMeta{ func ConvertRepo(taskCtx plugin.SubTaskContext) errors.Error { rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, models.AzuredevopsRepo{}.TableName()) + if data.Options.RepositoryType == models.RepositoryTypeProject { + return nil + } db := taskCtx.GetDal() clauses := []dal.Clause{ dal.From(&models.AzuredevopsRepo{}), @@ -94,7 +96,7 @@ func convertToCicdScopeModel(repo *models.AzuredevopsRepo) *devops.CicdScope { DomainEntity: domainlayer.DomainEntity{ Id: didgen.NewDomainIdGenerator(repo).Generate(repo.ConnectionId, repo.Id), }, - Name: repo.ProjectId + "/" + repo.Name, + Name: repo.DomainName(), Url: repo.Url, Description: "", // Not Supported in Azure DevOps } @@ -107,7 +109,7 @@ func convertToRepositoryModel(repo *models.AzuredevopsRepo) *code.Repo { DomainEntity: domainlayer.DomainEntity{ Id: repoIdGen.Generate(repo.ConnectionId, repo.Id), }, - Name: repo.ProjectId + "/" + repo.Name, + Name: repo.DomainName(), Url: repo.Url, } return domainRepository diff --git a/backend/plugins/azuredevops_go/tasks/test_plans.go b/backend/plugins/azuredevops_go/tasks/test_plans.go new file mode 100644 index 00000000000..87a35388a96 --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/test_plans.go @@ -0,0 +1,606 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/url" + "time" + + "github.com/apache/devlake/core/dal" + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/models/domainlayer" + "github.com/apache/devlake/core/models/domainlayer/didgen" + "github.com/apache/devlake/core/models/domainlayer/qa" + "github.com/apache/devlake/core/plugin" + "github.com/apache/devlake/helpers/pluginhelper/api" + "github.com/apache/devlake/plugins/azuredevops_go/models" +) + +func init() { + RegisterSubtaskMeta(&CollectApiTestPlansMeta) + RegisterSubtaskMeta(&ExtractApiTestPlansMeta) + RegisterSubtaskMeta(&CollectApiTestCasesMeta) + RegisterSubtaskMeta(&ExtractApiTestCasesMeta) + RegisterSubtaskMeta(&ConvertTestCasesMeta) + RegisterSubtaskMeta(&CollectApiTestRunsMeta) + RegisterSubtaskMeta(&ExtractApiTestRunsMeta) + RegisterSubtaskMeta(&CollectApiTestResultsMeta) + RegisterSubtaskMeta(&ExtractApiTestResultsMeta) + RegisterSubtaskMeta(&ConvertTestResultsMeta) +} + +const ( + RawTestPlanTable = "azuredevops_go_api_test_plans" + RawTestCaseTable = "azuredevops_go_api_test_cases" + RawTestRunTable = "azuredevops_go_api_test_runs" + RawTestResultTable = "azuredevops_go_api_test_results" +) + +var CollectApiTestPlansMeta = plugin.SubTaskMeta{ + Name: "collectApiTestPlans", + EntryPoint: CollectApiTestPlans, + EnabledByDefault: true, + Description: "Collect Azure Test Plans.", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY}, + ProductTables: []string{RawTestPlanTable}, +} + +var ExtractApiTestPlansMeta = plugin.SubTaskMeta{ + Name: "extractApiTestPlans", + EntryPoint: ExtractApiTestPlans, + EnabledByDefault: true, + Description: "Extract Azure Test Plans into the tool layer.", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY}, + DependencyTables: []string{RawTestPlanTable}, + ProductTables: []string{models.AzuredevopsTestPlan{}.TableName()}, +} + +var CollectApiTestCasesMeta = plugin.SubTaskMeta{ + Name: "collectApiTestCases", + EntryPoint: CollectApiTestCases, + EnabledByDefault: true, + Description: "Collect test cases for each Azure Test Plan.", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY}, + DependencyTables: []string{models.AzuredevopsTestPlan{}.TableName()}, + ProductTables: []string{RawTestCaseTable}, +} + +var ExtractApiTestCasesMeta = plugin.SubTaskMeta{ + Name: "extractApiTestCases", + EntryPoint: ExtractApiTestCases, + EnabledByDefault: true, + Description: "Extract Azure test cases into the tool layer.", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY}, + DependencyTables: []string{RawTestCaseTable}, + ProductTables: []string{models.AzuredevopsTestCase{}.TableName()}, +} + +var ConvertTestCasesMeta = plugin.SubTaskMeta{ + Name: "convertTestCases", + EntryPoint: ConvertTestCases, + EnabledByDefault: true, + Description: "Convert Azure test cases into QA test cases.", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY}, + DependencyTables: []string{models.AzuredevopsTestCase{}.TableName()}, + ProductTables: []string{(&qa.QaTestCase{}).TableName(), (&qa.QaProject{}).TableName()}, +} + +var CollectApiTestRunsMeta = plugin.SubTaskMeta{ + Name: "collectApiTestRuns", + EntryPoint: CollectApiTestRuns, + EnabledByDefault: true, + Description: "Collect Azure test runs.", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY}, + ProductTables: []string{RawTestRunTable}, +} + +var ExtractApiTestRunsMeta = plugin.SubTaskMeta{ + Name: "extractApiTestRuns", + EntryPoint: ExtractApiTestRuns, + EnabledByDefault: true, + Description: "Extract Azure test runs into the tool layer.", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY}, + DependencyTables: []string{RawTestRunTable}, + ProductTables: []string{models.AzuredevopsTestRun{}.TableName()}, +} + +var CollectApiTestResultsMeta = plugin.SubTaskMeta{ + Name: "collectApiTestResults", + EntryPoint: CollectApiTestResults, + EnabledByDefault: true, + Description: "Collect results for each Azure test run.", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY}, + DependencyTables: []string{models.AzuredevopsTestRun{}.TableName()}, + ProductTables: []string{RawTestResultTable}, +} + +var ExtractApiTestResultsMeta = plugin.SubTaskMeta{ + Name: "extractApiTestResults", + EntryPoint: ExtractApiTestResults, + EnabledByDefault: true, + Description: "Extract Azure test results into the tool layer.", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY}, + DependencyTables: []string{RawTestResultTable}, + ProductTables: []string{models.AzuredevopsTestResult{}.TableName()}, +} + +var ConvertTestResultsMeta = plugin.SubTaskMeta{ + Name: "convertTestResults", + EntryPoint: ConvertTestResults, + EnabledByDefault: true, + Description: "Convert Azure test results into QA test case executions.", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY}, + DependencyTables: []string{models.AzuredevopsTestResult{}.TableName(), models.AzuredevopsTestCase{}.TableName()}, + ProductTables: []string{(&qa.QaTestCaseExecution{}).TableName()}, +} + +func CollectApiTestPlans(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + return collectContinuation( + &api.RawDataSubTaskArgs{Ctx: taskCtx, Table: RawTestPlanTable, Options: data.Options}, + data.ApiClient, + adoScopePrefix(data)+"/_apis/testplan/plans", + apiVersionQuery("7.1"), + ) +} + +func CollectApiTestRuns(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + return collectContinuation( + &api.RawDataSubTaskArgs{Ctx: taskCtx, Table: RawTestRunTable, Options: data.Options}, + data.ApiClient, + adoScopePrefix(data)+"/_apis/test/runs", + apiVersionQuery("7.1"), + ) +} + +func apiVersionQuery(version string) func(reqData *api.RequestData) (url.Values, errors.Error) { + return func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("api-version", version) + return query, nil + } +} + +type testCaseInput struct { + PlanId int + SuiteId int +} + +type testResultInput struct { + RunId int + PlanId int +} + +func CollectApiTestCases(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + var plans []models.AzuredevopsTestPlan + err := taskCtx.GetDal().All(&plans, dal.Where("connection_id = ? AND project_id = ? AND root_suite_id <> 0", data.Options.ConnectionId, data.Options.ProjectId)) + if err != nil { + return err + } + iterator := api.NewQueueIterator() + for _, plan := range plans { + iterator.Push(testCaseInput{PlanId: plan.AzuredevopsId, SuiteId: plan.RootSuiteId}) + } + rawArgs := &api.RawDataSubTaskArgs{Ctx: taskCtx, Table: RawTestCaseTable, Options: data.Options} + collector, err := api.NewStatefulApiCollector(*rawArgs) + if err != nil { + return err + } + err = collector.InitCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: *rawArgs, + ApiClient: data.ApiClient, + Input: iterator, + UrlTemplate: adoScopePrefix(data) + "/_apis/testplan/plans/{{ .Input.PlanId }}/Suites/{{ .Input.SuiteId }}/TestCase", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("isRecursive", "true") + query.Set("api-version", "7.1") + return query, nil + }, + ResponseParser: ParseRawMessageFromValue, + AfterResponse: change203To401, + }) + if err != nil { + return err + } + return collector.Execute() +} + +func CollectApiTestResults(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + var runs []models.AzuredevopsTestRun + err := taskCtx.GetDal().All(&runs, dal.Where("connection_id = ? AND project_id = ?", data.Options.ConnectionId, data.Options.ProjectId)) + if err != nil { + return err + } + iterator := api.NewQueueIterator() + for _, run := range runs { + iterator.Push(testResultInput{RunId: run.AzuredevopsId, PlanId: run.PlanId}) + } + rawArgs := &api.RawDataSubTaskArgs{Ctx: taskCtx, Table: RawTestResultTable, Options: data.Options} + collector, err := api.NewStatefulApiCollector(*rawArgs) + if err != nil { + return err + } + err = collector.InitCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: *rawArgs, + ApiClient: data.ApiClient, + Input: iterator, + UrlTemplate: adoScopePrefix(data) + "/_apis/test/runs/{{ .Input.RunId }}/results", + Query: apiVersionQuery("7.1"), + ResponseParser: ParseRawMessageFromValue, + AfterResponse: change203To401, + }) + if err != nil { + return err + } + return collector.Execute() +} + +func ExtractApiTestPlans(taskCtx plugin.SubTaskContext) errors.Error { + rawArgs, data := CreateRawDataSubTaskArgs(taskCtx, RawTestPlanTable) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: *rawArgs, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var raw struct { + ID int `json:"id"` + Name string `json:"name"` + State string `json:"state"` + URL string `json:"url"` + StartDate interface{} `json:"startDate"` + EndDate interface{} `json:"endDate"` + RootSuite struct { + ID int `json:"id"` + } `json:"rootSuite"` + } + if err := errors.Convert(json.Unmarshal(row.Data, &raw)); err != nil { + return nil, err + } + return []interface{}{&models.AzuredevopsTestPlan{ + ConnectionId: data.Options.ConnectionId, + AzuredevopsId: raw.ID, + OrgId: data.Options.OrganizationId, + ProjectId: data.Options.ProjectId, + Name: raw.Name, + State: raw.State, + RootSuiteId: raw.RootSuite.ID, + StartedDate: parseAdoTime(raw.StartDate), + EndedDate: parseAdoTime(raw.EndDate), + Url: raw.URL, + }}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + +func ExtractApiTestRuns(taskCtx plugin.SubTaskContext) errors.Error { + rawArgs, data := CreateRawDataSubTaskArgs(taskCtx, RawTestRunTable) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: *rawArgs, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var raw struct { + ID int `json:"id"` + Name string `json:"name"` + State string `json:"state"` + URL string `json:"url"` + StartedDate interface{} `json:"startedDate"` + CompletedDate interface{} `json:"completedDate"` + TotalTests int `json:"totalTests"` + PassedTests int `json:"passedTests"` + Plan struct { + ID int `json:"id"` + } `json:"plan"` + } + if err := errors.Convert(json.Unmarshal(row.Data, &raw)); err != nil { + return nil, err + } + return []interface{}{&models.AzuredevopsTestRun{ + ConnectionId: data.Options.ConnectionId, + AzuredevopsId: raw.ID, + OrgId: data.Options.OrganizationId, + ProjectId: data.Options.ProjectId, + PlanId: raw.Plan.ID, + Name: raw.Name, + State: raw.State, + StartedDate: parseAdoTime(raw.StartedDate), + CompletedDate: parseAdoTime(raw.CompletedDate), + TotalTests: raw.TotalTests, + PassedTests: raw.PassedTests, + Url: raw.URL, + }}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + +func ExtractApiTestCases(taskCtx plugin.SubTaskContext) errors.Error { + rawArgs, data := CreateRawDataSubTaskArgs(taskCtx, RawTestCaseTable) + creators := loadTestCaseCreators(taskCtx, data) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: *rawArgs, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var raw struct { + ID int `json:"id"` + Name string `json:"name"` + WorkItem struct { + ID int `json:"id"` + Name string `json:"name"` + URL string `json:"url"` + } `json:"workItem"` + } + if err := errors.Convert(json.Unmarshal(row.Data, &raw)); err != nil { + return nil, err + } + var input testCaseInput + _ = json.Unmarshal(row.Input, &input) + workItemID := raw.WorkItem.ID + if workItemID == 0 { + workItemID = raw.ID + } + name := raw.WorkItem.Name + if name == "" { + name = raw.Name + } + creator := creators[workItemID] + return []interface{}{&models.AzuredevopsTestCase{ + ConnectionId: data.Options.ConnectionId, + AzuredevopsId: workItemID, + PlanId: input.PlanId, + OrgId: data.Options.OrganizationId, + ProjectId: data.Options.ProjectId, + Name: name, + WorkItemId: workItemID, + Url: raw.WorkItem.URL, + CreatedDate: creator.Created, + CreatorId: creator.CreatorId, + }}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + +type testCaseCreator struct { + Created *time.Time + CreatorId string +} + +func loadTestCaseCreators(taskCtx plugin.SubTaskContext, data *AzuredevopsTaskData) map[int]testCaseCreator { + creators := map[int]testCaseCreator{} + rawSub, err := api.NewRawDataSubTask(api.RawDataSubTaskArgs{Ctx: taskCtx, Table: RawTestCaseTable, Options: data.Options}) + if err != nil { + taskCtx.GetLogger().Warn(err, "skipping test case creator lookup") + return creators + } + var rows []api.RawData + if err = taskCtx.GetDal().All(&rows, dal.From(rawSub.GetTable()), dal.Where("params = ?", rawSub.GetParams())); err != nil { + taskCtx.GetLogger().Warn(err, "skipping test case creator lookup") + return creators + } + seen := map[int]struct{}{} + ids := make([]int, 0, len(rows)) + for _, row := range rows { + var raw struct { + ID int `json:"id"` + WorkItem struct { + ID int `json:"id"` + } `json:"workItem"` + } + if json.Unmarshal(row.Data, &raw) != nil { + continue + } + id := raw.WorkItem.ID + if id == 0 { + id = raw.ID + } + if id == 0 { + continue + } + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + ids = append(ids, id) + } + for _, chunk := range chunkInts(ids, 200) { + query := url.Values{} + query.Set("ids", joinInts(chunk)) + query.Set("fields", "System.CreatedDate,System.CreatedBy") + query.Set("api-version", "7.1") + res, getErr := data.ApiClient.Get(adoOrgPrefix(data)+"_apis/wit/workitems", query, nil) + if getErr != nil { + taskCtx.GetLogger().Warn(getErr, "could not load test case work items") + return creators + } + var body struct { + Value []struct { + ID int `json:"id"` + Fields map[string]interface{} `json:"fields"` + } `json:"value"` + } + if getErr = readJSON(res, &body); getErr != nil { + taskCtx.GetLogger().Warn(getErr, "could not load test case work items") + return creators + } + for _, item := range body.Value { + creatorID, _ := identityFromRef(item.Fields["System.CreatedBy"]) + creators[item.ID] = testCaseCreator{ + Created: parseAdoTime(item.Fields["System.CreatedDate"]), + CreatorId: creatorID, + } + } + } + return creators +} + +type adoTestResult struct { + ID int `json:"id"` + Outcome string `json:"outcome"` + State string `json:"state"` + URL string `json:"url"` + StartedDate interface{} `json:"startedDate"` + CompletedDate interface{} `json:"completedDate"` + DurationInMs *float64 `json:"durationInMs"` + TestCaseTitle string `json:"testCaseTitle"` + TestCaseReferenceId int `json:"testCaseReferenceId"` + TestCase struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"testCase"` + TestPlan struct { + ID string `json:"id"` + } `json:"testPlan"` + RunBy map[string]interface{} `json:"runBy"` + LastUpdatedBy map[string]interface{} `json:"lastUpdatedBy"` + Owner map[string]interface{} `json:"owner"` +} + +func ExtractApiTestResults(taskCtx plugin.SubTaskContext) errors.Error { + rawArgs, data := CreateRawDataSubTaskArgs(taskCtx, RawTestResultTable) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: *rawArgs, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var raw adoTestResult + if err := errors.Convert(json.Unmarshal(row.Data, &raw)); err != nil { + return nil, err + } + var input testResultInput + _ = json.Unmarshal(row.Input, &input) + caseID := asInt(raw.TestCase.ID) + if caseID == 0 { + caseID = raw.TestCaseReferenceId + } + planID := asInt(raw.TestPlan.ID) + if planID == 0 { + planID = input.PlanId + } + title := raw.TestCaseTitle + if title == "" { + title = raw.TestCase.Name + } + executor, _ := identityFromRef(firstIdentity(raw.RunBy, raw.LastUpdatedBy, raw.Owner)) + durationMs := 0 + if raw.DurationInMs != nil { + durationMs = int(*raw.DurationInMs) + } + return []interface{}{&models.AzuredevopsTestResult{ + ConnectionId: data.Options.ConnectionId, + AzuredevopsId: raw.ID, + RunId: input.RunId, + OrgId: data.Options.OrganizationId, + ProjectId: data.Options.ProjectId, + PlanId: planID, + TestCaseId: caseID, + TestCaseTitle: title, + Outcome: raw.Outcome, + State: raw.State, + StartedDate: parseAdoTime(raw.StartedDate), + CompletedDate: parseAdoTime(raw.CompletedDate), + DurationMs: durationMs, + Url: raw.URL, + CreatorId: executor, + }}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + +func firstIdentity(values ...map[string]interface{}) interface{} { + for _, value := range values { + if len(value) > 0 { + return value + } + } + return nil +} + +func ConvertTestCases(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + caseIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsTestCase{}) + projectID := projectBoardID(data) + qaProject := &qa.QaProject{ + DomainEntityExtended: domainlayer.DomainEntityExtended{Id: projectID}, + Name: data.Options.ProjectId, + } + return convertProjectRows(taskCtx, models.AzuredevopsTestCase{}.TableName(), &models.AzuredevopsTestCase{}, func(row interface{}) ([]interface{}, errors.Error) { + testCase := row.(*models.AzuredevopsTestCase) + var created time.Time + if testCase.CreatedDate != nil { + created = *testCase.CreatedDate + } + return []interface{}{ + qaProject, + &qa.QaTestCase{ + DomainEntityExtended: domainlayer.DomainEntityExtended{ + Id: caseIdGen.Generate(data.Options.ConnectionId, testCase.AzuredevopsId, testCase.PlanId), + }, + Name: testCase.Name, + Type: "functional", + CreateTime: created, + CreatorId: testCase.CreatorId, + QaProjectId: projectID, + }, + }, nil + }) +} + +func ConvertTestResults(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + caseIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsTestCase{}) + resultIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsTestResult{}) + projectID := projectBoardID(data) + return convertProjectRows(taskCtx, models.AzuredevopsTestResult{}.TableName(), &models.AzuredevopsTestResult{}, func(row interface{}) ([]interface{}, errors.Error) { + result := row.(*models.AzuredevopsTestResult) + if result.TestCaseId == 0 || result.PlanId == 0 { + return nil, nil + } + var started, finished time.Time + if result.StartedDate != nil { + started = *result.StartedDate + } + if result.CompletedDate != nil { + finished = *result.CompletedDate + } + return []interface{}{&qa.QaTestCaseExecution{ + DomainEntityExtended: domainlayer.DomainEntityExtended{ + Id: resultIdGen.Generate(data.Options.ConnectionId, result.RunId, result.AzuredevopsId), + }, + QaProjectId: projectID, + QaTestCaseId: caseIdGen.Generate(data.Options.ConnectionId, result.TestCaseId, result.PlanId), + CreateTime: started, + StartTime: started, + FinishTime: finished, + CreatorId: result.CreatorId, + Status: mapTestOutcome(result.Outcome), + }}, nil + }) +} diff --git a/backend/plugins/azuredevops_go/tasks/test_plans_test.go b/backend/plugins/azuredevops_go/tasks/test_plans_test.go new file mode 100644 index 00000000000..3a24527fad6 --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/test_plans_test.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTestResultReferenceIDsAreStrings(t *testing.T) { + var result adoTestResult + err := json.Unmarshal([]byte(`{ + "id": 501, + "testCase": {"id": "100", "name": "Verify checkout"}, + "testPlan": {"id": "7"}, + "testCaseReferenceId": 100, + "durationInMs": 5076.0 + }`), &result) + assert.NoError(t, err) + assert.Equal(t, "100", result.TestCase.ID) + assert.Equal(t, "7", result.TestPlan.ID) + assert.Equal(t, 100, result.TestCaseReferenceId) + assert.Equal(t, 100, asInt(result.TestCase.ID)) + assert.Equal(t, 7, asInt(result.TestPlan.ID)) + if assert.NotNil(t, result.DurationInMs) { + assert.Equal(t, 5076, int(*result.DurationInMs)) + } +} diff --git a/backend/plugins/azuredevops_go/tasks/work_item_children.go b/backend/plugins/azuredevops_go/tasks/work_item_children.go new file mode 100644 index 00000000000..e69b5d5bf7b --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/work_item_children.go @@ -0,0 +1,305 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "sync" + + "github.com/apache/devlake/core/dal" + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/models/domainlayer" + "github.com/apache/devlake/core/models/domainlayer/didgen" + "github.com/apache/devlake/core/models/domainlayer/ticket" + "github.com/apache/devlake/core/plugin" + "github.com/apache/devlake/helpers/pluginhelper/api" + "github.com/apache/devlake/plugins/azuredevops_go/models" +) + +func init() { + RegisterSubtaskMeta(&CollectApiWorkItemCommentsMeta) + RegisterSubtaskMeta(&ExtractApiWorkItemCommentsMeta) + RegisterSubtaskMeta(&ConvertWorkItemCommentsMeta) + RegisterSubtaskMeta(&CollectApiWorkItemUpdatesMeta) + RegisterSubtaskMeta(&ExtractApiWorkItemUpdatesMeta) + RegisterSubtaskMeta(&ConvertWorkItemUpdatesMeta) +} + +const ( + RawWorkItemCommentTable = "azuredevops_go_api_work_item_comments" + RawWorkItemUpdateTable = "azuredevops_go_api_work_item_updates" +) + +var CollectApiWorkItemCommentsMeta = plugin.SubTaskMeta{ + Name: "collectApiWorkItemComments", + EntryPoint: CollectApiWorkItemComments, + EnabledByDefault: true, + Description: "Collect Azure Boards work item comments when the scope config asks for them.", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.AzuredevopsWorkItem{}.TableName()}, + ProductTables: []string{RawWorkItemCommentTable}, +} + +var ExtractApiWorkItemCommentsMeta = plugin.SubTaskMeta{ + Name: "extractApiWorkItemComments", + EntryPoint: ExtractApiWorkItemComments, + EnabledByDefault: true, + Description: "Extract Azure Boards work item comments into the tool layer.", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{RawWorkItemCommentTable}, + ProductTables: []string{models.AzuredevopsWorkItemComment{}.TableName()}, +} + +var ConvertWorkItemCommentsMeta = plugin.SubTaskMeta{ + Name: "convertWorkItemComments", + EntryPoint: ConvertWorkItemComments, + EnabledByDefault: true, + Description: "Convert Azure Boards work item comments into issue comments.", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.AzuredevopsWorkItemComment{}.TableName()}, + ProductTables: []string{ticket.IssueComment{}.TableName()}, +} + +var CollectApiWorkItemUpdatesMeta = plugin.SubTaskMeta{ + Name: "collectApiWorkItemUpdates", + EntryPoint: CollectApiWorkItemUpdates, + EnabledByDefault: true, + Description: "Collect Azure Boards work item revisions when the scope config asks for changelogs.", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.AzuredevopsWorkItem{}.TableName()}, + ProductTables: []string{RawWorkItemUpdateTable}, +} + +var ExtractApiWorkItemUpdatesMeta = plugin.SubTaskMeta{ + Name: "extractApiWorkItemUpdates", + EntryPoint: ExtractApiWorkItemUpdates, + EnabledByDefault: true, + Description: "Extract mapped Azure Boards field changes into the tool layer.", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{RawWorkItemUpdateTable}, + ProductTables: []string{models.AzuredevopsWorkItemUpdate{}.TableName()}, +} + +var ConvertWorkItemUpdatesMeta = plugin.SubTaskMeta{ + Name: "convertWorkItemUpdates", + EntryPoint: ConvertWorkItemUpdates, + EnabledByDefault: true, + Description: "Convert Azure Boards field changes into issue changelogs.", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.AzuredevopsWorkItemUpdate{}.TableName()}, + ProductTables: []string{ticket.IssueChangelogs{}.TableName()}, +} + +type workItemRefInput struct { + WorkItemId int +} + +func CollectApiWorkItemComments(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + if data.Options.ScopeConfig == nil || !data.Options.ScopeConfig.CollectComments { + return nil + } + return collectWorkItemChildren(taskCtx, data, RawWorkItemCommentTable, "comments", "7.1-preview.3", true) +} + +func CollectApiWorkItemUpdates(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + if data.Options.ScopeConfig == nil || !data.Options.ScopeConfig.CollectChangelogs { + return nil + } + return collectWorkItemChildren(taskCtx, data, RawWorkItemUpdateTable, "updates", "7.1", false) +} + +func collectWorkItemChildren(taskCtx plugin.SubTaskContext, data *AzuredevopsTaskData, table, suffix, apiVersion string, comments bool) errors.Error { + ids, err := listWorkItemIDs(taskCtx, data) + if err != nil { + return err + } + rawArgs := &api.RawDataSubTaskArgs{Ctx: taskCtx, Table: table, Options: data.Options} + iterator := api.NewQueueIterator() + for _, id := range ids { + iterator.Push(workItemRefInput{WorkItemId: id}) + } + var tokens sync.Map + collector, err := api.NewStatefulApiCollector(*rawArgs) + if err != nil { + return err + } + err = collector.InitCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: *rawArgs, + ApiClient: data.ApiClient, + Input: iterator, + PageSize: 1, + UrlTemplate: fmt.Sprintf("%s/_apis/wit/workitems/{{ .Input.WorkItemId }}/%s", adoScopePrefix(data), suffix), + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("api-version", apiVersion) + if reqData.CustomData != nil { + query.Set("continuationToken", fmt.Sprint(reqData.CustomData)) + } + return query, nil + }, + GetNextPageCustomData: func(prevReqData *api.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) { + input := prevReqData.Input.(workItemRefInput) + token, _ := tokens.Load(input.WorkItemId) + text, _ := token.(string) + if text == "" { + return nil, api.ErrFinishCollect + } + tokens.Delete(input.WorkItemId) + return text, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + if comments { + var body struct { + Comments []json.RawMessage `json:"comments"` + ContinuationToken string `json:"continuationToken"` + } + if parseErr := api.UnmarshalResponse(res, &body); parseErr != nil { + return nil, parseErr + } + if id := workItemIDFromRequest(res); id != 0 && body.ContinuationToken != "" { + tokens.Store(id, body.ContinuationToken) + } + return body.Comments, nil + } + var body continuationBody + if parseErr := api.UnmarshalResponse(res, &body); parseErr != nil { + return nil, parseErr + } + if id := workItemIDFromRequest(res); id != 0 && body.ContinuationToken != "" { + tokens.Store(id, body.ContinuationToken) + } + return body.Value, nil + }, + AfterResponse: change203To401, + }) + if err != nil { + return err + } + return collector.Execute() +} + +func workItemIDFromRequest(res *http.Response) int { + if res == nil || res.Request == nil || res.Request.URL == nil { + return 0 + } + match := workItemIDFromURL.FindStringSubmatch(res.Request.URL.Path) + if len(match) < 2 { + return 0 + } + return asInt(match[1]) +} + +func listWorkItemIDs(taskCtx plugin.SubTaskContext, data *AzuredevopsTaskData) ([]int, errors.Error) { + var rows []models.AzuredevopsWorkItem + err := taskCtx.GetDal().All( + &rows, + dal.Select("azuredevops_id, work_item_type"), + dal.From(&models.AzuredevopsWorkItem{}), + dal.Where("connection_id = ? AND project_id = ?", data.Options.ConnectionId, data.Options.ProjectId), + ) + if err != nil { + return nil, err + } + ids := make([]int, 0, len(rows)) + for _, row := range rows { + if shouldConvertWorkItem(row.WorkItemType) { + ids = append(ids, row.AzuredevopsId) + } + } + return ids, nil +} + +func ExtractApiWorkItemComments(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + if data.Options.ScopeConfig == nil || !data.Options.ScopeConfig.CollectComments { + return nil + } + rawArgs, data := CreateRawDataSubTaskArgs(taskCtx, RawWorkItemCommentTable) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: *rawArgs, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var raw struct { + ID int `json:"id"` + Text string `json:"text"` + IsDeleted bool `json:"isDeleted"` + CreatedBy map[string]interface{} `json:"createdBy"` + CreatedDate interface{} `json:"createdDate"` + ModifiedDate interface{} `json:"modifiedDate"` + WorkItemId int `json:"workItemId"` + } + if err := errors.Convert(json.Unmarshal(row.Data, &raw)); err != nil { + return nil, err + } + if raw.IsDeleted || raw.Text == "" { + return nil, nil + } + workItemID := raw.WorkItemId + if workItemID == 0 { + var input workItemRefInput + _ = json.Unmarshal(row.Input, &input) + workItemID = input.WorkItemId + } + authorID, authorName := identityFromRef(raw.CreatedBy) + return []interface{}{&models.AzuredevopsWorkItemComment{ + ConnectionId: data.Options.ConnectionId, + WorkItemId: workItemID, + AzuredevopsId: raw.ID, + ProjectId: data.Options.ProjectId, + Body: raw.Text, + AuthorId: authorID, + AuthorName: authorName, + CreatedDate: parseAdoTime(raw.CreatedDate), + UpdatedDate: parseAdoTime(raw.ModifiedDate), + }}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + +func ConvertWorkItemComments(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + if data.Options.ScopeConfig == nil || !data.Options.ScopeConfig.CollectComments { + return nil + } + return convertProjectRows(taskCtx, models.AzuredevopsWorkItemComment{}.TableName(), &models.AzuredevopsWorkItemComment{}, func(row interface{}) ([]interface{}, errors.Error) { + comment := row.(*models.AzuredevopsWorkItemComment) + if comment.Body == "" { + return nil, nil + } + issueID := didgen.NewDomainIdGenerator(&models.AzuredevopsWorkItem{}).Generate(data.Options.ConnectionId, comment.WorkItemId) + commentID := didgen.NewDomainIdGenerator(&models.AzuredevopsWorkItemComment{}).Generate(data.Options.ConnectionId, comment.WorkItemId, comment.AzuredevopsId) + created := comment.CreatedDate + var createdValue = commentCreated(created) + return []interface{}{&ticket.IssueComment{ + DomainEntity: domainlayer.NewDomainEntity(commentID), + IssueId: issueID, + Body: comment.Body, + AccountId: comment.AuthorId, + CreatedDate: createdValue, + UpdatedDate: comment.UpdatedDate, + }}, nil + }) +} diff --git a/backend/plugins/azuredevops_go/tasks/work_item_collector.go b/backend/plugins/azuredevops_go/tasks/work_item_collector.go new file mode 100644 index 00000000000..b6625cb4683 --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/work_item_collector.go @@ -0,0 +1,115 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "net/url" + "time" + + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/plugin" + "github.com/apache/devlake/helpers/pluginhelper/api" +) + +func init() { + RegisterSubtaskMeta(&CollectApiWorkItemsMeta) +} + +const RawWorkItemTable = "azuredevops_go_api_work_items" + +var CollectApiWorkItemsMeta = plugin.SubTaskMeta{ + Name: "collectApiWorkItems", + EntryPoint: CollectApiWorkItems, + EnabledByDefault: true, + Description: "Collect Azure Boards work items with a WIQL query, then batch-load each work item.", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + ProductTables: []string{RawWorkItemTable}, +} + +func CollectApiWorkItems(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + rawArgs := &api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: RawWorkItemTable, + Options: data.Options, + } + collector, err := api.NewStatefulApiCollector(*rawArgs) + if err != nil { + return err + } + + var createdAfter, changedSince *time.Time + if collector.IsIncremental() { + changedSince = collector.GetSince() + } else { + createdAfter = collector.GetSince() + } + ids, err := queryWorkItemIds(data, createdAfter, changedSince) + if err != nil { + return err + } + iterator := api.NewQueueIterator() + for _, chunk := range chunkInts(ids, 200) { + iterator.Push(chunk) + } + + err = collector.InitCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: *rawArgs, + ApiClient: data.ApiClient, + Input: iterator, + UrlTemplate: adoOrgPrefix(data) + "_apis/wit/workitems", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("ids", joinInts(reqData.Input.([]int))) + query.Set("$expand", "all") + query.Set("api-version", "7.1") + return query, nil + }, + ResponseParser: ParseRawMessageFromValue, + AfterResponse: change203To401, + }) + if err != nil { + return err + } + return collector.Execute() +} + +func queryWorkItemIds(data *AzuredevopsTaskData, createdAfter, changedSince *time.Time) ([]int, errors.Error) { + query := projectWorkItemsWIQL(data.Options.ProjectId, createdAfter, changedSince) + path := fmt.Sprintf("%s/_apis/wit/wiql?api-version=7.1", adoScopePrefix(data)) + res, err := data.ApiClient.Post(path, nil, map[string]string{"query": query}, nil) + if err != nil { + return nil, err + } + var body struct { + WorkItems []struct { + ID int `json:"id"` + } `json:"workItems"` + } + if err = readJSON(res, &body); err != nil { + return nil, err + } + ids := make([]int, 0, len(body.WorkItems)) + for _, item := range body.WorkItems { + if item.ID != 0 { + ids = append(ids, item.ID) + } + } + return ids, nil +} diff --git a/backend/plugins/azuredevops_go/tasks/work_item_convert.go b/backend/plugins/azuredevops_go/tasks/work_item_convert.go new file mode 100644 index 00000000000..c1b2bfc1a73 --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/work_item_convert.go @@ -0,0 +1,407 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "reflect" + "regexp" + "strings" + "time" + + "github.com/apache/devlake/core/dal" + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/models/domainlayer" + "github.com/apache/devlake/core/models/domainlayer/didgen" + "github.com/apache/devlake/core/models/domainlayer/ticket" + "github.com/apache/devlake/core/plugin" + "github.com/apache/devlake/helpers/pluginhelper/api" + "github.com/apache/devlake/plugins/azuredevops_go/models" + "gorm.io/datatypes" +) + +func init() { + RegisterSubtaskMeta(&ExtractApiWorkItemsMeta) + RegisterSubtaskMeta(&ConvertWorkItemsMeta) +} + +var workItemIDFromURL = regexp.MustCompile(`(?i)/workitems/(\d+)`) + +var mappedWorkItemFields = map[string]struct{}{ + "System.Title": {}, "System.WorkItemType": {}, "System.State": {}, "System.Reason": {}, + "System.Description": {}, "System.CreatedDate": {}, "System.ChangedDate": {}, + "System.CreatedBy": {}, "System.AssignedTo": {}, "System.Parent": {}, + "System.IterationPath": {}, "System.AreaPath": {}, "System.Tags": {}, "System.TeamProject": {}, + "Microsoft.VSTS.Common.Priority": {}, "Microsoft.VSTS.Common.Severity": {}, "Microsoft.VSTS.Common.ClosedDate": {}, + "Microsoft.VSTS.Scheduling.TargetDate": {}, "Microsoft.VSTS.Scheduling.DueDate": {}, + "Microsoft.VSTS.Scheduling.OriginalEstimate": {}, "Microsoft.VSTS.Scheduling.CompletedWork": {}, + "Microsoft.VSTS.Scheduling.RemainingWork": {}, "Microsoft.VSTS.Scheduling.StoryPoints": {}, + "Microsoft.VSTS.Scheduling.Effort": {}, "Microsoft.VSTS.Scheduling.Size": {}, +} + +var ExtractApiWorkItemsMeta = plugin.SubTaskMeta{ + Name: "extractApiWorkItems", + EntryPoint: ExtractApiWorkItems, + EnabledByDefault: true, + Description: "Extract raw Azure Boards work items into the tool layer.", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{RawWorkItemTable}, + ProductTables: []string{models.AzuredevopsWorkItem{}.TableName()}, +} + +var ConvertWorkItemsMeta = plugin.SubTaskMeta{ + Name: "convertWorkItems", + EntryPoint: ConvertWorkItems, + EnabledByDefault: true, + Description: "Convert Azure Boards work items into domain issues, labels, and sprint links.", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{ + models.AzuredevopsWorkItem{}.TableName(), + models.AzuredevopsIteration{}.TableName(), + }, + ProductTables: []string{ + ticket.Issue{}.TableName(), + ticket.BoardIssue{}.TableName(), + ticket.SprintIssue{}.TableName(), + ticket.IssueAssignee{}.TableName(), + ticket.IssueLabel{}.TableName(), + ticket.IssueRelationship{}.TableName(), + }, +} + +type adoWorkItem struct { + ID int `json:"id"` + URL string `json:"url"` + Fields map[string]interface{} `json:"fields"` + Relations []map[string]interface{} `json:"relations"` +} + +func ExtractApiWorkItems(taskCtx plugin.SubTaskContext) errors.Error { + rawArgs, data := CreateRawDataSubTaskArgs(taskCtx, RawWorkItemTable) + displayNames := loadWorkItemFieldDisplayNames(taskCtx, data) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: *rawArgs, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + raw := &adoWorkItem{} + if err := errors.Convert(json.Unmarshal(row.Data, raw)); err != nil { + return nil, err + } + fields := raw.Fields + if fields == nil { + fields = map[string]interface{}{} + } + assigneeID, assigneeName := identityFromRef(fields["System.AssignedTo"]) + creatorID, creatorName := identityFromRef(fields["System.CreatedBy"]) + var parentID *int + if fields["System.Parent"] != nil { + parent := asInt(fields["System.Parent"]) + if parent != 0 { + parentID = &parent + } + } + relations, _ := json.Marshal(raw.Relations) + relationsJSON := "" + if len(raw.Relations) > 0 { + relationsJSON = string(relations) + } + url := raw.URL + if url == "" { + url = "https://dev.azure.com/" + data.Options.OrganizationId + "/_workitems/edit/" + asString(raw.ID) + } + return []interface{}{&models.AzuredevopsWorkItem{ + ConnectionId: data.Options.ConnectionId, + AzuredevopsId: raw.ID, + OrgId: data.Options.OrganizationId, + ProjectId: firstNonEmpty(asString(fields["System.TeamProject"]), data.Options.ProjectId), + Title: asString(fields["System.Title"]), + WorkItemType: asString(fields["System.WorkItemType"]), + State: asString(fields["System.State"]), + Reason: asString(fields["System.Reason"]), + Priority: asString(fields["Microsoft.VSTS.Common.Priority"]), + Severity: asString(fields["Microsoft.VSTS.Common.Severity"]), + Description: asString(fields["System.Description"]), + Url: url, + CreatedDate: parseAdoTime(fields["System.CreatedDate"]), + UpdatedDate: parseAdoTime(fields["System.ChangedDate"]), + ClosedDate: parseAdoTime(fields["Microsoft.VSTS.Common.ClosedDate"]), + DueDate: firstTime(fields["Microsoft.VSTS.Scheduling.TargetDate"], fields["Microsoft.VSTS.Scheduling.DueDate"]), + OriginalEstimateMinutes: hoursToMinutes(fields["Microsoft.VSTS.Scheduling.OriginalEstimate"]), + TimeSpentMinutes: hoursToMinutes(fields["Microsoft.VSTS.Scheduling.CompletedWork"]), + TimeRemainingMinutes: hoursToMinutes(fields["Microsoft.VSTS.Scheduling.RemainingWork"]), + CreatorId: creatorID, + CreatorName: creatorName, + AssigneeId: assigneeID, + AssigneeName: assigneeName, + ParentId: parentID, + StoryPoints: firstFloat(fields, "Microsoft.VSTS.Scheduling.StoryPoints", "Microsoft.VSTS.Scheduling.Effort", "Microsoft.VSTS.Scheduling.Size"), + IterationPath: asString(fields["System.IterationPath"]), + AreaPath: asString(fields["System.AreaPath"]), + Tags: asString(fields["System.Tags"]), + RelationsJson: relationsJSON, + CustomFieldsJson: customFieldsJSON(fields, displayNames), + }}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + +func ConvertWorkItems(taskCtx plugin.SubTaskContext) errors.Error { + rawArgs, data := CreateRawDataSubTaskArgs(taskCtx, models.AzuredevopsWorkItem{}.TableName()) + mapper := newWorkItemMapper(taskCtx, data) + var typeMappings datatypes.JSONMap + var statusMappings datatypes.JSONMap + if data.Options.ScopeConfig != nil { + typeMappings = data.Options.ScopeConfig.TypeMappings + statusMappings = data.Options.ScopeConfig.StatusMappings + } + workItemIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsWorkItem{}) + iterationIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsIteration{}) + boardID := projectBoardID(data) + db := taskCtx.GetDal() + cursor, err := db.Cursor( + dal.From(&models.AzuredevopsWorkItem{}), + dal.Where("connection_id = ? AND project_id = ?", data.Options.ConnectionId, data.Options.ProjectId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := api.NewDataConverter(api.DataConverterArgs{ + InputRowType: reflect.TypeOf(models.AzuredevopsWorkItem{}), + Input: cursor, + RawDataSubTaskArgs: *rawArgs, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + workItem := inputRow.(*models.AzuredevopsWorkItem) + if !shouldConvertWorkItem(workItem.WorkItemType) { + return nil, nil + } + mapper.loadStateCategory(data, workItem.WorkItemType) + hasParent := workItem.ParentId != nil + stdType := resolveWorkItemType(mapper.processTemplate, workItem.WorkItemType, hasParent, typeMappings) + stdStatus := resolveWorkItemStatus(workItem.WorkItemType, workItem.State, statusMappings, mapper.categories) + var resolution *time.Time + if stdStatus == ticket.DONE { + resolution = workItem.ClosedDate + if resolution == nil { + resolution = workItem.UpdatedDate + } + } + var leadTime *uint + if workItem.CreatedDate != nil && resolution != nil { + seconds := int64(resolution.Sub(*workItem.CreatedDate).Seconds()) + if seconds >= 0 { + minutes := uint(seconds / 60) + leadTime = &minutes + } + } + issueID := workItemIdGen.Generate(data.Options.ConnectionId, workItem.AzuredevopsId) + var parentIssueID string + if hasParent { + parentIssueID = workItemIdGen.Generate(data.Options.ConnectionId, *workItem.ParentId) + } + issue := &ticket.Issue{ + DomainEntity: domainlayer.NewDomainEntity(issueID), + Url: workItem.Url, + IssueKey: asString(workItem.AzuredevopsId), + Title: workItem.Title, + Description: workItem.Description, + Type: stdType, + OriginalType: workItem.WorkItemType, + Status: stdStatus, + OriginalStatus: workItem.State, + CreatedDate: workItem.CreatedDate, + UpdatedDate: workItem.UpdatedDate, + ResolutionDate: resolution, + LeadTimeMinutes: leadTime, + OriginalEstimateMinutes: workItem.OriginalEstimateMinutes, + TimeSpentMinutes: workItem.TimeSpentMinutes, + TimeRemainingMinutes: workItem.TimeRemainingMinutes, + CreatorId: workItem.CreatorId, + CreatorName: workItem.CreatorName, + AssigneeId: workItem.AssigneeId, + AssigneeName: workItem.AssigneeName, + ParentIssueId: parentIssueID, + Priority: workItem.Priority, + Severity: workItem.Severity, + Component: areaPathToComponent(workItem.AreaPath), + OriginalProject: workItem.ProjectId, + StoryPoint: workItem.StoryPoints, + IsSubtask: hasParent, + DueDate: workItem.DueDate, + } + results := []interface{}{ + issue, + &ticket.BoardIssue{BoardId: boardID, IssueId: issueID}, + } + if pathKey := iterationPathKey(workItem.IterationPath, workItem.ProjectId); pathKey != "" { + results = append(results, &ticket.SprintIssue{ + SprintId: iterationIdGen.Generate(data.Options.ConnectionId, pathKey), + IssueId: issueID, + }) + } + if workItem.AssigneeId != "" { + results = append(results, &ticket.IssueAssignee{ + IssueId: issueID, + AssigneeId: workItem.AssigneeId, + AssigneeName: workItem.AssigneeName, + }) + } + for _, tag := range splitTags(workItem.Tags) { + results = append(results, &ticket.IssueLabel{IssueId: issueID, LabelName: tag}) + } + results = append(results, workItemRelationships(workItem, issueID, workItemIdGen, data.Options.ConnectionId)...) + return results, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + return "" +} + +func firstTime(values ...interface{}) *time.Time { + for _, value := range values { + if parsed := parseAdoTime(value); parsed != nil { + return parsed + } + } + return nil +} + +func firstFloat(fields map[string]interface{}, names ...string) *float64 { + for _, name := range names { + if parsed := asFloatPtr(fields[name]); parsed != nil { + return parsed + } + } + return nil +} + +func loadWorkItemFieldDisplayNames(taskCtx plugin.SubTaskContext, data *AzuredevopsTaskData) map[string]string { + names := map[string]string{} + res, err := data.ApiClient.Get(fmt.Sprintf("%s/_apis/wit/fields?api-version=7.1", adoScopePrefix(data)), nil, nil) + if err != nil { + taskCtx.GetLogger().Warn(err, "failed to load Azure DevOps work item field names") + return names + } + var body struct { + Value []struct { + ReferenceName string `json:"referenceName"` + Name string `json:"name"` + } `json:"value"` + } + if err = readJSON(res, &body); err != nil { + taskCtx.GetLogger().Warn(err, "failed to load Azure DevOps work item field names") + return names + } + for _, item := range body.Value { + if item.ReferenceName != "" && item.Name != "" && item.Name != item.ReferenceName { + names[item.ReferenceName] = item.Name + } + } + return names +} + +func customFieldsJSON(fields map[string]interface{}, displayNames map[string]string) string { + custom := map[string]interface{}{} + for ref, value := range fields { + if value == nil || strings.HasPrefix(ref, "System.") { + continue + } + if _, mapped := mappedWorkItemFields[ref]; mapped { + continue + } + custom[ref] = value + if name := displayNames[ref]; name != "" { + if _, exists := custom[name]; !exists { + custom[name] = value + } + } + } + if len(custom) == 0 { + return "" + } + encoded, err := json.Marshal(custom) + if err != nil { + return "" + } + return string(encoded) +} + +func splitTags(tags string) []string { + if tags == "" { + return nil + } + parts := make([]string, 0) + for _, tag := range regexp.MustCompile(`\s*;\s*`).Split(tags, -1) { + if tag != "" { + parts = append(parts, tag) + } + } + return parts +} + +func workItemRelationships(workItem *models.AzuredevopsWorkItem, issueID string, idGen *didgen.DomainIdGenerator, connectionID uint64) []interface{} { + var relations []map[string]interface{} + if workItem.RelationsJson != "" { + _ = json.Unmarshal([]byte(workItem.RelationsJson), &relations) + } + linked := map[int]struct{}{} + results := make([]interface{}, 0) + for _, relation := range relations { + match := workItemIDFromURL.FindStringSubmatch(asString(relation["url"])) + relType := asString(relation["rel"]) + if len(match) < 2 || relType == "" { + continue + } + targetID := asInt(match[1]) + if targetID == 0 || targetID == workItem.AzuredevopsId { + continue + } + linked[targetID] = struct{}{} + results = append(results, &ticket.IssueRelationship{ + SourceIssueId: issueID, + TargetIssueId: idGen.Generate(connectionID, targetID), + OriginalType: relType, + }) + } + if workItem.ParentId != nil { + if _, ok := linked[*workItem.ParentId]; !ok { + results = append(results, &ticket.IssueRelationship{ + SourceIssueId: issueID, + TargetIssueId: idGen.Generate(connectionID, *workItem.ParentId), + OriginalType: "System.LinkTypes.Hierarchy-Reverse", + }) + } + } + return results +} diff --git a/backend/plugins/azuredevops_go/tasks/work_item_mapping.go b/backend/plugins/azuredevops_go/tasks/work_item_mapping.go new file mode 100644 index 00000000000..cc6d19ec0cf --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/work_item_mapping.go @@ -0,0 +1,378 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "math" + "strconv" + "strings" + "time" + + "gorm.io/datatypes" +) + +var excludedWorkItemTypes = []string{ + "Shared Parameter", + "Shared Steps", + "Test Case", + "Test Design", + "Test Plan", + "Test Suite", +} + +var stateCategoryToStatus = map[string]string{ + "Proposed": "TODO", + "InProgress": "IN_PROGRESS", + "Resolved": "IN_PROGRESS", + "Completed": "DONE", +} + +var commonTypeDefaults = map[string]string{ + "Epic": "REQUIREMENT", + "Feature": "REQUIREMENT", + "User Story": "REQUIREMENT", + "Product Backlog Item": "REQUIREMENT", + "Requirement": "REQUIREMENT", + "Bug": "BUG", + "Task": "TASK", + "Issue": "TASK", + "Impediment": "TASK", + "Change Request": "REQUIREMENT", + "Risk": "INCIDENT", + "Review": "TASK", +} + +var templateTypeDefaults = map[string]map[string]string{ + "Basic": { + "Epic": "REQUIREMENT", + "Issue": "TASK", + "Task": "TASK", + }, + "Agile": { + "Epic": "REQUIREMENT", + "Feature": "REQUIREMENT", + "User Story": "REQUIREMENT", + "Bug": "BUG", + "Task": "TASK", + "Issue": "TASK", + }, + "Scrum": { + "Epic": "REQUIREMENT", + "Feature": "REQUIREMENT", + "Product Backlog Item": "REQUIREMENT", + "Bug": "BUG", + "Task": "TASK", + }, + "CMMI": { + "Epic": "REQUIREMENT", + "Feature": "REQUIREMENT", + "Requirement": "REQUIREMENT", + "Bug": "BUG", + "Task": "TASK", + "Change Request": "REQUIREMENT", + "Risk": "INCIDENT", + "Review": "TASK", + }, +} + +var defaultStatusByName = map[string]string{ + "New": "TODO", + "Proposed": "TODO", + "To Do": "TODO", + "Active": "IN_PROGRESS", + "Doing": "IN_PROGRESS", + "Committed": "IN_PROGRESS", + "In Progress": "IN_PROGRESS", + "Resolved": "IN_PROGRESS", + "Closed": "DONE", + "Done": "DONE", + "Removed": "DONE", + "Cut": "DONE", +} + +var changelogFieldNames = map[string]string{ + "System.State": "status", + "System.AssignedTo": "assignee", + "System.IterationPath": "sprint", + "System.AreaPath": "component", + "Microsoft.VSTS.Common.Priority": "priority", + "System.Title": "title", + "System.Tags": "tags", +} + +func projectWorkItemsWIQL(projectId string, createdAfter, changedSince *time.Time) string { + clauses := []string{ + "SELECT [System.Id] FROM WorkItems", + fmt.Sprintf("WHERE [System.TeamProject] = '%s'", escapeWIQL(projectId)), + "AND " + wiqlExcludeTypesClause(), + } + if createdAfter != nil { + clauses = append(clauses, fmt.Sprintf("AND [System.CreatedDate] >= '%s'", formatWIQLDate(*createdAfter))) + } + if changedSince != nil { + clauses = append(clauses, fmt.Sprintf("AND [System.ChangedDate] >= '%s'", formatWIQLDate(*changedSince))) + } + clauses = append(clauses, "ORDER BY [System.ChangedDate] DESC") + return strings.Join(clauses, " ") +} + +func wiqlExcludeTypesClause() string { + parts := make([]string, len(excludedWorkItemTypes)) + for i, workItemType := range excludedWorkItemTypes { + parts[i] = fmt.Sprintf("[System.WorkItemType] <> '%s'", escapeWIQL(workItemType)) + } + return strings.Join(parts, " AND ") +} + +func escapeWIQL(value string) string { + return strings.ReplaceAll(value, "'", "''") +} + +func formatWIQLDate(value time.Time) string { + return value.UTC().Format("2006-01-02") +} + +func shouldConvertWorkItem(workItemType string) bool { + if workItemType == "" { + return false + } + for _, excluded := range excludedWorkItemTypes { + if workItemType == excluded { + return false + } + } + return true +} + +func resolveWorkItemType(processTemplate, workItemType string, hasParent bool, typeMappings datatypes.JSONMap) string { + if configured := mappingValue(typeMappings, workItemType, "standardType", "standard_type"); configured != "" { + return strings.ToUpper(configured) + } + stdType := "" + if defaults, ok := templateTypeDefaults[strings.TrimSpace(processTemplate)]; ok { + stdType = defaults[workItemType] + } + if stdType == "" { + stdType = commonTypeDefaults[workItemType] + } + if stdType == "" { + stdType = "TASK" + } + if hasParent && stdType == "TASK" { + return "SUBTASK" + } + return stdType +} + +func resolveWorkItemStatus(workItemType, stateName string, statusMappings datatypes.JSONMap, categories map[string]map[string]string) string { + if stateName == "" { + return "OTHER" + } + if configured := mappingValue(statusMappings, stateName, "standardStatus", "standard_status"); configured != "" { + return strings.ToUpper(configured) + } + if category := categories[workItemType][stateName]; category != "" { + if status, ok := stateCategoryToStatus[category]; ok { + return status + } + return "OTHER" + } + if status, ok := defaultStatusByName[stateName]; ok { + return status + } + return "OTHER" +} + +func mappingValue(mappings datatypes.JSONMap, name string, keys ...string) string { + if mappings == nil { + return "" + } + raw, ok := mappings[name] + if !ok || raw == nil { + return "" + } + entry, ok := raw.(map[string]interface{}) + if !ok { + return "" + } + for _, key := range keys { + if value := asString(entry[key]); value != "" { + return value + } + } + return "" +} + +func iterationPathKey(path, projectId string) string { + segments := iterationSegments(path) + projectSegments := iterationSegments(projectId) + if len(segments) < len(projectSegments) || len(projectSegments) == 0 { + return "" + } + for i, segment := range projectSegments { + if segments[i] != segment { + return "" + } + } + remainder := segments[len(projectSegments):] + if len(remainder) > 0 && strings.EqualFold(remainder[0], "iteration") { + remainder = remainder[1:] + } + if len(remainder) == 0 { + return "" + } + return strings.Join(append(append([]string{}, projectSegments...), remainder...), `\`) +} + +func iterationSegments(path string) []string { + normalized := strings.Trim(strings.ReplaceAll(path, "/", `\`), `\`) + if normalized == "" { + return nil + } + parts := strings.Split(normalized, `\`) + segments := make([]string, 0, len(parts)) + for _, part := range parts { + if part != "" { + segments = append(segments, part) + } + } + return segments +} + +func areaPathToComponent(areaPath string) string { + segments := iterationSegments(areaPath) + if len(segments) == 0 { + return "" + } + return strings.Join(segments, `\`) +} + +func mapTestOutcome(outcome string) string { + switch strings.ToLower(strings.TrimSpace(outcome)) { + case "": + return "PENDING" + case "passed", "pass": + return "SUCCESS" + case "failed", "error", "timeout", "blocked", "notapplicable": + return "FAILED" + case "inprogress", "in progress", "running": + return "IN_PROGRESS" + default: + return "PENDING" + } +} + +func identityFromRef(value interface{}) (string, string) { + switch typed := value.(type) { + case nil: + return "", "" + case string: + return typed, typed + case map[string]interface{}: + id := asString(typed["uniqueName"]) + if id == "" { + id = asString(typed["id"]) + } + return id, asString(typed["displayName"]) + default: + text := asString(typed) + return text, text + } +} + +func asString(value interface{}) string { + switch typed := value.(type) { + case nil: + return "" + case string: + return typed + case float64: + if typed == math.Trunc(typed) { + return strconv.FormatInt(int64(typed), 10) + } + return strconv.FormatFloat(typed, 'f', -1, 64) + case bool: + if typed { + return "true" + } + return "false" + default: + return fmt.Sprint(typed) + } +} + +func asInt(value interface{}) int { + switch typed := value.(type) { + case float64: + return int(typed) + case int: + return typed + case string: + parsed, _ := strconv.Atoi(typed) + return parsed + default: + parsed, _ := strconv.Atoi(asString(value)) + return parsed + } +} + +func asFloatPtr(value interface{}) *float64 { + if value == nil { + return nil + } + switch typed := value.(type) { + case float64: + return &typed + case string: + parsed, err := strconv.ParseFloat(typed, 64) + if err != nil { + return nil + } + return &parsed + default: + return nil + } +} + +func hoursToMinutes(value interface{}) *int64 { + hours := asFloatPtr(value) + if hours == nil { + return nil + } + minutes := int64(math.Round(*hours * 60)) + return &minutes +} + +func parseAdoTime(value interface{}) *time.Time { + text := asString(value) + if text == "" { + return nil + } + for _, layout := range []string{time.RFC3339Nano, "2006-01-02T15:04:05.9999999Z07:00", "2006-01-02T15:04:05Z07:00", "2006-01-02T15:04:05"} { + parsed, err := time.Parse(layout, text) + if err == nil { + return &parsed + } + } + return nil +} + +func isAdoOpenRevisionSentinel(value *time.Time) bool { + return value != nil && value.Year() >= 9999 +} diff --git a/backend/plugins/azuredevops_go/tasks/work_item_mapping_test.go b/backend/plugins/azuredevops_go/tasks/work_item_mapping_test.go new file mode 100644 index 00000000000..afa823d05a8 --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/work_item_mapping_test.go @@ -0,0 +1,98 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "gorm.io/datatypes" +) + +func TestProjectWorkItemsWIQL(t *testing.T) { + created := time.Date(2026, 3, 4, 15, 0, 0, 0, time.FixedZone("ART", -3*3600)) + query := projectWorkItemsWIQL("O'Brien", &created, nil) + assert.Contains(t, query, "[System.TeamProject] = 'O''Brien'") + assert.Contains(t, query, "[System.CreatedDate] >= '2026-03-04'") + assert.Contains(t, query, "[System.WorkItemType] <> 'Test Case'") + assert.NotContains(t, query, "AND [System.ChangedDate]") + assert.Contains(t, query, "ORDER BY [System.ChangedDate] DESC") +} + +func TestIterationPathKey(t *testing.T) { + assert.Equal(t, `Proj\Sprint 1`, iterationPathKey(`Proj\Iteration\Sprint 1`, "Proj")) + assert.Equal(t, `Proj\Sprint 1`, iterationPathKey(`Proj/Sprint 1`, "Proj")) + assert.Equal(t, `Proj\Area\Sprint 1`, iterationPathKey(`Proj\Iteration\Area\Sprint 1`, "Proj")) + assert.Equal(t, "", iterationPathKey("Proj", "Proj")) + assert.Equal(t, "", iterationPathKey(`Proj\Iteration`, "Proj")) + assert.Equal(t, "", iterationPathKey(`Other\Sprint 1`, "Proj")) +} + +func TestResolveWorkItemType(t *testing.T) { + mappings := datatypes.JSONMap{ + "Task": map[string]interface{}{"standardType": "REQUIREMENT"}, + } + assert.Equal(t, "REQUIREMENT", resolveWorkItemType("Agile", "Task", true, mappings)) + assert.Equal(t, "REQUIREMENT", resolveWorkItemType("Agile", "User Story", false, nil)) + assert.Equal(t, "BUG", resolveWorkItemType("Scrum", "Bug", false, nil)) + assert.Equal(t, "SUBTASK", resolveWorkItemType("Agile", "Task", true, nil)) + assert.Equal(t, "TASK", resolveWorkItemType("Agile", "Task", false, nil)) + assert.Equal(t, "TASK", resolveWorkItemType("", "Custom", false, nil)) + assert.False(t, shouldConvertWorkItem("Test Case")) + assert.True(t, shouldConvertWorkItem("Bug")) +} + +func TestResolveWorkItemStatus(t *testing.T) { + mappings := datatypes.JSONMap{ + "Blocked": map[string]interface{}{"standardStatus": "IN_PROGRESS"}, + } + categories := map[string]map[string]string{ + "Bug": {"Approved": "InProgress"}, + } + assert.Equal(t, "IN_PROGRESS", resolveWorkItemStatus("Bug", "Blocked", mappings, categories)) + assert.Equal(t, "IN_PROGRESS", resolveWorkItemStatus("Bug", "Approved", nil, categories)) + assert.Equal(t, "DONE", resolveWorkItemStatus("Bug", "Closed", nil, nil)) + assert.Equal(t, "OTHER", resolveWorkItemStatus("Bug", "", nil, nil)) + assert.Equal(t, "OTHER", resolveWorkItemStatus("Bug", "Mystery", nil, nil)) +} + +func TestMapTestOutcome(t *testing.T) { + assert.Equal(t, "SUCCESS", mapTestOutcome("Passed")) + assert.Equal(t, "FAILED", mapTestOutcome("Blocked")) + assert.Equal(t, "IN_PROGRESS", mapTestOutcome("In Progress")) + assert.Equal(t, "PENDING", mapTestOutcome("")) + assert.Equal(t, "PENDING", mapTestOutcome("Not Executed")) +} + +func TestHoursToMinutes(t *testing.T) { + minutes := hoursToMinutes(1.5) + if assert.NotNil(t, minutes) { + assert.Equal(t, int64(90), *minutes) + } + assert.Nil(t, hoursToMinutes(nil)) +} + +func TestCustomFieldsKeepDisplayNames(t *testing.T) { + encoded := customFieldsJSON(map[string]interface{}{ + "System.Title": "skip", + "Custom.Risk": "high", + "Microsoft.VSTS.Common.Priority": 1, + }, map[string]string{"Custom.Risk": "Risk"}) + assert.JSONEq(t, `{"Custom.Risk":"high","Risk":"high"}`, encoded) +} diff --git a/backend/plugins/azuredevops_go/tasks/work_item_state.go b/backend/plugins/azuredevops_go/tasks/work_item_state.go new file mode 100644 index 00000000000..8e5b1643aa2 --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/work_item_state.go @@ -0,0 +1,108 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "net/url" + + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/plugin" +) + +type workItemMapper struct { + processTemplate string + categories map[string]map[string]string +} + +func newWorkItemMapper(taskCtx plugin.SubTaskContext, data *AzuredevopsTaskData) *workItemMapper { + mapper := &workItemMapper{categories: map[string]map[string]string{}} + if data.Options.ScopeConfig != nil { + mapper.processTemplate = data.Options.ScopeConfig.ProcessTemplate + } + if mapper.processTemplate == "" { + detected, err := detectProcessTemplate(data) + if err != nil { + taskCtx.GetLogger().Warn(err, "failed to detect Azure DevOps process template for %s/%s", data.Options.OrganizationId, data.Options.ProjectId) + } else { + mapper.processTemplate = detected + } + } + return mapper +} + +func detectProcessTemplate(data *AzuredevopsTaskData) (string, errors.Error) { + path := fmt.Sprintf("%s_apis/projects/%s/properties?keys=%s&api-version=7.1-preview.1", adoOrgPrefix(data), url.PathEscape(data.Options.ProjectId), url.QueryEscape("System.Process Template")) + res, err := data.ApiClient.Get(path, nil, nil) + if err != nil { + return "", err + } + var body struct { + Value []struct { + Name string `json:"name"` + Value interface{} `json:"value"` + } `json:"value"` + } + if err = readJSON(res, &body); err != nil { + return "", err + } + for _, item := range body.Value { + if item.Name == "System.Process Template" { + return asString(item.Value), nil + } + } + return "", nil +} + +func (mapper *workItemMapper) loadStateCategory(data *AzuredevopsTaskData, workItemType string) { + if workItemType == "" { + return + } + if _, ok := mapper.categories[workItemType]; ok { + return + } + path := fmt.Sprintf("%s/_apis/wit/workitemtypes/%s/states?api-version=7.1", adoScopePrefix(data), url.PathEscape(workItemType)) + res, err := data.ApiClient.Get(path, nil, nil) + if err != nil { + mapper.categories[workItemType] = map[string]string{} + return + } + var body struct { + Value []struct { + Name string `json:"name"` + Category string `json:"category"` + StateCategory string `json:"stateCategory"` + } `json:"value"` + } + if err = readJSON(res, &body); err != nil { + mapper.categories[workItemType] = map[string]string{} + return + } + categories := map[string]string{} + for _, item := range body.Value { + if item.Name == "" { + continue + } + category := item.Category + if category == "" { + category = item.StateCategory + } + categories[item.Name] = category + } + mapper.categories[workItemType] = categories +} diff --git a/backend/plugins/azuredevops_go/tasks/work_item_updates.go b/backend/plugins/azuredevops_go/tasks/work_item_updates.go new file mode 100644 index 00000000000..41fb1860bba --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/work_item_updates.go @@ -0,0 +1,244 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "reflect" + "time" + + "github.com/apache/devlake/core/dal" + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/models/domainlayer" + "github.com/apache/devlake/core/models/domainlayer/didgen" + "github.com/apache/devlake/core/models/domainlayer/ticket" + "github.com/apache/devlake/core/plugin" + "github.com/apache/devlake/helpers/pluginhelper/api" + "github.com/apache/devlake/plugins/azuredevops_go/models" + "gorm.io/datatypes" +) + +func commentCreated(value *time.Time) time.Time { + if value == nil { + return time.Time{} + } + return *value +} + +func convertProjectRows(taskCtx plugin.SubTaskContext, table string, sample interface{}, convert func(interface{}) ([]interface{}, errors.Error)) errors.Error { + rawArgs, data := CreateRawDataSubTaskArgs(taskCtx, table) + cursor, err := taskCtx.GetDal().Cursor( + dal.From(sample), + dal.Where("connection_id = ? AND project_id = ?", data.Options.ConnectionId, data.Options.ProjectId), + ) + if err != nil { + return err + } + defer cursor.Close() + converter, err := api.NewDataConverter(api.DataConverterArgs{ + InputRowType: reflect.TypeOf(sample).Elem(), + Input: cursor, + RawDataSubTaskArgs: *rawArgs, + Convert: convert, + }) + if err != nil { + return err + } + return converter.Execute() +} + +func ExtractApiWorkItemUpdates(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + if data.Options.ScopeConfig == nil || !data.Options.ScopeConfig.CollectChangelogs { + return nil + } + rawArgs, data := CreateRawDataSubTaskArgs(taskCtx, RawWorkItemUpdateTable) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: *rawArgs, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var raw struct { + ID int `json:"id"` + Rev int `json:"rev"` + RevisedBy map[string]interface{} `json:"revisedBy"` + RevisedDate interface{} `json:"revisedDate"` + Fields map[string]interface{} `json:"fields"` + WorkItemId int `json:"workItemId"` + } + if err := errors.Convert(json.Unmarshal(row.Data, &raw)); err != nil { + return nil, err + } + workItemID := raw.WorkItemId + if workItemID == 0 { + var input workItemRefInput + _ = json.Unmarshal(row.Input, &input) + workItemID = input.WorkItemId + } + authorID, authorName := identityFromRef(raw.RevisedBy) + created := updateCreatedDate(raw.RevisedDate, raw.Fields) + revision := raw.Rev + if revision == 0 { + revision = raw.ID + } + workItemType := fieldChangeString(raw.Fields, "System.WorkItemType", true) + results := make([]interface{}, 0) + for fieldRef, rawChange := range raw.Fields { + change, _ := rawChange.(map[string]interface{}) + if _, mapped := changelogFieldNames[fieldRef]; !mapped || change == nil { + continue + } + fromValue := formatFieldValue(change["oldValue"]) + toValue := formatFieldValue(change["newValue"]) + if fromValue == "" && toValue == "" { + continue + } + results = append(results, &models.AzuredevopsWorkItemUpdate{ + ConnectionId: data.Options.ConnectionId, + WorkItemId: workItemID, + Revision: revision, + FieldRef: fieldRef, + ProjectId: data.Options.ProjectId, + WorkItemType: workItemType, + AuthorId: authorID, + AuthorName: authorName, + OriginalFrom: fromValue, + OriginalTo: toValue, + CreatedDate: created, + }) + } + return results, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + +func ConvertWorkItemUpdates(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + if data.Options.ScopeConfig == nil || !data.Options.ScopeConfig.CollectChangelogs { + return nil + } + mapper := newWorkItemMapper(taskCtx, data) + var statusMappings datatypes.JSONMap + if data.Options.ScopeConfig != nil { + statusMappings = data.Options.ScopeConfig.StatusMappings + } + workItemIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsWorkItem{}) + updateIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsWorkItemUpdate{}) + return convertProjectRows(taskCtx, models.AzuredevopsWorkItemUpdate{}.TableName(), &models.AzuredevopsWorkItemUpdate{}, func(row interface{}) ([]interface{}, errors.Error) { + update := row.(*models.AzuredevopsWorkItemUpdate) + fieldName := changelogFieldNames[update.FieldRef] + if fieldName == "" || (update.OriginalFrom == "" && update.OriginalTo == "") { + return nil, nil + } + if update.WorkItemType != "" && !shouldConvertWorkItem(update.WorkItemType) { + return nil, nil + } + fromValue, toValue := update.OriginalFrom, update.OriginalTo + if fieldName == "status" { + mapper.loadStateCategory(data, update.WorkItemType) + if fromValue != "" { + fromValue = resolveWorkItemStatus(update.WorkItemType, fromValue, statusMappings, mapper.categories) + } + if toValue != "" { + toValue = resolveWorkItemStatus(update.WorkItemType, toValue, statusMappings, mapper.categories) + } + } + var created time.Time + if update.CreatedDate != nil { + created = *update.CreatedDate + } + return []interface{}{&ticket.IssueChangelogs{ + DomainEntity: domainlayer.NewDomainEntity(updateIdGen.Generate(data.Options.ConnectionId, update.WorkItemId, update.Revision, update.FieldRef)), + IssueId: workItemIdGen.Generate(data.Options.ConnectionId, update.WorkItemId), + AuthorId: update.AuthorId, + AuthorName: update.AuthorName, + FieldId: update.FieldRef, + FieldName: fieldName, + OriginalFromValue: update.OriginalFrom, + OriginalToValue: update.OriginalTo, + FromValue: fromValue, + ToValue: toValue, + CreatedDate: created, + }}, nil + }) +} + +func formatFieldValue(value interface{}) string { + switch typed := value.(type) { + case map[string]interface{}: + for _, key := range []string{"uniqueName", "displayName", "id"} { + if text := asString(typed[key]); text != "" { + return text + } + } + return "" + default: + return asString(value) + } +} + +func fieldChangeString(fields map[string]interface{}, fieldRef string, preferNew bool) string { + change, _ := fields[fieldRef].(map[string]interface{}) + if change == nil { + return "" + } + key := "oldValue" + if preferNew { + key = "newValue" + } + value := formatFieldValue(change[key]) + if value == "" { + if preferNew { + return formatFieldValue(change["oldValue"]) + } + return formatFieldValue(change["newValue"]) + } + return value +} + +func updateCreatedDate(revised interface{}, fields map[string]interface{}) *time.Time { + parsed := parseAdoTime(revised) + if parsed != nil && !isAdoOpenRevisionSentinel(parsed) { + return parsed + } + fallbacks := []struct { + field string + key string + }{ + {"System.ChangedDate", "newValue"}, + {"System.AuthorizedDate", "newValue"}, + {"System.RevisedDate", "oldValue"}, + {"Microsoft.VSTS.Common.StateChangeDate", "newValue"}, + } + for _, fallback := range fallbacks { + change, _ := fields[fallback.field].(map[string]interface{}) + if change == nil { + continue + } + candidate := parseAdoTime(change[fallback.key]) + if candidate != nil && !isAdoOpenRevisionSentinel(candidate) { + return candidate + } + } + if isAdoOpenRevisionSentinel(parsed) { + return nil + } + return parsed +} diff --git a/config-ui/src/plugins/components/connection-form/fields/username.tsx b/config-ui/src/plugins/components/connection-form/fields/username.tsx index 9c294eadbae..b23444503e7 100644 --- a/config-ui/src/plugins/components/connection-form/fields/username.tsx +++ b/config-ui/src/plugins/components/connection-form/fields/username.tsx @@ -25,6 +25,7 @@ interface Props { label?: string; subLabel?: string; placeholder?: string; + required?: boolean; initialValue: string; value: string; error: string; @@ -36,6 +37,7 @@ export const ConnectionUsername = ({ label, subLabel, placeholder, + required = true, initialValue, value, setValue, @@ -46,15 +48,15 @@ export const ConnectionUsername = ({ }, [initialValue]); useEffect(() => { - setError(value ? '' : 'username is required'); - }, [value]); + setError(!required || value ? '' : 'username is required'); + }, [required, value]); const handleChange = (e: React.ChangeEvent) => { setValue(e.target.value); }; return ( - + - For Azure DevOps Cloud, use a Personal Access Token (PAT). For On-Premises Server, use a PAT or your domain - password. - - ), + subLabel: + 'For Azure DevOps Cloud, use a Personal Access Token. For On-Premises Server, use a PAT or your domain password. Grant Read access to Build, Code, Graph, Project and Team, Work Items, and Test Management. Service Connections (Read) is also required to list externally connected repositories (GitHub, Bitbucket).', }, ({ initialValues, values, setValues }: any) => ( | undefined, standardType: StandardType) => + Object.entries(typeMappings ?? {}) + .filter(([, value]) => (value?.standardType ?? value?.standard_type) === standardType) + .map(([name]) => name); + +const namesForStandardStatus = (statusMappings: Record | undefined, standardStatus: StandardStatus) => + Object.entries(statusMappings ?? {}) + .filter(([, value]) => (value?.standardStatus ?? value?.standard_status) === standardStatus) + .map(([name]) => name); + +const buildTypeMappings = (requirements: string[], bugs: string[], incidents: string[]) => ({ + ...Object.fromEntries(requirements.map((name) => [name, { standardType: StandardType.Requirement }])), + ...Object.fromEntries(bugs.map((name) => [name, { standardType: StandardType.Bug }])), + ...Object.fromEntries(incidents.map((name) => [name, { standardType: StandardType.Incident }])), +}); + +const buildStatusMappings = (todo: string[], inProgress: string[], done: string[]) => ({ + ...Object.fromEntries(todo.map((name) => [name, { standardStatus: StandardStatus.Todo }])), + ...Object.fromEntries(inProgress.map((name) => [name, { standardStatus: StandardStatus.InProgress }])), + ...Object.fromEntries(done.map((name) => [name, { standardStatus: StandardStatus.Done }])), +}); + +const selectOptions = (defaults: string[], selected: string[]) => + [...new Set([...defaults, ...selected])].sort().map((name) => ({ label: name, value: name })); + +const OPTIONAL_COLLECT_FIELDS = [ + { + key: 'collectChangelogs', + label: 'Work item changelogs', + description: + 'Field history (status, assignee, sprint, etc.) for time-in-status and Issue Trace metrics.', + defaultEnabled: false, + }, + { + key: 'collectComments', + label: 'Work item comments', + description: + 'Discussion comments for response-time and collaboration metrics.', + defaultEnabled: false, + }, +] as const; + +const isFieldEnabled = (transformation: any, key: string, defaultEnabled: boolean) => + transformation[key] ?? defaultEnabled; + +interface Props { + transformation: any; + setTransformation: React.Dispatch>; +} + +export const AzureTicketTransformation = ({ transformation, setTransformation }: Props) => { + const [requirements, setRequirements] = useState([]); + const [bugs, setBugs] = useState([]); + const [incidents, setIncidents] = useState([]); + const [todoStatuses, setTodoStatuses] = useState([]); + const [inProgressStatuses, setInProgressStatuses] = useState([]); + const [doneStatuses, setDoneStatuses] = useState([]); + + useEffect(() => { + setRequirements(namesForStandardType(transformation.typeMappings, StandardType.Requirement)); + setBugs(namesForStandardType(transformation.typeMappings, StandardType.Bug)); + setIncidents(namesForStandardType(transformation.typeMappings, StandardType.Incident)); + setTodoStatuses(namesForStandardStatus(transformation.statusMappings, StandardStatus.Todo)); + setInProgressStatuses(namesForStandardStatus(transformation.statusMappings, StandardStatus.InProgress)); + setDoneStatuses(namesForStandardStatus(transformation.statusMappings, StandardStatus.Done)); + }, [transformation.typeMappings, transformation.statusMappings]); + + const workItemTypeOptions = useMemo( + () => selectOptions(DEFAULT_WORK_ITEM_TYPES, [...requirements, ...bugs, ...incidents]), + [requirements, bugs, incidents], + ); + + const statusOptions = useMemo( + () => selectOptions(DEFAULT_STATUS_NAMES, [...todoStatuses, ...inProgressStatuses, ...doneStatuses]), + [todoStatuses, inProgressStatuses, doneStatuses], + ); + + const updateTypeMappings = (nextRequirements: string[], nextBugs: string[], nextIncidents: string[]) => { + setRequirements(nextRequirements); + setBugs(nextBugs); + setIncidents(nextIncidents); + setTransformation({ + ...transformation, + typeMappings: buildTypeMappings(nextRequirements, nextBugs, nextIncidents), + }); + }; + + const updateStatusMappings = (nextTodo: string[], nextInProgress: string[], nextDone: string[]) => { + setTodoStatuses(nextTodo); + setInProgressStatuses(nextInProgress); + setDoneStatuses(nextDone); + setTransformation({ + ...transformation, + statusMappings: buildStatusMappings(nextTodo, nextInProgress, nextDone), + }); + }; + + return ( + <> + +
+ {OPTIONAL_COLLECT_FIELDS.map(({ key, label, description, defaultEnabled }) => ( +
+ + setTransformation({ + ...transformation, + [key]: event.target.checked, + }) + } + > + {label} + +
{description}
+
+ ))} +
+ + + + + updateTypeMappings(value, bugs, incidents)} + /> + + + updateTypeMappings(requirements, bugs, value)} + /> + + + + + + updateStatusMappings(todoStatuses, value, doneStatuses)} + /> + + +