feat(azuredevops): collect work items and test plans

backend/plugins/azuredevops_go/README.md

@@ -26,5 +26,11 @@ The plugin is able to coexist with the Python version as both implementations co
 - Code
 - Graph (collectAccounts task)
 - Release
+- Work Items, to collect Azure Boards work items, iterations, comments, and changelogs
+- Test Management, to collect test plans, test cases, test runs, and test results
+
+Selecting a repository with work item or test entities collects that repository's project once. Further repositories of the same project in a blueprint do not collect the project again, and the work items and tests are stored on one board. Repository scope names stay the repository name. A project with no enabled repository lists one scope named after the project with a "(project)" suffix, and that scope collects its work items and tests.
+
+Work items are stored in the ticket domain. The board is named after the project, uses the project URL, and has type Scrum. Custom fields are stored under both the Azure reference name and the display name. Test plans, cases, and executions are stored in the QA domain, including each test case's creation time and creator. The scope config can map work item types and states, and can turn comment and changelog collection on.
 
 Access to Service Connections has been removed as they usually contain sensitive security information.

backend/plugins/azuredevops_go/api/azuredevops/client.go

@@ -311,3 +311,81 @@ func (c *Client) GetRemoteRepositories(args GetRemoteRepositoriesArgs) ([]Remote
 		query.Set("continuationToken", contToken)
 	}
 }
+
+const reposFeatureID = "ms.vss-code.version-control"
+
+func (c *Client) GitReposEnabled(orgId, projectName string) (bool, errors.Error) {
+	project, err := c.getProject(orgId, projectName)
+	if err != nil {
+		return false, err
+	}
+	query := url.Values{}
+	query.Set("api-version", "4.1-preview.1")
+	path := fmt.Sprintf("_apis/FeatureManagement/FeatureStatesQuery/host/project/%s", url.PathEscape(project.Id))
+	if orgId != "" {
+		path = url.PathEscape(orgId) + "/" + path
+	}
+	res, err := c.apiClient.Post(path, query, map[string]interface{}{
+		"featureIds":  []string{reposFeatureID},
+		"scopeValues": map[string]string{"project": project.Id},
+	}, nil)
+	if err != nil {
+		return false, err
+	}
+	if res.StatusCode == http.StatusNonAuthoritativeInfo || res.StatusCode == http.StatusUnauthorized || res.StatusCode >= 400 {
+		_, _ = io.Copy(io.Discard, res.Body)
+		res.Body.Close()
+		return false, errors.Default.New(fmt.Sprintf("failed to read Azure Repos service state (%d)", res.StatusCode))
+	}
+	var body struct {
+		FeatureStates map[string]struct {
+			State json.RawMessage `json:"state"`
+		} `json:"featureStates"`
+	}
+	if err = api.UnmarshalResponse(res, &body); err != nil {
+		return false, err
+	}
+	feature, ok := body.FeatureStates[reposFeatureID]
+	if !ok {
+		return true, nil
+	}
+	return !featureStateDisabled(feature.State), nil
+}
+
+func (c *Client) getProject(orgId, projectName string) (Project, errors.Error) {
+	query := url.Values{}
+	query.Set("api-version", apiVersion)
+	path := fmt.Sprintf("_apis/projects/%s", url.PathEscape(projectName))
+	if orgId != "" {
+		path = url.PathEscape(orgId) + "/" + path
+	}
+	res, err := c.apiClient.Get(path, query, nil)
+	if err != nil {
+		return Project{}, err
+	}
+	if res.StatusCode == http.StatusNonAuthoritativeInfo || res.StatusCode == http.StatusUnauthorized || res.StatusCode >= 400 {
+		_, _ = io.Copy(io.Discard, res.Body)
+		res.Body.Close()
+		return Project{}, errors.Default.New(fmt.Sprintf("failed to read Azure DevOps project %s (%d)", projectName, res.StatusCode))
+	}
+	var project Project
+	if err = api.UnmarshalResponse(res, &project); err != nil {
+		return Project{}, err
+	}
+	if project.Id == "" {
+		return Project{}, errors.Default.New(fmt.Sprintf("Azure DevOps project %s has no id", projectName))
+	}
+	return project, nil
+}
+
+func featureStateDisabled(raw json.RawMessage) bool {
+	var text string
+	if json.Unmarshal(raw, &text) == nil {
+		return strings.EqualFold(text, "disabled") || text == "0"
+	}
+	var number float64
+	if json.Unmarshal(raw, &number) == nil {
+		return number == 0
+	}
+	return false
+}

backend/plugins/azuredevops_go/api/azuredevops/feature_state_test.go

@@ -0,0 +1,32 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azuredevops
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFeatureStateDisabled(t *testing.T) {
+	assert.True(t, featureStateDisabled(json.RawMessage(`"disabled"`)))
+	assert.True(t, featureStateDisabled(json.RawMessage(`0`)))
+	assert.False(t, featureStateDisabled(json.RawMessage(`"enabled"`)))
+	assert.False(t, featureStateDisabled(json.RawMessage(`1`)))
+}

backend/plugins/azuredevops_go/api/blueprint_v200.go

@@ -20,15 +20,16 @@ package api
 import (
 	"net/url"
 
-	"golang.org/x/exp/slices"
-
 	"github.com/apache/devlake/core/errors"
 	"github.com/apache/devlake/core/utils"
 
 	coreModels "github.com/apache/devlake/core/models"
+	"github.com/apache/devlake/core/models/common"
+	"github.com/apache/devlake/core/models/domainlayer"
 	"github.com/apache/devlake/core/models/domainlayer/code"
 	"github.com/apache/devlake/core/models/domainlayer/devops"
 	"github.com/apache/devlake/core/models/domainlayer/didgen"
+	"github.com/apache/devlake/core/models/domainlayer/qa"
 	"github.com/apache/devlake/core/models/domainlayer/ticket"
 	"github.com/apache/devlake/core/plugin"
 	helper "github.com/apache/devlake/helpers/pluginhelper/api"
@@ -51,7 +52,7 @@ func MakePipelinePlanV200(
 		return nil, nil, err
 	}
 
-	sc, err := makeScopeV200(connectionId, scopeDetails)
+	sc, err := makeScopeV200(connection, scopeDetails)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -65,73 +66,82 @@ func MakePipelinePlanV200(
 }
 
 func makeScopeV200(
-	connectionId uint64,
+	connection *models.AzuredevopsConnection,
 	scopeDetails []*srvhelper.ScopeDetail[models.AzuredevopsRepo, models.AzuredevopsScopeConfig],
 ) ([]plugin.Scope, errors.Error) {
 	sc := make([]plugin.Scope, 0, 3*len(scopeDetails))
-
-	for _, scope := range scopeDetails {
-		azuredevopsRepo, scopeConfig := scope.Scope, scope.ScopeConfig
-		if azuredevopsRepo.Type != models.RepositoryTypeADO {
-			continue
-		}
-		id := didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connectionId, azuredevopsRepo.Id)
-
-		// if no entities specified, use all entities enabled by default
-		if len(scopeConfig.Entities) == 0 {
-			scopeConfig.Entities = plugin.DOMAIN_TYPES
-		}
-
-		if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE_REVIEW) ||
-			utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) ||
-			utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CROSS) {
-			// if we don't need to collect gitex, we need to add repo to scopes here
-			scopeRepo := code.NewRepo(id, azuredevopsRepo.Name)
-			sc = append(sc, scopeRepo)
-		}
-
-		// add cicd_scope to scopes
-		if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CICD) {
-			scopeCICD := devops.NewCicdScope(id, azuredevopsRepo.Name)
-			sc = append(sc, scopeCICD)
-		}
-
-		// add board to scopes
-		if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) {
-			scopeTicket := ticket.NewBoard(id, azuredevopsRepo.Name)
-			sc = append(sc, scopeTicket)
-		}
+	connectionId := uint64(0)
+	endpoint := ""
+	if connection != nil {
+		connectionId = connection.ID
+		endpoint = connection.GetEndpoint()
 	}
+	claimedProjects := map[string]struct{}{}
 
 	for _, scope := range scopeDetails {
 		azuredevopsRepo, scopeConfig := scope.Scope, scope.ScopeConfig
-		if azuredevopsRepo.Type == models.RepositoryTypeADO {
-			continue
-		}
-		id := didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connectionId, azuredevopsRepo.Id)
+		entities := models.ClaimProjectEntities(
+			models.ScopeEntities(azuredevopsRepo.Type, scopeConfig.Entities),
+			azuredevopsRepo.ProjectKey(),
+			claimedProjects,
+		)
+		repoID := didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connectionId, azuredevopsRepo.Id)
+		name := azuredevopsRepo.DomainName()
 
-		// Azure DevOps Pipeline can be used with remote repositories such as GitHub and Bitbucket
-		if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CICD) {
-			scopeCICD := devops.NewCicdScope(id, azuredevopsRepo.Name)
-			sc = append(sc, scopeCICD)
-		}
-
-		// DOMAIN_TYPE_CODE (i.e. gitextractor, rediff) only works if the repository is public
-		if !azuredevopsRepo.IsPrivate && utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) {
-			scopeRepo := code.NewRepo(id, azuredevopsRepo.Name)
-			sc = append(sc, scopeRepo)
+		if azuredevopsRepo.Type == models.RepositoryTypeADO {
+			if utils.StringsContains(entities, plugin.DOMAIN_TYPE_CODE_REVIEW) ||
+				utils.StringsContains(entities, plugin.DOMAIN_TYPE_CODE) ||
+				utils.StringsContains(entities, plugin.DOMAIN_TYPE_CROSS) {
+				sc = append(sc, code.NewRepo(repoID, name))
+			}
+			if utils.StringsContains(entities, plugin.DOMAIN_TYPE_CICD) {
+				sc = append(sc, devops.NewCicdScope(repoID, name))
+			}
+		} else if azuredevopsRepo.Type != models.RepositoryTypeProject {
+			if utils.StringsContains(entities, plugin.DOMAIN_TYPE_CICD) {
+				sc = append(sc, devops.NewCicdScope(repoID, name))
+			}
+			if !azuredevopsRepo.IsPrivate && utils.StringsContains(entities, plugin.DOMAIN_TYPE_CODE) {
+				sc = append(sc, code.NewRepo(repoID, name))
+			}
 		}
+		sc = appendProjectScopes(sc, connectionId, endpoint, azuredevopsRepo, entities)
 	}
 
 	return sc, nil
 }
 
+func appendProjectScopes(sc []plugin.Scope, connectionId uint64, endpoint string, repo models.AzuredevopsRepo, entities []string) []plugin.Scope {
+	if !models.HasProjectEntity(entities) {
+		return sc
+	}
+	id := didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connectionId, repo.ProjectDomainScopeID())
+	projectName := repo.ProjectName()
+	if utils.StringsContains(entities, plugin.DOMAIN_TYPE_TICKET) {
+		board := ticket.NewBoard(id, projectName)
+		board.Type = "Scrum"
+		board.Url = models.ProjectBoardURL(endpoint, repo.OrganizationId, projectName)
+		sc = append(sc, board)
+	}
+	if utils.StringsContains(entities, plugin.DOMAIN_TYPE_CODE_QUALITY) {
+		sc = append(sc, &qa.QaProject{
+			DomainEntityExtended: domainlayer.DomainEntityExtended{
+				Id:        id,
+				NoPKModel: common.NewNoPKModel(),
+			},
+			Name: projectName,
+		})
+	}
+	return sc
+}
+
 func makePipelinePlanV200(
 	subtaskMetas []plugin.SubTaskMeta,
 	connection *models.AzuredevopsConnection,
 	scopeDetails []*srvhelper.ScopeDetail[models.AzuredevopsRepo, models.AzuredevopsScopeConfig],
 ) (coreModels.PipelinePlan, errors.Error) {
 	plans := make(coreModels.PipelinePlan, 0, 3*len(scopeDetails))
+	claimedProjects := map[string]struct{}{}
 	for _, scope := range scopeDetails {
 		azuredevopsRepo, scopeConfig := scope.Scope, scope.ScopeConfig
 		var stage coreModels.PipelineStage
@@ -147,18 +157,19 @@ func makePipelinePlanV200(
 		options["repositoryId"] = azuredevopsRepo.Id
 		options["repositoryType"] = azuredevopsRepo.Type
 
-		// construct subtasks
-		var entities []string
-		if scope.Scope.Type == models.RepositoryTypeADO {
-			entities = append(entities, scopeConfig.Entities...)
-		} else {
-			if i := slices.Index(scopeConfig.Entities, plugin.DOMAIN_TYPE_CICD); i >= 0 {
-				entities = append(entities, scopeConfig.Entities[i])
-			}
-
-			if i := slices.Index(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE); i >= 0 && !scope.Scope.IsPrivate {
-				entities = append(entities, scopeConfig.Entities[i])
+		entities := models.ClaimProjectEntities(
+			models.ScopeEntities(azuredevopsRepo.Type, scopeConfig.Entities),
+			azuredevopsRepo.ProjectKey(),
+			claimedProjects,
+		)
+		if azuredevopsRepo.Type != models.RepositoryTypeADO && azuredevopsRepo.Type != models.RepositoryTypeProject && azuredevopsRepo.IsPrivate {
+			filtered := make([]string, 0, len(entities))
+			for _, entity := range entities {
+				if entity != plugin.DOMAIN_TYPE_CODE {
+					filtered = append(filtered, entity)
+				}
 			}
+			entities = filtered
 		}
 
 		subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, entities)
@@ -173,7 +184,7 @@ func makePipelinePlanV200(
 		})
 
 		// collect git data by gitextractor if CODE was requested
-		if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) && !scope.Scope.IsPrivate || len(scopeConfig.Entities) == 0 {
+		if azuredevopsRepo.Type != models.RepositoryTypeProject && (utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) && !scope.Scope.IsPrivate || len(scopeConfig.Entities) == 0) {
 			cloneUrl, err := errors.Convert01(url.Parse(azuredevopsRepo.RemoteUrl))
 			if err != nil {
 				return nil, err
@@ -201,7 +212,7 @@ func makePipelinePlanV200(
 		plans = append(plans, stage)
 
 		// refdiff part
-		if scopeConfig.Refdiff != nil {
+		if azuredevopsRepo.Type != models.RepositoryTypeProject && scopeConfig.Refdiff != nil {
 			task := &coreModels.PipelineTask{
 				Plugin:  "refdiff",
 				Options: scopeConfig.Refdiff,