From 082b8653ade9a0c3968575ca96d40bdc0c0c8c3b Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Sun, 1 Nov 2026 12:37:07 +0800 Subject: [PATCH 1/5] test(gitextractor): cover collectors with git CLI fixtures Build repository fixtures with the git command line instead of go-git and add a contract suite that records everything a RepoCollector writes: counts, refs, commits, parents, commit files, components, snapshots and line changes. The suite runs against every implementation. Behaviour that deviates from the intended contract is listed per implementation, so the git CLI collector that replaces libgit2 and go-git can be held to the contract without inheriting the deviations. No production code changes. --- .../parser/collector_contract_test.go | 737 ++++++++++++++++++ .../gitextractor/parser/repo_gogit_test.go | 122 --- .../gitextractor/parser/testrepo_test.go | 316 ++++++++ 3 files changed, 1053 insertions(+), 122 deletions(-) create mode 100644 backend/plugins/gitextractor/parser/collector_contract_test.go delete mode 100644 backend/plugins/gitextractor/parser/repo_gogit_test.go create mode 100644 backend/plugins/gitextractor/parser/testrepo_test.go diff --git a/backend/plugins/gitextractor/parser/collector_contract_test.go b/backend/plugins/gitextractor/parser/collector_contract_test.go new file mode 100644 index 00000000000..8ec51598a3a --- /dev/null +++ b/backend/plugins/gitextractor/parser/collector_contract_test.go @@ -0,0 +1,737 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "context" + "sort" + "testing" + + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/models/domainlayer/code" + "github.com/apache/devlake/helpers/unithelper" + mockdal "github.com/apache/devlake/mocks/core/dal" + mockplugin "github.com/apache/devlake/mocks/core/plugin" + "github.com/apache/devlake/plugins/gitextractor/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// collectorCase describes one RepoCollector implementation under test. +// +// The assertions in this file describe the collector contract that a +// replacement implementation must satisfy. The deviations block records where +// an existing implementation is known to violate that contract; they are +// documented instead of being silently tolerated so that a replacement cannot +// inherit them. +type collectorCase struct { + name string + open func(t *testing.T, localDir, repoId string, store models.Store) RepoCollector + + deviations collectorDeviations +} + +type collectorDeviations struct { + // storesOnlyLastComponent reports that only the component of the last file + // of each commit is stored. + storesOnlyLastComponent bool + // countTagsIgnoresContext reports that CountTags ignores a cancelled context. + countTagsIgnoresContext bool + // collectsShallowBoundary reports that the shallow boundary commit is + // collected with a full-tree diff instead of being skipped. + collectsShallowBoundary bool + // failsOnDeletedFiles reports that CollectDiffLine fails when the history + // contains a deleted file. + failsOnDeletedFiles bool + // skipsLineChanges reports that CollectDiffLine does not store + // CommitLineChange rows. + skipsLineChanges bool + // failsOnEmptyRepo reports that the Count* helpers fail on a repository + // without commits. + failsOnEmptyRepo bool + // omitsZeroLineFiles reports that changed files without line changes + // (binary files, empty files) are not stored. + omitsZeroLineFiles bool +} + +func collectorCases() []collectorCase { + return []collectorCase{ + { + name: "libgit2", + open: func(t *testing.T, localDir, repoId string, store models.Store) RepoCollector { + t.Helper() + collector, err := NewLibgit2RepoCollector(localDir, repoId, store, unithelper.DummyLogger()) + require.Nil(t, err) + return collector + }, + deviations: collectorDeviations{ + storesOnlyLastComponent: true, + countTagsIgnoresContext: true, + collectsShallowBoundary: true, + }, + }, + { + name: "gogit", + open: func(t *testing.T, localDir, repoId string, store models.Store) RepoCollector { + t.Helper() + collector, err := NewGogitRepoCollector(localDir, repoId, store, unithelper.DummyLogger()) + require.Nil(t, err) + return collector + }, + deviations: collectorDeviations{ + failsOnDeletedFiles: true, + skipsLineChanges: true, + failsOnEmptyRepo: true, + omitsZeroLineFiles: true, + }, + }, + } +} + +func forEachCollector(t *testing.T, fn func(t *testing.T, c collectorCase)) { + t.Helper() + for _, c := range collectorCases() { + t.Run(c.name, func(t *testing.T) { + fn(t, c) + }) + } +} + +// recordStore is a models.Store that records everything the collectors write. +type recordStore struct { + incremental bool + closed bool + repoCommits []*code.RepoCommit + commits []*code.Commit + refs []*code.Ref + commitFiles []*code.CommitFile + parents []*code.CommitParent + components []*code.CommitFileComponent + lineChanges []*code.CommitLineChange + snapshots []*code.RepoSnapshot +} + +var _ models.Store = (*recordStore)(nil) + +func (s *recordStore) SetIncrementalMode(v bool) { s.incremental = v } + +func (s *recordStore) RepoCommits(v *code.RepoCommit) errors.Error { + s.repoCommits = append(s.repoCommits, v) + return nil +} + +func (s *recordStore) Commits(v *code.Commit) errors.Error { + cp := *v + s.commits = append(s.commits, &cp) + return nil +} + +func (s *recordStore) Refs(v *code.Ref) errors.Error { + cp := *v + s.refs = append(s.refs, &cp) + return nil +} + +func (s *recordStore) CommitFiles(v *code.CommitFile) errors.Error { + cp := *v + s.commitFiles = append(s.commitFiles, &cp) + return nil +} + +func (s *recordStore) CommitParents(v []*code.CommitParent) errors.Error { + s.parents = append(s.parents, v...) + return nil +} + +func (s *recordStore) CommitFileComponents(v *code.CommitFileComponent) errors.Error { + cp := *v + s.components = append(s.components, &cp) + return nil +} + +func (s *recordStore) CommitLineChange(v *code.CommitLineChange) errors.Error { + cp := *v + s.lineChanges = append(s.lineChanges, &cp) + return nil +} + +func (s *recordStore) RepoSnapshot(v *code.RepoSnapshot) errors.Error { + cp := *v + s.snapshots = append(s.snapshots, &cp) + return nil +} + +func (s *recordStore) Close() errors.Error { + s.closed = true + return nil +} + +func boolPtr(v bool) *bool { return &v } + +func defaultOptions(repoId string) *GitExtractorOptions { + return &GitExtractorOptions{ + GitExtractorApiParams: GitExtractorApiParams{RepoId: repoId}, + SkipCommitStat: boolPtr(false), + SkipCommitFiles: boolPtr(false), + } +} + +// newCollectorContext returns a subtask context backed by a mocked DAL. The +// components are returned from Dal.All, which the collectors use to map files +// to components. +func newCollectorContext(t *testing.T, opts *GitExtractorOptions, components []code.Component) *mockplugin.SubTaskContext { + t.Helper() + dalMock := new(mockdal.Dal) + dalMock.On("All", mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + if len(components) == 0 { + return + } + dst, ok := args.Get(0).(*[]code.Component) + if !ok { + return + } + *dst = append(*dst, components...) + }) + dalMock.On("Delete", mock.Anything, mock.Anything).Return(nil) + + ctx := unithelper.DummySubTaskContext(dalMock) + ctx.On("GetContext").Return(context.Background()) + ctx.On("GetData").Return(&GitExtractorTaskData{Options: opts}) + return ctx +} + +func commitsBySha(commits []*code.Commit) map[string]*code.Commit { + out := make(map[string]*code.Commit, len(commits)) + for _, commit := range commits { + out[commit.Sha] = commit + } + return out +} + +func refsByName(refs []*code.Ref) map[string]*code.Ref { + out := make(map[string]*code.Ref, len(refs)) + for _, ref := range refs { + out[ref.Name] = ref + } + return out +} + +func componentsById(components []*code.CommitFileComponent) map[string]string { + out := make(map[string]string, len(components)) + for _, component := range components { + out[component.CommitFileId] = component.ComponentName + } + return out +} + +// filesByCommit returns commit sha -> file path -> [additions, deletions]. +func filesByCommit(files []*code.CommitFile) map[string]map[string][2]int { + out := make(map[string]map[string][2]int) + for _, file := range files { + if out[file.CommitSha] == nil { + out[file.CommitSha] = make(map[string][2]int) + } + out[file.CommitSha][file.FilePath] = [2]int{file.Additions, file.Deletions} + } + return out +} + +func parentShas(parents []*code.CommitParent, commitSha string) []string { + var out []string + for _, parent := range parents { + if parent.CommitSha == commitSha { + out = append(out, parent.ParentCommitSha) + } + } + sort.Strings(out) + return out +} + +func TestCollectorCountsAndRefs(t *testing.T) { + repo := buildCanonicalRepo(t) + + forEachCollector(t, func(t *testing.T, c collectorCase) { + store := &recordStore{} + collector := c.open(t, repo.Dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + commits, err := collector.CountCommits(context.Background()) + require.NoError(t, err) + assert.Equal(t, 5, commits) + + tags, err := collector.CountTags(context.Background()) + require.NoError(t, err) + assert.Equal(t, 2, tags) + // HEAD points at main, which is not counted. + branches, err := collector.CountBranches(context.Background()) + require.NoError(t, err) + assert.Equal(t, 1, branches) + + assert.NoError(t, collector.CollectTags(ctx)) + assert.NoError(t, collector.CollectBranches(ctx)) + + refs := refsByName(store.refs) + require.Len(t, refs, 4) + + // Tag names are stored as the full ref name. + light := refs["refs/tags/v1.0.0"] + require.NotNil(t, light, "lightweight tag v1.0.0") + assert.Equal(t, "test-repo:refs/tags/v1.0.0", light.Id) + assert.Equal(t, repo.C1, light.CommitSha) + assert.Equal(t, TAG, light.RefType) + assert.Equal(t, "test-repo", light.RepoId) + + annotated := refs["refs/tags/v2.0.0"] + require.NotNil(t, annotated, "annotated tag v2.0.0 should resolve to its target commit") + assert.Equal(t, repo.C3, annotated.CommitSha) + assert.Equal(t, TAG, annotated.RefType) + + mainRef := refs["main"] + require.NotNil(t, mainRef) + assert.Equal(t, repo.C5, mainRef.CommitSha) + assert.Equal(t, BRANCH, mainRef.RefType) + assert.True(t, mainRef.IsDefault) + + featureRef := refs["feature"] + require.NotNil(t, featureRef) + assert.Equal(t, repo.C2, featureRef.CommitSha) + assert.Equal(t, BRANCH, featureRef.RefType) + assert.False(t, featureRef.IsDefault) + }) +} + +func TestCollectorCommits(t *testing.T) { + repo := buildCanonicalRepo(t) + repoId := "test-repo" + + forEachCollector(t, func(t *testing.T, c collectorCase) { + store := &recordStore{} + collector := c.open(t, repo.Dir, repoId, store) + ctx := newCollectorContext(t, defaultOptions(repoId), nil) + + require.NoError(t, collector.CollectCommits(ctx)) + + commits := commitsBySha(store.commits) + require.Len(t, commits, 5) + require.Len(t, store.repoCommits, 5) + + for _, tc := range []struct { + sha string + when string + msg string + }{ + {repo.C1, "2026-01-01T00:00:00Z", "c1: initial import"}, + {repo.C2, "2026-01-01T01:00:00Z", "c2: add feature file"}, + {repo.C3, "2026-01-01T02:00:00Z", "c3: reword and extend a.txt"}, + {repo.C4, "2026-01-01T03:00:00Z", "c4: mode and binary change"}, + {repo.C5, "2026-01-01T04:00:00Z", "c5: merge feature"}, + } { + commit := commits[tc.sha] + require.NotNil(t, commit, tc.msg) + assert.Equal(t, tc.msg+"\n", commit.Message) + assert.Equal(t, testAuthorName, commit.AuthorName) + assert.Equal(t, testAuthorEmail, commit.AuthorEmail) + assert.Equal(t, testAuthorEmail, commit.AuthorId) + assert.Equal(t, testAuthorName, commit.CommitterName) + assert.Equal(t, testAuthorEmail, commit.CommitterEmail) + assert.Equal(t, testAuthorEmail, commit.CommitterId) + assert.Equal(t, tc.when, commit.AuthoredDate.UTC().Format("2006-01-02T15:04:05Z")) + assert.Equal(t, tc.when, commit.CommittedDate.UTC().Format("2006-01-02T15:04:05Z")) + } + + assert.Empty(t, parentShas(store.parents, repo.C1)) + assert.Equal(t, []string{repo.C1}, parentShas(store.parents, repo.C2)) + assert.Equal(t, []string{repo.C1}, parentShas(store.parents, repo.C3)) + assert.Equal(t, []string{repo.C3}, parentShas(store.parents, repo.C4)) + assert.ElementsMatch(t, []string{repo.C2, repo.C4}, parentShas(store.parents, repo.C5)) + + files := filesByCommit(store.commitFiles) + expectedFiles := map[string]map[string][2]int{ + repo.C1: { + "a.txt": {3, 0}, + "bin.dat": {0, 0}, + "dir with space/b.txt": {1, 0}, + "empty.txt": {0, 0}, + }, + repo.C2: {"feature.txt": {1, 0}}, + repo.C3: { + "a.txt": {2, 1}, + "empty.txt": {0, 0}, + }, + repo.C4: { + "a.txt": {0, 0}, + "bin.dat": {0, 0}, + }, + // The merge is compared against its first parent, so only the file + // the merge brought in is reported. + repo.C5: {"feature.txt": {1, 0}}, + } + for sha, want := range expectedFiles { + if !c.deviations.omitsZeroLineFiles { + assert.Equal(t, want, files[sha], sha) + continue + } + for path, stat := range want { + if stat != [2]int{0, 0} { + assert.Equal(t, stat, files[sha][path], "%s %s", sha, path) + } + } + for path, stat := range files[sha] { + if stat != [2]int{0, 0} { + assert.Contains(t, want, path, "%s %s", sha, path) + } + } + } + + assert.Equal(t, 4, commits[repo.C1].Additions) + assert.Equal(t, 0, commits[repo.C1].Deletions) + assert.Equal(t, 1, commits[repo.C2].Additions) + assert.Equal(t, 2, commits[repo.C3].Additions) + assert.Equal(t, 1, commits[repo.C3].Deletions) + assert.Equal(t, 0, commits[repo.C4].Additions) + assert.Equal(t, 0, commits[repo.C4].Deletions) + assert.Equal(t, 1, commits[repo.C5].Additions) + assert.Equal(t, 0, commits[repo.C5].Deletions) + + for _, file := range store.commitFiles { + assert.Equal(t, genCommitFileId(file.CommitSha, file.FilePath), file.Id) + } + if !c.deviations.storesOnlyLastComponent { + components := componentsById(store.components) + for _, file := range store.commitFiles { + assert.Equal(t, "Default", components[file.Id], file.FilePath) + } + } + }) +} + +func TestCollectorUsesConfiguredComponents(t *testing.T) { + repo := buildLinearRepo(t) + + forEachCollector(t, func(t *testing.T, c collectorCase) { + store := &recordStore{} + collector := c.open(t, repo.Dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), []code.Component{ + {RepoId: "test-repo", Name: "docs", PathRegex: `^c\.txt$`}, + }) + + require.NoError(t, collector.CollectCommits(ctx)) + + components := componentsById(store.components) + files := filesByCommit(store.commitFiles)[repo.C4] + require.Contains(t, files, "c.txt") + assert.Equal(t, "docs", components[genCommitFileId(repo.C4, "c.txt")]) + if !c.deviations.storesOnlyLastComponent { + assert.Equal(t, "Default", components[genCommitFileId(repo.C4, "b.txt")]) + } + }) +} + +func TestCollectorSkipCommitStat(t *testing.T) { + repo := buildCanonicalRepo(t) + + forEachCollector(t, func(t *testing.T, c collectorCase) { + store := &recordStore{} + collector := c.open(t, repo.Dir, "test-repo", store) + opts := defaultOptions("test-repo") + opts.SkipCommitStat = boolPtr(true) + opts.SkipCommitFiles = boolPtr(true) + ctx := newCollectorContext(t, opts, nil) + + require.NoError(t, collector.CollectCommits(ctx)) + + require.Len(t, store.commits, 5) + for _, commit := range store.commits { + assert.Zero(t, commit.Additions) + assert.Zero(t, commit.Deletions) + } + assert.Empty(t, store.commitFiles) + assert.Empty(t, store.components) + }) +} + +func TestCollectorSkipCommitFiles(t *testing.T) { + repo := buildCanonicalRepo(t) + + forEachCollector(t, func(t *testing.T, c collectorCase) { + store := &recordStore{} + collector := c.open(t, repo.Dir, "test-repo", store) + opts := defaultOptions("test-repo") + opts.SkipCommitFiles = boolPtr(true) + ctx := newCollectorContext(t, opts, nil) + + require.NoError(t, collector.CollectCommits(ctx)) + + assert.Empty(t, store.commitFiles) + assert.Empty(t, store.components) + commits := commitsBySha(store.commits) + assert.Equal(t, 4, commits[repo.C1].Additions) + assert.Equal(t, 1, commits[repo.C2].Additions) + }) +} + +func TestCollectorExcludeFileExtensions(t *testing.T) { + repo := buildCanonicalRepo(t) + + forEachCollector(t, func(t *testing.T, c collectorCase) { + store := &recordStore{} + collector := c.open(t, repo.Dir, "test-repo", store) + opts := defaultOptions("test-repo") + opts.ExcludeFileExtensions = []string{".DAT"} + ctx := newCollectorContext(t, opts, nil) + + require.NoError(t, collector.CollectCommits(ctx)) + + commits := commitsBySha(store.commits) + assert.Equal(t, 4, commits[repo.C1].Additions) + assert.Equal(t, 0, commits[repo.C4].Additions) + assert.Equal(t, 0, commits[repo.C4].Deletions) + + files := filesByCommit(store.commitFiles) + assert.NotContains(t, files[repo.C1], "bin.dat") + assert.NotContains(t, files[repo.C4], "bin.dat") + assert.Contains(t, files[repo.C1], "a.txt") + }) +} + +func TestCollectorSkipsShallowBoundaryCommit(t *testing.T) { + f := newGitFixture(t) + f.writeFile("a.txt", "l1\n") + c1 := f.commit(testTimeC1, "c1") + f.writeFile("a.txt", "l1\nl2\n") + c2 := f.commit(testTimeC2, "c2") + require.NotEqual(t, c1, c2) + + dir := f.shallowBareClone() + + forEachCollector(t, func(t *testing.T, c collectorCase) { + if c.deviations.collectsShallowBoundary { + t.Skip("known deviation: the shallow boundary is collected with a full-tree diff") + } + store := &recordStore{} + collector := c.open(t, dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + // The tip is a shallow boundary: its parent is not in the object + // database, so the collector must not compute stats for it (a full-tree + // diff would store an absurdly large addition count). + require.NoError(t, collector.CollectCommits(ctx)) + assert.Empty(t, store.commits) + assert.Empty(t, store.commitFiles) + }) +} + +func TestCollectorEmptyRepo(t *testing.T) { + dir := newGitFixture(t).bareClone() + + forEachCollector(t, func(t *testing.T, c collectorCase) { + if c.deviations.failsOnEmptyRepo { + t.Skip("known deviation: the Count* helpers need at least one commit") + } + store := &recordStore{} + collector := c.open(t, dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + commits, err := collector.CountCommits(context.Background()) + require.NoError(t, err) + assert.Zero(t, commits) + tags, err := collector.CountTags(context.Background()) + require.NoError(t, err) + assert.Zero(t, tags) + branches, err := collector.CountBranches(context.Background()) + require.NoError(t, err) + assert.Zero(t, branches) + + assert.NoError(t, collector.CollectTags(ctx)) + assert.NoError(t, collector.CollectBranches(ctx)) + assert.NoError(t, collector.CollectCommits(ctx)) + assert.Empty(t, store.refs) + assert.Empty(t, store.commits) + }) +} + +func TestCollectorHonoursCancelledContext(t *testing.T) { + repo := buildCanonicalRepo(t) + + forEachCollector(t, func(t *testing.T, c collectorCase) { + store := &recordStore{} + collector := c.open(t, repo.Dir, "test-repo", store) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := collector.CountCommits(ctx) + assert.ErrorIs(t, err, context.Canceled) + if !c.deviations.countTagsIgnoresContext { + _, err = collector.CountTags(ctx) + assert.ErrorIs(t, err, context.Canceled) + } + _, err = collector.CountBranches(ctx) + assert.ErrorIs(t, err, context.Canceled) + }) +} + +func TestCollectorRejectsNonRepo(t *testing.T) { + dir := t.TempDir() + for _, c := range collectorCases() { + t.Run(c.name, func(t *testing.T) { + var err error + switch c.name { + case "libgit2": + _, err = NewLibgit2RepoCollector(dir, "test-repo", nil, unithelper.DummyLogger()) + case "gogit": + _, err = NewGogitRepoCollector(dir, "test-repo", nil, unithelper.DummyLogger()) + } + assert.Error(t, err, "%s should reject a non-repository", c.name) + }) + } +} + +func TestCollectorCountsWhenHeadIsNotDefault(t *testing.T) { + f := newGitFixture(t) + f.writeFile("a.txt", "l1\n") + c1 := f.commit(testTimeC1, "c1") + f.checkout("feature", true) + f.writeFile("a.txt", "l1\nl2\n") + c2 := f.commit(testTimeC2, "c2") + require.NotEqual(t, c1, c2) + dir := f.bareClone() + + forEachCollector(t, func(t *testing.T, c collectorCase) { + store := &recordStore{} + collector := c.open(t, dir, "test-repo", store) + + // HEAD points at feature, which is not counted but is the default ref. + branches, err := collector.CountBranches(context.Background()) + require.NoError(t, err) + assert.Equal(t, 1, branches) + + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + require.NoError(t, collector.CollectBranches(ctx)) + + refs := refsByName(store.refs) + require.Len(t, refs, 2) + assert.True(t, refs["feature"].IsDefault) + assert.False(t, refs["main"].IsDefault) + }) +} + +// TestCollectorDiffLineSnapshot checks the repo snapshot against git blame, +// which is an independent oracle for "which commit introduced each line". +func TestCollectorDiffLineSnapshot(t *testing.T) { + repo := buildLinearRepo(t) + expected := blameAtHEAD(t, repo.Dir) + + forEachCollector(t, func(t *testing.T, c collectorCase) { + if c.deviations.failsOnDeletedFiles { + t.Skip("known deviation: CollectDiffLine fails on a deleted file") + } + store := &recordStore{} + collector := c.open(t, repo.Dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + require.NoError(t, collector.CollectDiffLine(ctx)) + + got := make(map[string]map[int]string) + for _, snapshot := range store.snapshots { + if got[snapshot.FilePath] == nil { + got[snapshot.FilePath] = make(map[int]string) + } + got[snapshot.FilePath][snapshot.LineNo] = snapshot.CommitSha + } + + // b.txt is deleted in c4, so it has no lines in HEAD and no snapshot rows. + assert.NotContains(t, got, "b.txt") + for file, lines := range expected { + assert.Equal(t, lines, got[file], file) + } + + if c.deviations.skipsLineChanges { + assert.Empty(t, store.lineChanges) + } + }) +} + +func TestCollectorCollectAllAndClose(t *testing.T) { + repo := buildCanonicalRepo(t) + + forEachCollector(t, func(t *testing.T, c collectorCase) { + if c.deviations.failsOnDeletedFiles { + t.Skip("known deviation: CollectDiffLine fails on a deleted file") + } + store := &recordStore{} + collector := c.open(t, repo.Dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + cleanedUp := false + require.NoError(t, collector.SetCleanUp(func() { cleanedUp = true })) + + require.NoError(t, collector.CollectAll(ctx)) + assert.Len(t, store.commits, 5) + assert.Len(t, store.repoCommits, 5) + assert.Len(t, store.refs, 4) + assert.NotEmpty(t, store.commitFiles) + assert.NotEmpty(t, store.snapshots) + if c.deviations.skipsLineChanges { + assert.Empty(t, store.lineChanges) + } else { + assert.NotEmpty(t, store.lineChanges) + } + + require.NoError(t, collector.Close(context.Background())) + assert.True(t, store.closed) + assert.True(t, cleanedUp) + }) +} + +func TestLibgit2CollectorCommitLineChanges(t *testing.T) { + repo := buildLinearRepo(t) + + store := &recordStore{} + collector, err := NewLibgit2RepoCollector(repo.Dir, "test-repo", store, unithelper.DummyLogger()) + require.Nil(t, err) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + require.NoError(t, collector.CollectDiffLine(ctx)) + require.NotEmpty(t, store.lineChanges) + + // c3 appends line 4 to a.txt, so a.txt gets one addition at new line 4. + var additions []*code.CommitLineChange + for _, change := range store.lineChanges { + if change.CommitSha == repo.C3 && change.NewFilePath == "a.txt" && change.ChangedType == "Addition" { + additions = append(additions, change) + } + } + require.Len(t, additions, 1) + assert.Equal(t, 4, additions[0].LineNoNew) + assert.Equal(t, -1, additions[0].LineNoOld) + assert.Equal(t, 1, additions[0].HunkNum) + assert.Equal(t, "", additions[0].PrevCommit) + assert.Equal(t, repo.C3+":a.txt:-1:4", additions[0].Id) + + // c2 rewrites line 2 of a.txt; the deletion must be attributed to c1. + var deletions []*code.CommitLineChange + for _, change := range store.lineChanges { + if change.CommitSha == repo.C2 && change.NewFilePath == "a.txt" && change.ChangedType == "Deletion" { + deletions = append(deletions, change) + } + } + require.Len(t, deletions, 1) + assert.Equal(t, 2, deletions[0].LineNoOld) + assert.Equal(t, repo.C1, deletions[0].PrevCommit) +} diff --git a/backend/plugins/gitextractor/parser/repo_gogit_test.go b/backend/plugins/gitextractor/parser/repo_gogit_test.go deleted file mode 100644 index a4dfa321b3e..00000000000 --- a/backend/plugins/gitextractor/parser/repo_gogit_test.go +++ /dev/null @@ -1,122 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package parser - -import ( - "context" - "os" - "path/filepath" - "testing" - "time" - - gogit "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// buildTestRepo creates a repository on disk with a known shape: 3 commits on -// the default branch, one lightweight tag and a second branch which is checked -// out at the end, so HEAD does not point at the default branch. -func buildTestRepo(t *testing.T) string { - t.Helper() - dir := t.TempDir() - - repo, err := gogit.PlainInit(dir, false) - require.NoError(t, err) - wt, err := repo.Worktree() - require.NoError(t, err) - - sig := func(n int) *object.Signature { - return &object.Signature{ - Name: "DevLake Test", - Email: "test@devlake.apache.org", - When: time.Date(2026, 1, 1, 0, 0, n, 0, time.UTC), - } - } - - var last plumbing.Hash - for i := 1; i <= 3; i++ { - name := filepath.Join(dir, "file.txt") - require.NoError(t, os.WriteFile(name, []byte{byte('a' + i)}, 0600)) - _, err = wt.Add("file.txt") - require.NoError(t, err) - last, err = wt.Commit("commit", &gogit.CommitOptions{Author: sig(i)}) - require.NoError(t, err) - } - - _, err = repo.CreateTag("v1.0.0", last, nil) - require.NoError(t, err) - - require.NoError(t, wt.Checkout(&gogit.CheckoutOptions{ - Branch: plumbing.NewBranchReferenceName("feature"), - Create: true, - })) - - return dir -} - -func TestGogitRepoCollectorCounts(t *testing.T) { - dir := buildTestRepo(t) - - // store and logger are only used by the Collect* subtasks, not by the - // counting helpers exercised here. - collector, err := NewGogitRepoCollector(dir, "test-repo", nil, nil) - require.Nil(t, err) - - ctx := context.Background() - - commits, cErr := collector.CountCommits(ctx) - assert.NoError(t, cErr) - assert.Equal(t, 3, commits) - - tags, tErr := collector.CountTags(ctx) - assert.NoError(t, tErr) - assert.Equal(t, 1, tags) - - // HEAD is on "feature", so only the default branch is counted. - branches, bErr := collector.CountBranches(ctx) - assert.NoError(t, bErr) - assert.Equal(t, 1, branches) -} - -func TestGogitRepoCollectorCountsHonourCancelledContext(t *testing.T) { - dir := buildTestRepo(t) - - collector, err := NewGogitRepoCollector(dir, "test-repo", nil, nil) - require.Nil(t, err) - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - _, cErr := collector.CountCommits(ctx) - assert.ErrorIs(t, cErr, context.Canceled) - - _, tErr := collector.CountTags(ctx) - assert.ErrorIs(t, tErr, context.Canceled) - - _, bErr := collector.CountBranches(ctx) - assert.ErrorIs(t, bErr, context.Canceled) -} - -func TestNewGogitRepoCollectorRejectsNonRepo(t *testing.T) { - _, err := NewGogitRepoCollector(t.TempDir(), "test-repo", nil, nil) - assert.NotNil(t, err) -} - diff --git a/backend/plugins/gitextractor/parser/testrepo_test.go b/backend/plugins/gitextractor/parser/testrepo_test.go new file mode 100644 index 00000000000..400acd63730 --- /dev/null +++ b/backend/plugins/gitextractor/parser/testrepo_test.go @@ -0,0 +1,316 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// This file builds deterministic git repositories with the git command line. +// The collector tests use them as fixtures so that the fixtures do not depend +// on any of the git libraries under test. + +const ( + testAuthorName = "DevLake Test" + testAuthorEmail = "test@devlake.apache.org" +) + +var ( + testTimeBase = time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + testTimeC1 = testTimeBase + testTimeC2 = testTimeBase.Add(1 * time.Hour) + testTimeC3 = testTimeBase.Add(2 * time.Hour) + testTimeC4 = testTimeBase.Add(3 * time.Hour) + testTimeC5 = testTimeBase.Add(4 * time.Hour) +) + +func requireGit(t *testing.T) { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git is not installed") + } +} + +// gitFixture is a scriptable git repository. Commands run with an isolated +// git configuration and a fixed identity so the fixtures are reproducible. +type gitFixture struct { + t *testing.T + dir string +} + +func newGitFixture(t *testing.T) *gitFixture { + t.Helper() + requireGit(t) + f := newGitFixtureAt(t, t.TempDir()) + f.git("init", "-q", "-b", "main") + return f +} + +func newGitFixtureAt(t *testing.T, dir string) *gitFixture { + t.Helper() + requireGit(t) + return &gitFixture{t: t, dir: dir} +} + +func (f *gitFixture) env(when time.Time) []string { + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+testAuthorName, + "GIT_AUTHOR_EMAIL="+testAuthorEmail, + "GIT_COMMITTER_NAME="+testAuthorName, + "GIT_COMMITTER_EMAIL="+testAuthorEmail, + "GIT_CONFIG_NOSYSTEM=1", + "GIT_CONFIG_GLOBAL="+os.DevNull, + "GIT_TERMINAL_PROMPT=0", + "TZ=UTC", + // keep the fixtures independent of the host user configuration + "HOME="+f.dir, + ) + if !when.IsZero() { + date := when.UTC().Format(time.RFC3339) + env = append(env, "GIT_AUTHOR_DATE="+date, "GIT_COMMITTER_DATE="+date) + } + return env +} + +func (f *gitFixture) git(args ...string) string { + f.t.Helper() + return f.gitAt(time.Time{}, args...) +} + +func (f *gitFixture) gitAt(when time.Time, args ...string) string { + f.t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = f.dir + cmd.Env = f.env(when) + output, err := cmd.CombinedOutput() + require.NoError(f.t, err, "git %v in %s: %s", args, f.dir, output) + return strings.TrimSpace(string(output)) +} + +func (f *gitFixture) head() string { + f.t.Helper() + return f.git("rev-parse", "HEAD") +} + +func (f *gitFixture) writeFile(rel, content string) { + f.t.Helper() + full := filepath.Join(f.dir, rel) + require.NoError(f.t, os.MkdirAll(filepath.Dir(full), 0o755)) + require.NoError(f.t, os.WriteFile(full, []byte(content), 0o644)) +} + +func (f *gitFixture) chmod(rel string, mode os.FileMode) { + f.t.Helper() + require.NoError(f.t, os.Chmod(filepath.Join(f.dir, rel), mode)) +} + +func (f *gitFixture) removeFile(rel string) { + f.t.Helper() + require.NoError(f.t, os.Remove(filepath.Join(f.dir, rel))) +} + +// commit stages everything and commits with the fixture identity at when. +func (f *gitFixture) commit(when time.Time, message string) string { + f.t.Helper() + f.git("add", "-A") + f.gitAt(when, "commit", "-q", "-m", message) + return f.head() +} + +func (f *gitFixture) checkout(branch string, create bool) { + f.t.Helper() + if create { + f.git("checkout", "-q", "-b", branch) + return + } + f.git("checkout", "-q", branch) +} + +func (f *gitFixture) merge(when time.Time, message, branch string) string { + f.t.Helper() + f.gitAt(when, "merge", "-q", "--no-ff", "-m", message, branch) + return f.head() +} + +func (f *gitFixture) tag(name, target string, annotated bool) { + f.t.Helper() + if annotated { + f.git("tag", "-a", name, target, "-m", "release "+name) + return + } + f.git("tag", name, target) +} + +// bareClone clones the fixture into a new bare repository, which is the shape +// the gitextractor produces before collecting. +func (f *gitFixture) bareClone() string { + f.t.Helper() + dir := filepath.Join(f.t.TempDir(), "repo.git") + f.git("clone", "-q", "--bare", f.dir, dir) + return dir +} + +// shallowBareClone creates a depth-1 bare clone, so the tip commit is a +// shallow boundary whose parent is missing from the object database. +func (f *gitFixture) shallowBareClone() string { + f.t.Helper() + dir := filepath.Join(f.t.TempDir(), "shallow.git") + f.git("clone", "-q", "--bare", "--depth=1", "file://"+f.dir, dir) + return dir +} + +// canonicalRepo is the repository used by the collector contract tests. +// +// main: c1 ---- c3 ---- c4 ---- c5 (merge) +// \ / +// feature: c2 ---------------- +// +// c1 adds a.txt (3 lines), a file in a directory with a space, a binary file +// and an empty file. c2 adds a file on the feature branch. c3 rewrites a line +// in a.txt, appends a line and deletes the empty file. c4 only changes the file +// mode of a.txt and rewrites the binary file. c5 merges feature. +// c1 carries the lightweight tag v1.0.0, c3 the annotated tag v2.0.0. +type canonicalRepo struct { + Dir string + C1 string + C2 string + C3 string + C4 string + C5 string +} + +func buildCanonicalRepo(t *testing.T) canonicalRepo { + t.Helper() + f := newGitFixture(t) + + f.writeFile("a.txt", "l1\nl2\nl3\n") + f.writeFile("dir with space/b.txt", "b1\n") + f.writeFile("bin.dat", "before\x00binary\n") + f.writeFile("empty.txt", "") + c1 := f.commit(testTimeC1, "c1: initial import") + + f.checkout("feature", true) + f.writeFile("feature.txt", "f1\n") + c2 := f.commit(testTimeC2, "c2: add feature file") + + f.checkout("main", false) + f.writeFile("a.txt", "l1\nL2\nl3\nl4\n") + f.removeFile("empty.txt") + c3 := f.commit(testTimeC3, "c3: reword and extend a.txt") + + f.chmod("a.txt", 0o755) + f.writeFile("bin.dat", "after\x00binary\n") + c4 := f.commit(testTimeC4, "c4: mode and binary change") + + c5 := f.merge(testTimeC5, "c5: merge feature", "feature") + + f.tag("v1.0.0", c1, false) + f.tag("v2.0.0", c3, true) + + return canonicalRepo{ + Dir: f.bareClone(), + C1: c1, + C2: c2, + C3: c3, + C4: c4, + C5: c5, + } +} + +// buildLinearRepo creates a repository without merges for the diff line tests. +// +// c1 adds a.txt and b.txt, c2 rewrites a line in a.txt, c3 appends a line to +// a.txt and rewrites a line in b.txt, c4 deletes b.txt and adds c.txt. +type linearRepo struct { + Dir string + C1 string + C2 string + C3 string + C4 string +} + +func buildLinearRepo(t *testing.T) linearRepo { + t.Helper() + f := newGitFixture(t) + + f.writeFile("a.txt", "l1\nl2\nl3\n") + f.writeFile("b.txt", "b1\nb2\n") + c1 := f.commit(testTimeC1, "c1") + + f.writeFile("a.txt", "l1\nL2\nl3\n") + c2 := f.commit(testTimeC2, "c2") + + f.writeFile("a.txt", "l1\nL2\nl3\nl4\n") + f.writeFile("b.txt", "b1\nB2\n") + c3 := f.commit(testTimeC3, "c3") + + f.removeFile("b.txt") + f.writeFile("c.txt", "c1\nc2\n") + c4 := f.commit(testTimeC4, "c4") + + return linearRepo{ + Dir: f.bareClone(), + C1: c1, + C2: c2, + C3: c3, + C4: c4, + } +} + +// blameAtHEAD returns file -> line number -> commit sha for every file in HEAD, +// using the git command line as an independent oracle. +func blameAtHEAD(t *testing.T, dir string) map[string]map[int]string { + t.Helper() + repo := newGitFixtureAt(t, dir) + blames := make(map[string]map[int]string) + files := strings.Split(strings.TrimRight(repo.git("ls-tree", "-r", "--name-only", "-z", "HEAD"), "\x00"), "\x00") + for _, file := range files { + if file == "" { + continue + } + out := repo.git("blame", "-l", "--line-porcelain", "HEAD", "--", file) + lines := make(map[int]string) + for _, line := range strings.Split(out, "\n") { + // The header of the first line of each group carries the group + // size, later headers do not. Content lines are tab-prefixed. + if strings.HasPrefix(line, "\t") { + continue + } + fields := strings.Fields(line) + if len(fields) < 3 || len(fields[0]) != 40 { + continue + } + finalLine, err := strconv.Atoi(fields[2]) + if err != nil { + continue + } + lines[finalLine] = fields[0] + } + blames[file] = lines + } + return blames +} From 517a0a175ae94d80012484f01b342732797da276 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Sun, 1 Nov 2026 13:13:59 +0800 Subject: [PATCH 2/5] feat(gitextractor): add git CLI collector Implement RepoCollector on top of git plumbing and porcelain commands and select it with GIT_EXTRACTOR_COLLECTOR=gitcli (or --useGitCli) for A/B testing. libgit2 and go-git stay the default for now. The collector streams rev-list, cat-file and numstat output in bounded batches, skips shallow boundary commits, honours context cancellation, and rebuilds the line snapshot from the first-parent patch stream. The contract suite now runs against all three collectors; the git CLI collector passes without any of the documented deviations. On apache/devlake (8383 reachable commits) CollectCommits takes 2.9s versus 21.3s with libgit2, and CollectDiffLine produces 559k snapshot rows and 2.7M line changes in 3.4s. --- backend/plugins/gitextractor/gitextractor.go | 2 + backend/plugins/gitextractor/impl/impl.go | 3 + .../gitextractor/parser/clone_gitcli.go | 2 + .../parser/collector_contract_test.go | 11 + .../gitextractor/parser/commitparse.go | 172 ++++ .../gitextractor/parser/commitparse_test.go | 115 +++ .../plugins/gitextractor/parser/gitrunner.go | 155 ++++ .../gitextractor/parser/numstatparse.go | 130 +++ .../plugins/gitextractor/parser/parse_test.go | 243 ++++++ .../plugins/gitextractor/parser/patchparse.go | 332 ++++++++ .../gitextractor/parser/repo_gitcli.go | 753 ++++++++++++++++++ .../plugins/gitextractor/parser/taskdata.go | 4 + .../plugins/gitextractor/tasks/repo_cloner.go | 4 +- 13 files changed, 1925 insertions(+), 1 deletion(-) create mode 100644 backend/plugins/gitextractor/parser/commitparse.go create mode 100644 backend/plugins/gitextractor/parser/commitparse_test.go create mode 100644 backend/plugins/gitextractor/parser/gitrunner.go create mode 100644 backend/plugins/gitextractor/parser/numstatparse.go create mode 100644 backend/plugins/gitextractor/parser/parse_test.go create mode 100644 backend/plugins/gitextractor/parser/patchparse.go create mode 100644 backend/plugins/gitextractor/parser/repo_gitcli.go diff --git a/backend/plugins/gitextractor/gitextractor.go b/backend/plugins/gitextractor/gitextractor.go index 67e0b7f0f2a..1261c3834fe 100644 --- a/backend/plugins/gitextractor/gitextractor.go +++ b/backend/plugins/gitextractor/gitextractor.go @@ -36,6 +36,7 @@ func main() { // pkPass := cmd.Flags().StringP("privateKeyPassPhrase", "P", "", "passphrase for private key") proxy := cmd.Flags().StringP("proxy", "x", "", "proxy") useGoGit := cmd.Flags().BoolP("useGoGit", "g", false, "use go-git instead of libgit2") + useGitCli := cmd.Flags().BoolP("useGitCli", "", false, "use the git command line collector") skipCommitStat := cmd.Flags().BoolP("skipCommitStat", "S", false, "") skipCommitFiles := cmd.Flags().BoolP("skipCommitFiles", "F", true, "") noShallowClone := cmd.Flags().BoolP("noShallowClone", "A", false, "") @@ -53,6 +54,7 @@ func main() { // "passphrase" "proxy": *proxy, "useGoGit": *useGoGit, + "useGitCli": *useGitCli, "skipCommitStat": skipCommitStat, "skipCommitFiles": skipCommitFiles, "noShallowClone": noShallowClone, diff --git a/backend/plugins/gitextractor/impl/impl.go b/backend/plugins/gitextractor/impl/impl.go index d0f763489af..746afee85b9 100644 --- a/backend/plugins/gitextractor/impl/impl.go +++ b/backend/plugins/gitextractor/impl/impl.go @@ -20,6 +20,7 @@ package impl import ( "fmt" "net/url" + "strings" "github.com/apache/devlake/core/dal" "github.com/apache/devlake/core/errors" @@ -118,7 +119,9 @@ func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options map[st loadBool(&op.UseGoGit, "UseGoGit", false) loadBool(&op.SkipCommitStat, "SKIP_COMMIT_STAT", false) loadBool(&op.SkipCommitFiles, "SKIP_COMMIT_FILES", true) + op.UseGitCli = op.UseGitCli || strings.EqualFold(cfg.GetString("GIT_EXTRACTOR_COLLECTOR"), "gitcli") log.Info("UseGoGit: %v", *op.UseGoGit) + log.Info("UseGitCli: %v", op.UseGitCli) log.Info("SkipCommitStat: %v", *op.SkipCommitStat) log.Info("SkipCommitFiles: %v", *op.SkipCommitFiles) diff --git a/backend/plugins/gitextractor/parser/clone_gitcli.go b/backend/plugins/gitextractor/parser/clone_gitcli.go index 82fc9d57afb..3c88899f7d1 100644 --- a/backend/plugins/gitextractor/parser/clone_gitcli.go +++ b/backend/plugins/gitextractor/parser/clone_gitcli.go @@ -47,6 +47,7 @@ const maxExtraDeepenRounds = 10 // the subtask should run in Full Sync mode whenever the configuration is changed type CloneRepoConfig struct { UseGoGit *bool + UseGitCli bool SkipCommitStat *bool SkipCommitFiles *bool NoShallowClone bool @@ -73,6 +74,7 @@ func NewGitcliCloner(ctx plugin.SubTaskContext, localDir string) (*GitcliCloner, Params: taskData.Options.GitExtractorApiParams, SubtaskConfig: CloneRepoConfig{ UseGoGit: taskData.Options.UseGoGit, + UseGitCli: taskData.Options.UseGitCli, SkipCommitStat: taskData.Options.SkipCommitStat, SkipCommitFiles: taskData.Options.SkipCommitFiles, NoShallowClone: taskData.Options.NoShallowClone, diff --git a/backend/plugins/gitextractor/parser/collector_contract_test.go b/backend/plugins/gitextractor/parser/collector_contract_test.go index 8ec51598a3a..52b97ece95e 100644 --- a/backend/plugins/gitextractor/parser/collector_contract_test.go +++ b/backend/plugins/gitextractor/parser/collector_contract_test.go @@ -72,6 +72,15 @@ type collectorDeviations struct { func collectorCases() []collectorCase { return []collectorCase{ + { + name: "gitcli", + open: func(t *testing.T, localDir, repoId string, store models.Store) RepoCollector { + t.Helper() + collector, err := NewGitcliRepoCollector(localDir, repoId, store, unithelper.DummyLogger()) + require.Nil(t, err) + return collector + }, + }, { name: "libgit2", open: func(t *testing.T, localDir, repoId string, store models.Store) RepoCollector { @@ -592,6 +601,8 @@ func TestCollectorRejectsNonRepo(t *testing.T) { t.Run(c.name, func(t *testing.T) { var err error switch c.name { + case "gitcli": + _, err = NewGitcliRepoCollector(dir, "test-repo", nil, unithelper.DummyLogger()) case "libgit2": _, err = NewLibgit2RepoCollector(dir, "test-repo", nil, unithelper.DummyLogger()) case "gogit": diff --git a/backend/plugins/gitextractor/parser/commitparse.go b/backend/plugins/gitextractor/parser/commitparse.go new file mode 100644 index 00000000000..4f52f89f824 --- /dev/null +++ b/backend/plugins/gitextractor/parser/commitparse.go @@ -0,0 +1,172 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strconv" + "strings" + "time" +) + +// parsedCommit is the metadata of one commit object, as stored by git. +type parsedCommit struct { + Sha string + Parents []string + Author parsedSignature + Committer parsedSignature + Message string +} + +type parsedSignature struct { + Name string + Email string + When time.Time +} + +// parseCommitObject parses the raw content of a commit object. +func parseCommitObject(sha string, content []byte) (*parsedCommit, error) { + commit := &parsedCommit{Sha: sha} + + header := content + if idx := bytes.Index(content, []byte("\n\n")); idx >= 0 { + header = content[:idx] + commit.Message = string(content[idx+2:]) + } + + for _, line := range strings.Split(string(header), "\n") { + if line == "" { + continue + } + if line[0] == ' ' { + // continuation of a multi-line header such as gpgsig, not used here + continue + } + key, value, ok := strings.Cut(line, " ") + if !ok { + continue + } + switch key { + case "parent": + commit.Parents = append(commit.Parents, value) + case "author": + signature, err := parseSignature(value) + if err != nil { + return nil, fmt.Errorf("commit %s: %w", sha, err) + } + commit.Author = signature + case "committer": + signature, err := parseSignature(value) + if err != nil { + return nil, fmt.Errorf("commit %s: %w", sha, err) + } + commit.Committer = signature + } + } + return commit, nil +} + +// parseSignature parses "Name ". +func parseSignature(value string) (parsedSignature, error) { + var signature parsedSignature + + tzSpace := strings.LastIndexByte(value, ' ') + if tzSpace < 0 { + return signature, fmt.Errorf("invalid signature %q", value) + } + timezone := value[tzSpace+1:] + rest := value[:tzSpace] + + tsSpace := strings.LastIndexByte(rest, ' ') + if tsSpace < 0 { + return signature, fmt.Errorf("invalid signature %q", value) + } + seconds, err := strconv.ParseInt(rest[tsSpace+1:], 10, 64) + if err != nil { + return signature, fmt.Errorf("invalid timestamp in signature %q", value) + } + identity := rest[:tsSpace] + + if lt := strings.LastIndexByte(identity, '<'); lt >= 0 { + if gt := strings.LastIndexByte(identity[lt:], '>'); gt >= 0 { + signature.Email = identity[lt+1 : lt+gt] + signature.Name = strings.TrimSpace(identity[:lt]) + } else { + signature.Name = strings.TrimSpace(identity) + } + } else { + signature.Name = strings.TrimSpace(identity) + } + + signature.When = time.Unix(seconds, 0).In(parseTimezone(timezone)) + return signature, nil +} + +// parseTimezone converts a git timezone such as "+0800" into a fixed zone so +// the original offset of the commit date is preserved. +func parseTimezone(timezone string) *time.Location { + if len(timezone) != 5 || (timezone[0] != '+' && timezone[0] != '-') { + return time.UTC + } + hours, errH := strconv.Atoi(timezone[1:3]) + minutes, errM := strconv.Atoi(timezone[3:5]) + if errH != nil || errM != nil { + return time.UTC + } + offset := hours*3600 + minutes*60 + if timezone[0] == '-' { + offset = -offset + } + return time.FixedZone(timezone, offset) +} + +// readCatFileObject reads one object from `git cat-file --batch` output. +// It returns io.EOF when the stream is exhausted. +func readCatFileObject(reader *bufio.Reader) (string, []byte, error) { + header, err := reader.ReadString('\n') + if err == io.EOF && header == "" { + return "", nil, io.EOF + } + if err != nil && err != io.EOF { + return "", nil, err + } + + fields := strings.Fields(header) + if len(fields) != 3 { + return "", nil, fmt.Errorf("unexpected cat-file header %q", strings.TrimSpace(header)) + } + if fields[1] != "commit" { + return "", nil, fmt.Errorf("object %s is a %s, not a commit", fields[0], fields[1]) + } + size, err := strconv.Atoi(fields[2]) + if err != nil { + return "", nil, fmt.Errorf("unexpected cat-file size in %q", strings.TrimSpace(header)) + } + + content := make([]byte, size) + if _, err := io.ReadFull(reader, content); err != nil { + return "", nil, err + } + if _, err := reader.Discard(1); err != nil { // trailing newline + return "", nil, err + } + return fields[0], content, nil +} diff --git a/backend/plugins/gitextractor/parser/commitparse_test.go b/backend/plugins/gitextractor/parser/commitparse_test.go new file mode 100644 index 00000000000..473fc070c0f --- /dev/null +++ b/backend/plugins/gitextractor/parser/commitparse_test.go @@ -0,0 +1,115 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "bufio" + "io" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const rawCommitObject = `tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904 +parent 1111111111111111111111111111111111111111 +parent 2222222222222222222222222222222222222222 +author Jane Doe 1704067200 +0800 +committer DevLake Test 1704070800 -0530 +gpgsig -----BEGIN PGP SIGNATURE----- + + fakesignature + -----END PGP SIGNATURE----- + +subject line + +body line 1 +body line 2 +` + +func TestParseCommitObject(t *testing.T) { + commit, err := parseCommitObject("abc", []byte(rawCommitObject)) + require.NoError(t, err) + + assert.Equal(t, "abc", commit.Sha) + assert.Equal(t, []string{ + "1111111111111111111111111111111111111111", + "2222222222222222222222222222222222222222", + }, commit.Parents) + + assert.Equal(t, "Jane Doe", commit.Author.Name) + assert.Equal(t, "jane@example.com", commit.Author.Email) + assert.Equal(t, int64(1704067200), commit.Author.When.Unix()) + _, offset := commit.Author.When.Zone() + assert.Equal(t, 8*3600, offset) + + assert.Equal(t, "DevLake Test", commit.Committer.Name) + assert.Equal(t, "test@devlake.apache.org", commit.Committer.Email) + _, offset = commit.Committer.When.Zone() + assert.Equal(t, -(5*3600 + 30*60), offset) + + assert.Equal(t, "subject line\n\nbody line 1\nbody line 2\n", commit.Message) +} + +func TestParseCommitObjectWithoutMessage(t *testing.T) { + content := "tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904\n" + + "author A 1704067200 +0000\n" + + "committer A 1704067200 +0000\n" + + commit, err := parseCommitObject("abc", []byte(content)) + require.NoError(t, err) + assert.Empty(t, commit.Parents) + assert.Empty(t, commit.Message) + assert.Equal(t, time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), commit.Author.When.UTC()) +} + +func TestParseSignature(t *testing.T) { + signature, err := parseSignature("A U Thor 1704067200 +0000") + require.NoError(t, err) + assert.Equal(t, "A U Thor", signature.Name) + assert.Equal(t, "a@example.com", signature.Email) + + // a name containing '<' still resolves to the last email-like part + signature, err = parseSignature("Weird 1704067200 +0000") + require.NoError(t, err) + assert.Equal(t, "Weird ", signature.Name) + assert.Equal(t, "weird@example.com", signature.Email) + + _, err = parseSignature("no timestamp") + assert.Error(t, err) +} + +func TestReadCatFileObject(t *testing.T) { + stream := "abc commit 5\nhello\n" + "def commit 0\n\n" + reader := bufio.NewReader(strings.NewReader(stream)) + + sha, content, err := readCatFileObject(reader) + require.NoError(t, err) + assert.Equal(t, "abc", sha) + assert.Equal(t, "hello", string(content)) + + sha, content, err = readCatFileObject(reader) + require.NoError(t, err) + assert.Equal(t, "def", sha) + assert.Empty(t, content) + + _, _, err = readCatFileObject(reader) + assert.ErrorIs(t, err, io.EOF) +} diff --git a/backend/plugins/gitextractor/parser/gitrunner.go b/backend/plugins/gitextractor/parser/gitrunner.go new file mode 100644 index 00000000000..f3bb8e3d6dd --- /dev/null +++ b/backend/plugins/gitextractor/parser/gitrunner.go @@ -0,0 +1,155 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + "os/exec" + "strings" +) + +// gitRunner runs read-only git commands against one local repository. +// +// Output and Stream both run the git command in a way that never writes to the +// repository and never needs a working tree, so bare clones are supported. +type gitRunner struct { + dir string +} + +func newGitRunner(dir string) *gitRunner { + return &gitRunner{dir: dir} +} + +// Output runs git and returns stdout. Stderr is reported in the error. +func (r *gitRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + var stdout, stderr bytes.Buffer + cmd := r.command(ctx, args...) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } + return nil, fmt.Errorf("git %s failed: %s", strings.Join(sanitizeArgs(args), " "), errorMessage(stderr.Bytes(), err)) + } + return stdout.Bytes(), nil +} + +// OutputLines runs git and returns its stdout lines with the trailing newline +// removed from each line. +func (r *gitRunner) OutputLines(ctx context.Context, args ...string) ([]string, error) { + out, err := r.Output(ctx, args...) + if err != nil { + return nil, err + } + return splitLines(out), nil +} + +// Stream runs git and returns its stdout. The caller must read to EOF and then +// call the returned wait function; closing the reader early kills git. +func (r *gitRunner) Stream(ctx context.Context, args ...string) (io.ReadCloser, func() error, error) { + cmd := r.command(ctx, args...) + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, nil, err + } + stderr := &bytes.Buffer{} + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + return nil, nil, err + } + return stdout, func() error { + err := cmd.Wait() + if err == nil { + return nil + } + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + return fmt.Errorf("git %s failed: %s", strings.Join(sanitizeArgs(args), " "), errorMessage(stderr.Bytes(), err)) + }, nil +} + +// Pipe runs git with stdin and returns a reader over its stdout. +func (r *gitRunner) Pipe(ctx context.Context, stdin []byte, args ...string) (io.ReadCloser, func() error, error) { + cmd := r.command(ctx, args...) + cmd.Stdin = bytes.NewReader(stdin) + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, nil, err + } + stderr := &bytes.Buffer{} + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + return nil, nil, err + } + return stdout, func() error { + err := cmd.Wait() + if err == nil { + return nil + } + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + return fmt.Errorf("git %s failed: %s", strings.Join(sanitizeArgs(args), " "), errorMessage(stderr.Bytes(), err)) + }, nil +} + +func (r *gitRunner) command(ctx context.Context, args ...string) *exec.Cmd { + fullArgs := append([]string{"-c", "core.quotePath=false", "-c", "gc.auto=0", "-c", "maintenance.auto=false"}, args...) + cmd := exec.CommandContext(ctx, "git", fullArgs...) + cmd.Dir = r.dir + cmd.Env = append(cmd.Environ(), + "GIT_OPTIONAL_LOCKS=0", + "GIT_NO_LAZY_FETCH=1", + "GIT_PAGER=cat", + "GIT_TERMINAL_PROMPT=0", + "LC_ALL=C", + ) + return cmd +} + +func errorMessage(stderr []byte, err error) string { + if msg := strings.TrimSpace(string(stderr)); msg != "" { + return msg + } + return err.Error() +} + +func splitLines(out []byte) []string { + text := strings.TrimSuffix(string(out), "\n") + if text == "" { + return nil + } + return strings.Split(text, "\n") +} + +func scanLines(reader io.Reader, fn func(line string) error) error { + scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) + for scanner.Scan() { + if err := fn(scanner.Text()); err != nil { + return err + } + } + return scanner.Err() +} diff --git a/backend/plugins/gitextractor/parser/numstatparse.go b/backend/plugins/gitextractor/parser/numstatparse.go new file mode 100644 index 00000000000..1d96dd6ae89 --- /dev/null +++ b/backend/plugins/gitextractor/parser/numstatparse.go @@ -0,0 +1,130 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "bufio" + "fmt" + "io" + "strconv" + "strings" +) + +const numstatRecordMarker = '\x01' + +type commitFileStat struct { + Path string + Additions int + Deletions int +} + +type commitStats struct { + Sha string + Files []commitFileStat +} + +// parseNumstatStream parses the output of +// +// git log --format=%x01%H --numstat --no-renames -z ... +// +// Each commit is introduced by a \x01 segment; every following NUL +// terminated segment is "additionsdeletionspath". Paths may contain +// tabs, spaces and newlines, but never NUL, so splitting on NUL is safe. +func parseNumstatStream(reader io.Reader, fn func(*commitStats) error) error { + buffered := bufio.NewReaderSize(reader, 256*1024) + + var current *commitStats + flush := func() error { + if current == nil { + return nil + } + if err := fn(current); err != nil { + return err + } + current = nil + return nil + } + + for { + segment, err := buffered.ReadString(0) + if err != nil && err != io.EOF { + return err + } + // ReadString returns data without the delimiter on EOF + segment = strings.TrimSuffix(segment, "\x00") + + switch { + case segment == "": + // nothing to do; covers a trailing NUL + case segment[0] == numstatRecordMarker: + if err := flush(); err != nil { + return err + } + current = &commitStats{Sha: segment[1:]} + default: + if current == nil { + return fmt.Errorf("numstat entry %q before any commit", segment) + } + entry := strings.TrimPrefix(segment, "\n") + stat, err := parseNumstatEntry(entry) + if err != nil { + return fmt.Errorf("commit %s: %w", current.Sha, err) + } + current.Files = append(current.Files, stat) + } + + if err == io.EOF { + break + } + } + return flush() +} + +func parseNumstatEntry(entry string) (commitFileStat, error) { + var stat commitFileStat + parts := strings.SplitN(entry, "\t", 3) + if len(parts) != 3 { + return stat, fmt.Errorf("unexpected numstat entry %q", entry) + } + stat.Path = parts[2] + + additions, err := parseNumstatCount(parts[0]) + if err != nil { + return stat, err + } + deletions, err := parseNumstatCount(parts[1]) + if err != nil { + return stat, err + } + stat.Additions = additions + stat.Deletions = deletions + return stat, nil +} + +// parseNumstatCount parses a numstat count. Binary files are reported as "-" +// and count as zero changed lines, just like libgit2 reports them. +func parseNumstatCount(value string) (int, error) { + if value == "-" { + return 0, nil + } + count, err := strconv.Atoi(value) + if err != nil { + return 0, fmt.Errorf("unexpected numstat count %q", value) + } + return count, nil +} diff --git a/backend/plugins/gitextractor/parser/parse_test.go b/backend/plugins/gitextractor/parser/parse_test.go new file mode 100644 index 00000000000..23783c3296d --- /dev/null +++ b/backend/plugins/gitextractor/parser/parse_test.go @@ -0,0 +1,243 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseNumstatStream(t *testing.T) { + stream := strings.Join([]string{ + "\x01sha1\x00", + "\n1\t0\ta.txt\x00", + "-\t-\tbin.dat\x00", + "\x01sha2\x00", + "\x01sha3\x00", + "\n2\t1\tdir with space/f.txt\x00", + "1\t0\tüñï.txt\x00", + "0\t0\tempty.txt\x00", + "1\t0\tnew\nline.txt\x00", + }, "") + + var commits []*commitStats + err := parseNumstatStream(strings.NewReader(stream), func(commit *commitStats) error { + commits = append(commits, commit) + return nil + }) + require.NoError(t, err) + + require.Len(t, commits, 3) + assert.Equal(t, "sha1", commits[0].Sha) + assert.Equal(t, []commitFileStat{ + {Path: "a.txt", Additions: 1}, + {Path: "bin.dat"}, + }, commits[0].Files) + assert.Equal(t, "sha2", commits[1].Sha) + assert.Empty(t, commits[1].Files) + assert.Equal(t, []commitFileStat{ + {Path: "dir with space/f.txt", Additions: 2, Deletions: 1}, + {Path: "üñï.txt", Additions: 1}, + {Path: "empty.txt"}, + {Path: "new\nline.txt", Additions: 1}, + }, commits[2].Files) +} + +func TestParseNumstatStreamRejectsBadInput(t *testing.T) { + err := parseNumstatStream(strings.NewReader("1\t0\ta.txt\x00"), func(*commitStats) error { return nil }) + assert.Error(t, err, "an entry without a commit marker is invalid") + + err = parseNumstatStream(strings.NewReader("\x01sha\x00\nx\t0\ta.txt\x00"), func(*commitStats) error { return nil }) + assert.Error(t, err, "a non-numeric count is invalid") +} + +func TestParsePatchStream(t *testing.T) { + patch := strings.Join([]string{ + "\x01c1", + "diff --git a/a.txt b/a.txt", + "new file mode 100644", + "index 0000000..1111111", + "--- /dev/null", + "+++ b/a.txt", + "@@ -0,0 +1,3 @@", + "+l1", + "+l2", + "+l3", + "\x01c2", + "diff --git a/a.txt b/a.txt", + "index 1111111..2222222 100644", + "--- a/a.txt", + "+++ b/a.txt", + "@@ -1,3 +1,4 @@ heading", + " l1", + "-l2", + "+L2", + " l3", + "+l4", + "\\ No newline at end of file", + "diff --git a/bin.dat b/bin.dat", + "index 3333333..4444444 100644", + "Binary files a/bin.dat and b/bin.dat differ", + "diff --git a/empty.txt b/empty.txt", + "deleted file mode 100644", + "index 5555555..0000000", + "--- a/empty.txt", + "+++ /dev/null", + "@@ -1 +0,0 @@", + "-x", + "diff --git a/mode.txt b/mode.txt", + "old mode 100644", + "new mode 100755", + "diff --git \"a/tab\\tname.txt\" \"b/tab\\tname.txt\"", + "index 6666666..7777777 100644", + "--- \"a/tab\\tname.txt\"", + "+++ \"b/tab\\tname.txt\"", + "@@ -1 +1 @@", + "-a", + "+b", + "", + }, "\n") + + var commits []*commitPatch + err := parsePatchStream(strings.NewReader(patch), func(commit *commitPatch) error { + commits = append(commits, commit) + return nil + }) + require.NoError(t, err) + + require.Len(t, commits, 2) + + c1 := commits[0] + assert.Equal(t, "c1", c1.Sha) + require.Len(t, c1.Files, 1) + assert.Equal(t, patchFile{ + OldPath: "a.txt", + NewPath: "a.txt", + Lines: []patchLine{ + {Origin: '+', OldLine: -1, NewLine: 1, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 2, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 3, HunkNum: 1}, + }, + }, c1.Files[0]) + + c2 := commits[1] + assert.Equal(t, "c2", c2.Sha) + require.Len(t, c2.Files, 5) + + assert.Equal(t, patchFile{ + OldPath: "a.txt", + NewPath: "a.txt", + Lines: []patchLine{ + {Origin: ' ', OldLine: 1, NewLine: 1, HunkNum: 1}, + {Origin: '-', OldLine: 2, NewLine: -1, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 2, HunkNum: 1}, + {Origin: ' ', OldLine: 3, NewLine: 3, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 4, HunkNum: 1}, + }, + }, c2.Files[0]) + + // binary file: both paths are known, but there are no line changes + assert.Equal(t, patchFile{OldPath: "bin.dat", NewPath: "bin.dat"}, c2.Files[1]) + + // deleted file: libgit2 reports the path on both sides + assert.Equal(t, patchFile{ + OldPath: "empty.txt", + NewPath: "empty.txt", + Lines: []patchLine{{Origin: '-', OldLine: 1, NewLine: -1, HunkNum: 1}}, + }, c2.Files[2]) + + // mode-only change + assert.Equal(t, patchFile{OldPath: "mode.txt", NewPath: "mode.txt"}, c2.Files[3]) + + // quoted path + assert.Equal(t, patchFile{ + OldPath: "tab\tname.txt", + NewPath: "tab\tname.txt", + Lines: []patchLine{ + {Origin: '-', OldLine: 1, NewLine: -1, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 1, HunkNum: 1}, + }, + }, c2.Files[4]) +} + +func TestParsePatchStreamMultipleHunksAndZeroStart(t *testing.T) { + patch := strings.Join([]string{ + "\x01c1", + "diff --git a/a.txt b/a.txt", + "--- a/a.txt", + "+++ b/a.txt", + "@@ -2,1 +2,1 @@", + "-old", + "+new", + "@@ -10,0 +11,2 @@", + "+x", + "+y", + "", + }, "\n") + + var commits []*commitPatch + require.NoError(t, parsePatchStream(strings.NewReader(patch), func(commit *commitPatch) error { + commits = append(commits, commit) + return nil + })) + require.Len(t, commits, 1) + assert.Equal(t, []patchLine{ + {Origin: '-', OldLine: 2, NewLine: -1, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 2, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 11, HunkNum: 2}, + {Origin: '+', OldLine: -1, NewLine: 12, HunkNum: 2}, + }, commits[0].Files[0].Lines) +} + +func TestParseGitVersion(t *testing.T) { + for _, tc := range []struct { + output string + want int + }{ + {"git version 2.43.0\n", 2*10000 + 43*100}, + {"git version 2.31.0", 2*10000 + 31*100}, + {"git version 2.39.2.windows.1", 2*10000 + 39*100 + 2}, + {"git version 2.56.0", 2*10000 + 56*100}, + } { + got, err := parseGitVersion(tc.output) + if assert.NoError(t, err, tc.output) { + assert.Equal(t, tc.want, got, tc.output) + } + } + + _, err := parseGitVersion("not git") + assert.Error(t, err) +} + +func TestUnquoteGitPath(t *testing.T) { + for _, tc := range []struct { + in, want string + }{ + {`a/plain.txt`, `a/plain.txt`}, + {`"a/tab\tname.txt"`, "a/tab\tname.txt"}, + {`"a/quote\"name.txt"`, `a/quote"name.txt`}, + {`"a/back\\slash"`, `a/back\slash`}, + {`"a/\303\274ber.txt"`, "a/über.txt"}, + {`"a/new\nline.txt"`, "a/new\nline.txt"}, + } { + assert.Equal(t, tc.want, unquoteGitPath(tc.in), tc.in) + } +} diff --git a/backend/plugins/gitextractor/parser/patchparse.go b/backend/plugins/gitextractor/parser/patchparse.go new file mode 100644 index 00000000000..e4d3c8dc8a9 --- /dev/null +++ b/backend/plugins/gitextractor/parser/patchparse.go @@ -0,0 +1,332 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "bufio" + "fmt" + "io" + "strconv" + "strings" +) + +const ( + patchChangeContext = ' ' + patchChangeAddition = '+' + patchChangeDeletion = '-' + + // patchRecordMarker introduces the commit sha in the `git log --format` + // output. A patch line can never start with it: every patch content line + // starts with ' ', '+' or '-'. + patchRecordMarker = "\x01" +) + +type patchLine struct { + Origin byte + OldLine int + NewLine int + HunkNum int +} + +type patchFile struct { + OldPath string + NewPath string + Lines []patchLine +} + +type commitPatch struct { + Sha string + Files []patchFile +} + +type patchParser struct { + commit *commitPatch + current *patchFile + + oldLine int + newLine int + hunkNum int + + inHunk bool +} + +// parsePatchStream parses `git log --format=%x01%H -p ...` output. +func parsePatchStream(reader io.Reader, fn func(*commitPatch) error) error { + scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 0, 256*1024), 16*1024*1024) + + var parser *patchParser + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, patchRecordMarker) { + if err := flushPatch(parser, fn); err != nil { + return err + } + parser = &patchParser{commit: &commitPatch{Sha: line[len(patchRecordMarker):]}} + continue + } + if parser == nil { + continue + } + if err := parser.line(line); err != nil { + return err + } + } + if err := scanner.Err(); err != nil { + return err + } + return flushPatch(parser, fn) +} + +func flushPatch(parser *patchParser, fn func(*commitPatch) error) error { + if parser == nil || parser.commit == nil { + return nil + } + parser.endFile() + commit := parser.commit + parser.commit = nil + return fn(commit) +} + +func (p *patchParser) line(line string) error { + switch { + case strings.HasPrefix(line, "diff --git "): + p.endFile() + oldPath, newPath, err := parseDiffGitPaths(strings.TrimPrefix(line, "diff --git ")) + if err != nil { + return err + } + p.current = &patchFile{OldPath: oldPath, NewPath: newPath} + p.inHunk = false + return nil + case strings.HasPrefix(line, "--- "): + if p.current == nil { + p.current = &patchFile{} + } + p.current.OldPath = parsePatchPath(strings.TrimPrefix(line, "--- ")) + return nil + case strings.HasPrefix(line, "+++ "): + if p.current == nil { + p.current = &patchFile{} + } + p.current.NewPath = parsePatchPath(strings.TrimPrefix(line, "+++ ")) + return nil + case strings.HasPrefix(line, "@@ "): + return p.hunkHeader(line) + } + + if !p.inHunk { + return nil + } + + switch line[0] { + case patchChangeContext: + p.appendLine(patchChangeContext, p.oldLine, p.newLine) + p.oldLine++ + p.newLine++ + case patchChangeAddition: + p.appendLine(patchChangeAddition, -1, p.newLine) + p.newLine++ + case patchChangeDeletion: + p.appendLine(patchChangeDeletion, p.oldLine, -1) + p.oldLine++ + case '\\': + // "\ No newline at end of file" + } + return nil +} + +func (p *patchParser) appendLine(origin byte, oldLine, newLine int) { + if p.current == nil { + return + } + p.current.Lines = append(p.current.Lines, patchLine{ + Origin: origin, + OldLine: oldLine, + NewLine: newLine, + HunkNum: p.hunkNum, + }) +} + +func (p *patchParser) hunkHeader(line string) error { + // @@ -oldStart[,oldCount] +newStart[,newCount] @@ optional heading + end := strings.Index(line[3:], " @@") + if end < 0 { + return fmt.Errorf("unexpected hunk header %q", line) + } + fields := strings.SplitN(line[3:3+end], " ", 2) + if len(fields) != 2 { + return fmt.Errorf("unexpected hunk header %q", line) + } + oldStart, err := parseHunkRange(fields[0]) + if err != nil { + return fmt.Errorf("unexpected hunk header %q: %w", line, err) + } + newStart, err := parseHunkRange(fields[1]) + if err != nil { + return fmt.Errorf("unexpected hunk header %q: %w", line, err) + } + p.oldLine = oldStart + p.newLine = newStart + p.hunkNum++ + p.inHunk = true + return nil +} + +func parseHunkRange(value string) (int, error) { + value = strings.TrimPrefix(value, "-") + value = strings.TrimPrefix(value, "+") + if idx := strings.IndexByte(value, ','); idx >= 0 { + value = value[:idx] + } + start, err := strconv.Atoi(value) + if err != nil { + return 0, err + } + return start, nil +} + +func (p *patchParser) endFile() { + if p.current != nil { + // libgit2 reports the same path on both sides for additions and + // deletions, so mirror that here. + if p.current.OldPath == "" { + p.current.OldPath = p.current.NewPath + } + if p.current.NewPath == "" { + p.current.NewPath = p.current.OldPath + } + p.commit.Files = append(p.commit.Files, *p.current) + p.current = nil + } + p.inHunk = false + p.hunkNum = 0 +} + +// parsePatchPath strips the a/ or b/ prefix and unquotes git's C-style path +// quoting. /dev/null becomes the empty path. +func parsePatchPath(value string) string { + if value == "/dev/null" { + return "" + } + return stripGitPathPrefix(unquoteGitPath(value)) +} + +// parseDiffGitPaths extracts both paths from "a/ b/". +// +// Without rename detection both sides are the same path, so finding the split +// point is unambiguous. Quoted paths are parsed as two quoted tokens. +func parseDiffGitPaths(value string) (string, string, error) { + if strings.HasPrefix(value, `"`) { + first, rest, err := readQuotedGitPath(value) + if err != nil { + return "", "", fmt.Errorf("invalid diff header %q: %w", value, err) + } + rest = strings.TrimPrefix(rest, " ") + second, _, err := readQuotedGitPath(rest) + if err != nil { + return "", "", fmt.Errorf("invalid diff header %q: %w", value, err) + } + return stripGitPathPrefix(first), stripGitPathPrefix(second), nil + } + for i := 2; i+3 <= len(value); i++ { + if value[i:i+3] != " b/" { + continue + } + oldPath := value[2:i] + newPath := value[i+3:] + if oldPath == newPath { + return oldPath, newPath, nil + } + } + return "", "", fmt.Errorf("invalid diff header %q", value) +} + +func stripGitPathPrefix(value string) string { + if len(value) > 2 && (value[0] == 'a' || value[0] == 'b') && value[1] == '/' { + return value[2:] + } + return value +} +func readQuotedGitPath(value string) (string, string, error) { + if !strings.HasPrefix(value, `"`) { + return "", value, fmt.Errorf("expected a quoted path") + } + for i := 1; i < len(value); i++ { + if value[i] == '\\' { + i++ + continue + } + if value[i] == '"' { + return unquoteGitPath(value[:i+1]), value[i+1:], nil + } + } + return "", value, fmt.Errorf("unterminated quoted path") +} + +// unquoteGitPath decodes git's C-style quoted paths, e.g. "a/\303\274ber". +func unquoteGitPath(value string) string { + if len(value) < 2 || value[0] != '"' || value[len(value)-1] != '"' { + return value + } + var out strings.Builder + for i := 1; i < len(value)-1; i++ { + char := value[i] + if char != '\\' { + out.WriteByte(char) + continue + } + i++ + if i >= len(value)-1 { + break + } + switch value[i] { + case 'a': + out.WriteByte('\a') + case 'b': + out.WriteByte('\b') + case 'f': + out.WriteByte('\f') + case 'n': + out.WriteByte('\n') + case 'r': + out.WriteByte('\r') + case 't': + out.WriteByte('\t') + case 'v': + out.WriteByte('\v') + case '\\', '"': + out.WriteByte(value[i]) + case '0', '1', '2', '3', '4', '5', '6', '7': + end := i + 3 + if end > len(value)-1 { + end = len(value) - 1 + } + code, err := strconv.ParseUint(value[i:end], 8, 8) + if err != nil { + out.WriteByte(value[i]) + continue + } + out.WriteByte(byte(code)) + i = end - 1 + default: + out.WriteByte(value[i]) + } + } + return out.String() +} diff --git a/backend/plugins/gitextractor/parser/repo_gitcli.go b/backend/plugins/gitextractor/parser/repo_gitcli.go new file mode 100644 index 00000000000..4cf1fc0d1e3 --- /dev/null +++ b/backend/plugins/gitextractor/parser/repo_gitcli.go @@ -0,0 +1,753 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/apache/devlake/core/dal" + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/log" + "github.com/apache/devlake/core/models/domainlayer" + "github.com/apache/devlake/core/models/domainlayer/code" + "github.com/apache/devlake/core/plugin" + "github.com/apache/devlake/plugins/gitextractor/models" +) + +// minGitVersion is the first release with --diff-merges=first-parent, which is +// required to diff merge commits against their first parent only. +const minGitVersion = 2*10000 + 31 + +// commitBatchSize bounds how many commits are fetched and diffed at a time so +// that a repository with a long history does not need to be held in memory. +const commitBatchSize = 1024 + +var _ RepoCollector = (*GitcliRepoCollector)(nil) + +// GitcliRepoCollector collects git data with the git command line, so the +// server no longer needs libgit2 or a pure Go git implementation. +type GitcliRepoCollector struct { + id string + logger log.Logger + store models.Store + git *gitRunner + cleanup func() +} + +func NewGitcliRepoCollector(localDir string, repoId string, store models.Store, logger log.Logger) (*GitcliRepoCollector, errors.Error) { + git := newGitRunner(localDir) + if _, err := git.Output(context.Background(), "rev-parse", "--git-dir"); err != nil { + return nil, errors.Convert(err) + } + version, err := gitVersion(git) + if err != nil { + return nil, errors.Convert(err) + } + if version < minGitVersion { + return nil, errors.Default.New(fmt.Sprintf( + "git %s is not supported, gitextractor requires git 2.31 or newer", formatGitVersion(version))) + } + return &GitcliRepoCollector{ + id: repoId, + logger: logger, + store: store, + git: git, + }, nil +} + +func (r *GitcliRepoCollector) SetCleanUp(f func()) error { + if f != nil { + r.cleanup = f + } + return nil +} + +func (r *GitcliRepoCollector) Close(ctx context.Context) error { + defer func() { + if r.cleanup != nil { + r.cleanup() + } + }() + return r.store.Close() +} + +// CollectAll The main parser subtask +func (r *GitcliRepoCollector) CollectAll(subtaskCtx plugin.SubTaskContext) error { + subtaskCtx.SetProgress(0, -1) + if err := r.CollectTags(subtaskCtx); err != nil { + return err + } + if err := r.CollectBranches(subtaskCtx); err != nil { + return err + } + if err := r.CollectCommits(subtaskCtx); err != nil { + return err + } + if *subtaskCtx.GetData().(*GitExtractorTaskData).Options.SkipCommitStat { + return nil + } + return r.CollectDiffLine(subtaskCtx) +} + +// CountTags Count git tags subtask +func (r *GitcliRepoCollector) CountTags(ctx context.Context) (int, error) { + tags, err := r.tagRefs(ctx) + if err != nil { + return 0, err + } + return len(tags), nil +} + +// CountBranches count the number of branches in a git repo +func (r *GitcliRepoCollector) CountBranches(ctx context.Context) (int, error) { + branches, err := r.branchRefs(ctx) + if err != nil { + return 0, err + } + head, err := r.headRef(ctx) + if err != nil { + return 0, err + } + count := 0 + for _, branch := range branches { + if branch.refname != head { + count++ + } + } + return count, nil +} + +// CountCommits count the number of commits in a git repo +func (r *GitcliRepoCollector) CountCommits(ctx context.Context) (int, error) { + out, err := r.git.Output(ctx, "rev-list", "--all", "--count") + if err != nil { + return 0, err + } + count, err := strconv.Atoi(strings.TrimSpace(string(out))) + if err != nil { + return 0, fmt.Errorf("unexpected commit count %q", strings.TrimSpace(string(out))) + } + return count, nil +} + +// CollectTags Collect Tags data +func (r *GitcliRepoCollector) CollectTags(subtaskCtx plugin.SubTaskContext) error { + tags, err := r.tagRefs(subtaskCtx.GetContext()) + if err != nil { + return err + } + for _, tag := range tags { + commitSha := tag.peeledSha + if commitSha == "" { + commitSha = tag.sha + } + ref := &code.Ref{ + DomainEntityExtended: domainlayer.DomainEntityExtended{Id: fmt.Sprintf("%s:%s", r.id, tag.refname)}, + RepoId: r.id, + Name: tag.refname, + CommitSha: commitSha, + RefType: TAG, + } + if err := r.store.Refs(ref); err != nil { + return err + } + subtaskCtx.IncProgress(1) + } + return nil +} + +// CollectBranches Collect branch data +func (r *GitcliRepoCollector) CollectBranches(subtaskCtx plugin.SubTaskContext) error { + branches, err := r.branchRefs(subtaskCtx.GetContext()) + if err != nil { + return err + } + head, err := r.headRef(subtaskCtx.GetContext()) + if err != nil { + return err + } + for _, branch := range branches { + ref := &code.Ref{ + DomainEntityExtended: domainlayer.DomainEntityExtended{Id: fmt.Sprintf("%s:%s", r.id, branch.name)}, + RepoId: r.id, + Name: branch.name, + CommitSha: branch.sha, + RefType: BRANCH, + IsDefault: branch.refname == head, + } + if err := r.store.Refs(ref); err != nil { + return err + } + subtaskCtx.IncProgress(1) + } + return nil +} + +// CollectCommits Collect data from each commit, we can also get the diff line +func (r *GitcliRepoCollector) CollectCommits(subtaskCtx plugin.SubTaskContext) (err error) { + ctx := subtaskCtx.GetContext() + taskOpts := subtaskCtx.GetData().(*GitExtractorTaskData).Options + + shas, err := r.git.OutputLines(ctx, "rev-list", "--all") + if err != nil { + return err + } + if len(shas) == 0 { + return nil + } + present := make(map[string]struct{}, len(shas)) + for _, sha := range shas { + present[sha] = struct{}{} + } + shallow, err := r.shallowCommits(ctx) + if err != nil { + return err + } + componentMap, err := r.componentMap(subtaskCtx) + if err != nil { + return err + } + excluded := normalizeExtensions(taskOpts.ExcludeFileExtensions) + + for start := 0; start < len(shas); start += commitBatchSize { + end := start + commitBatchSize + if end > len(shas) { + end = len(shas) + } + batch := shas[start:end] + + var statsBySha map[string]*commitStats + if !*taskOpts.SkipCommitStat { + statsBySha, err = r.commitStats(ctx, batch) + if err != nil { + return err + } + } + commits, err := r.readCommits(ctx, batch) + if err != nil { + return err + } + for _, metadata := range commits { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + // A shallow boundary commit's parent is not in the object + // database. Diffing it would produce a full-tree diff and inflate + // the addition count, so it is skipped entirely. + if _, ok := shallow[metadata.Sha]; ok { + r.logger.Info("skip commit %s because it is a shallow boundary", metadata.Sha) + continue + } + // The same guard as above, for repositories where the boundary is + // not recorded in the shallow file. + if len(metadata.Parents) > 0 { + if _, ok := present[metadata.Parents[0]]; !ok { + r.logger.Info("skip commit %s because it has no parent commit", metadata.Sha) + continue + } + } + + commit := &code.Commit{ + Sha: metadata.Sha, + Message: metadata.Message, + AuthorName: metadata.Author.Name, + AuthorEmail: metadata.Author.Email, + AuthorId: metadata.Author.Email, + AuthoredDate: metadata.Author.When, + CommitterName: metadata.Committer.Name, + CommitterEmail: metadata.Committer.Email, + CommitterId: metadata.Committer.Email, + CommittedDate: metadata.Committer.When, + } + + var files []commitFileStat + if stats := statsBySha[metadata.Sha]; stats != nil { + files = stats.Files + } + for _, file := range files { + if isExcludedPath(file.Path, excluded) { + continue + } + commit.Additions += file.Additions + commit.Deletions += file.Deletions + } + + if err := r.storeParentCommits(metadata, present); err != nil { + return err + } + if err := r.store.Commits(commit); err != nil { + return err + } + if err := r.store.RepoCommits(&code.RepoCommit{RepoId: r.id, CommitSha: commit.Sha}); err != nil { + return err + } + + if !*taskOpts.SkipCommitFiles { + for _, file := range files { + if isExcludedPath(file.Path, excluded) { + continue + } + if err := r.storeCommitFile(commit.Sha, file, componentMap); err != nil { + return err + } + } + } + subtaskCtx.IncProgress(1) + } + } + return nil +} + +func (r *GitcliRepoCollector) storeParentCommits(commit *parsedCommit, present map[string]struct{}) errors.Error { + var parents []*code.CommitParent + for _, parentSha := range commit.Parents { + // Skip parents that are not in the object database, which happens for + // shallow clones, instead of storing a dangling reference. + if _, ok := present[parentSha]; !ok { + continue + } + parents = append(parents, &code.CommitParent{ + CommitSha: commit.Sha, + ParentCommitSha: parentSha, + }) + } + return r.store.CommitParents(parents) +} + +func (r *GitcliRepoCollector) storeCommitFile(commitSha string, file commitFileStat, componentMap map[string]*regexp.Regexp) errors.Error { + commitFile := &code.CommitFile{ + CommitSha: commitSha, + FilePath: file.Path, + Additions: file.Additions, + Deletions: file.Deletions, + } + commitFile.Id = genCommitFileId(commitFile.CommitSha, commitFile.FilePath) + + componentName := "Default" + for _, name := range sortedComponentNames(componentMap) { + if componentMap[name].MatchString(commitFile.FilePath) { + componentName = name + break + } + } + if err := r.store.CommitFileComponents(&code.CommitFileComponent{ + CommitFileId: commitFile.Id, + ComponentName: componentName, + }); err != nil { + return err + } + return r.store.CommitFiles(commitFile) +} + +// CollectDiffLine replays the first-parent history of HEAD and records the +// origin commit of every line, plus the per-line change rows. +func (r *GitcliRepoCollector) CollectDiffLine(subtaskCtx plugin.SubTaskContext) error { + ctx := subtaskCtx.GetContext() + + if _, err := r.git.Output(ctx, "rev-parse", "--verify", "HEAD"); err != nil { + // an empty repository has no HEAD and therefore no snapshot + r.logger.Info("skip diff line collection because the repository has no HEAD") + return nil + } + + reader, wait, err := r.git.Stream(ctx, + "log", "--first-parent", "--reverse", "--no-renames", "-p", "-U3", + "--format="+patchRecordMarker+"%H", + "--no-color", "--no-ext-diff", "--no-show-signature", "--no-notes", + "HEAD", + ) + if err != nil { + return err + } + + snapshot := make(map[string]*models.FileBlame) + parseErr := parsePatchStream(reader, func(patch *commitPatch) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + return r.storeSnapshotForCommit(patch, snapshot) + }) + waitErr := wait() + if parseErr != nil { + return parseErr + } + if waitErr != nil { + return waitErr + } + + db := subtaskCtx.GetDal() + if err := db.Delete(&code.RepoSnapshot{}, dal.Where("repo_id= ?", r.id)); err != nil { + return err + } + + files := make([]string, 0, len(snapshot)) + for file := range snapshot { + files = append(files, file) + } + sort.Strings(files) + for _, file := range files { + lineNo := 0 + for element := snapshot[file].Lines.Front(); element != nil; element = element.Next() { + lineNo++ + commitSha, ok := element.Value.(string) + if !ok { + // A placeholder without a commit should not happen, but do not + // fail the whole collection if it does. + r.logger.Warn(nil, "empty snapshot line for %s:%d", file, lineNo) + continue + } + if err := r.store.RepoSnapshot(&code.RepoSnapshot{ + RepoId: r.id, + CommitSha: commitSha, + FilePath: file, + LineNo: lineNo, + }); err != nil { + return err + } + } + } + r.logger.Info("collect snapshot finished") + return nil +} + +func (r *GitcliRepoCollector) storeSnapshotForCommit(patch *commitPatch, snapshot map[string]*models.FileBlame) error { + pending := &snapshotChanges{} + lastFile := "" + + for _, file := range patch.Files { + if _, ok := snapshot[file.OldPath]; !ok { + fileBlame, err := models.NewFileBlame() + if err != nil { + return err + } + snapshot[file.OldPath] = fileBlame + } + if lastFile == "" { + lastFile = file.NewPath + } else if lastFile != file.NewPath { + applySnapshotChanges(snapshot, lastFile, patch.Sha, pending) + pending = &snapshotChanges{} + lastFile = file.NewPath + } + + for _, line := range file.Lines { + change := &code.CommitLineChange{ + CommitSha: patch.Sha, + ChangedType: lineChangeType(line.Origin), + LineNoNew: line.NewLine, + LineNoOld: line.OldLine, + OldFilePath: file.OldPath, + NewFilePath: file.NewPath, + HunkNum: line.HunkNum, + Id: fmt.Sprintf("%s:%s:%d:%d", patch.Sha, file.NewPath, line.OldLine, line.NewLine), + } + switch line.Origin { + case patchChangeAddition: + pending.added = append(pending.added, line) + case patchChangeDeletion: + if blame := snapshot[file.OldPath]; blame != nil { + if element := blame.Find(line.OldLine); element != nil && element.Value != nil { + change.PrevCommit = element.Value.(string) + } + } + pending.deleted = append(pending.deleted, line) + } + if err := r.store.CommitLineChange(change); err != nil { + return err + } + } + } + applySnapshotChanges(snapshot, lastFile, patch.Sha, pending) + return nil +} + +type snapshotChanges struct { + deleted []patchLine + added []patchLine +} + +// applySnapshotChanges mirrors the line bookkeeping of the libgit2 collector: +// deletions remove the old line numbers and additions insert the commit sha at +// the new line numbers. +func applySnapshotChanges(snapshot map[string]*models.FileBlame, filePath, commitSha string, changes *snapshotChanges) { + fileBlame, ok := snapshot[filePath] + if !ok || fileBlame == nil || (len(changes.deleted) == 0 && len(changes.added) == 0) { + return + } + sort.SliceStable(changes.deleted, func(i, j int) bool { + return changes.deleted[i].OldLine > changes.deleted[j].OldLine + }) + for _, line := range changes.deleted { + fileBlame.RemoveLine(line.OldLine) + } + for _, line := range changes.added { + fileBlame.AddLine(line.NewLine, commitSha) + } +} + +func lineChangeType(origin byte) string { + switch origin { + case patchChangeAddition: + return "Addition" + case patchChangeDeletion: + return "Deletion" + default: + return "Context" + } +} + +// readCommits returns the commit objects of the batch in the order of the batch. +func (r *GitcliRepoCollector) readCommits(ctx context.Context, shas []string) ([]*parsedCommit, error) { + stdin := []byte(strings.Join(shas, "\n") + "\n") + reader, wait, err := r.git.Pipe(ctx, stdin, "cat-file", "--batch") + if err != nil { + return nil, err + } + buffered := bufio.NewReaderSize(reader, 256*1024) + commits := make([]*parsedCommit, 0, len(shas)) + for { + sha, content, readErr := readCatFileObject(buffered) + if readErr == io.EOF { + break + } + if readErr != nil { + _ = wait() + return nil, readErr + } + commit, parseErr := parseCommitObject(sha, content) + if parseErr != nil { + _ = wait() + return nil, parseErr + } + commits = append(commits, commit) + } + if err := wait(); err != nil { + return nil, err + } + return commits, nil +} + +// commitStats returns the per-file line counts of the batch. +func (r *GitcliRepoCollector) commitStats(ctx context.Context, shas []string) (map[string]*commitStats, error) { + stdin := []byte(strings.Join(shas, "\n") + "\n") + reader, wait, err := r.git.Pipe(ctx, stdin, + "log", "--no-walk=unsorted", "--stdin", "--root", "--diff-merges=first-parent", + "--numstat", "--no-renames", "-z", "--format="+patchRecordMarker+"%H", + "--no-color", "--no-ext-diff", "--no-show-signature", "--no-notes", + ) + if err != nil { + return nil, err + } + statsBySha := make(map[string]*commitStats, len(shas)) + err = parseNumstatStream(reader, func(stats *commitStats) error { + statsBySha[stats.Sha] = stats + return nil + }) + waitErr := wait() + if err != nil { + return nil, err + } + if waitErr != nil { + return nil, waitErr + } + return statsBySha, nil +} + +// shallowCommits returns the commits recorded as shallow boundaries. +func (r *GitcliRepoCollector) shallowCommits(ctx context.Context) (map[string]struct{}, error) { + shallow := make(map[string]struct{}) + out, err := r.git.Output(ctx, "rev-parse", "--git-path", "shallow") + if err != nil { + return shallow, err + } + content, err := os.ReadFile(strings.TrimSpace(string(out))) + if os.IsNotExist(err) { + return shallow, nil + } + if err != nil { + return shallow, err + } + for _, line := range strings.Split(string(content), "\n") { + if sha := strings.TrimSpace(line); sha != "" { + shallow[sha] = struct{}{} + } + } + return shallow, nil +} + +type tagRef struct { + refname string + sha string + peeledSha string +} + +func (r *GitcliRepoCollector) tagRefs(ctx context.Context) ([]tagRef, error) { + lines, err := r.git.OutputLines(ctx, "for-each-ref", + "--format=%(refname)%00%(objectname)%00%(*objectname)", + "refs/tags") + if err != nil { + return nil, err + } + tags := make([]tagRef, 0, len(lines)) + for _, line := range lines { + fields := strings.SplitN(line, "\x00", 3) + if len(fields) != 3 { + return nil, fmt.Errorf("unexpected tag ref %q", line) + } + tags = append(tags, tagRef{refname: fields[0], sha: fields[1], peeledSha: fields[2]}) + } + return tags, nil +} + +type branchRef struct { + refname string + name string + sha string +} + +func (r *GitcliRepoCollector) branchRefs(ctx context.Context) ([]branchRef, error) { + lines, err := r.git.OutputLines(ctx, "for-each-ref", + "--format=%(refname)%00%(objectname)", + "refs/heads", "refs/remotes") + if err != nil { + return nil, err + } + branches := make([]branchRef, 0, len(lines)) + for _, line := range lines { + fields := strings.SplitN(line, "\x00", 2) + if len(fields) != 2 { + return nil, fmt.Errorf("unexpected branch ref %q", line) + } + name := strings.TrimPrefix(fields[0], "refs/heads/") + name = strings.TrimPrefix(name, "refs/remotes/") + branches = append(branches, branchRef{refname: fields[0], name: name, sha: fields[1]}) + } + return branches, nil +} + +func (r *GitcliRepoCollector) headRef(ctx context.Context) (string, error) { + out, err := r.git.Output(ctx, "symbolic-ref", "-q", "HEAD") + if err != nil { + // a detached HEAD has no default branch + return "", nil + } + return strings.TrimSpace(string(out)), nil +} + +func (r *GitcliRepoCollector) componentMap(subtaskCtx plugin.SubTaskContext) (map[string]*regexp.Regexp, error) { + db := subtaskCtx.GetDal() + components := make([]code.Component, 0) + if err := db.All(&components, dal.From(components), dal.Where("repo_id= ?", r.id)); err != nil { + return nil, err + } + componentMap := make(map[string]*regexp.Regexp, len(components)) + for _, component := range components { + componentMap[component.Name] = regexp.MustCompile(component.PathRegex) + } + return componentMap, nil +} + +func sortedComponentNames(componentMap map[string]*regexp.Regexp) []string { + names := make([]string, 0, len(componentMap)) + for name := range componentMap { + names = append(names, name) + } + sort.Strings(names) + return names +} + +func normalizeExtensions(extensions []string) map[string]struct{} { + excluded := make(map[string]struct{}, len(extensions)) + for _, extension := range extensions { + value := strings.ToLower(strings.TrimSpace(extension)) + if value == "" { + continue + } + excluded[value] = struct{}{} + } + return excluded +} + +func isExcludedPath(path string, excluded map[string]struct{}) bool { + if len(excluded) == 0 { + return false + } + lower := strings.ToLower(path) + for extension := range excluded { + if strings.HasSuffix(lower, extension) { + return true + } + } + return false +} + +// gitVersion returns the version of git as major*10000 + minor*100 + patch. +func gitVersion(git *gitRunner) (int, error) { + out, err := git.Output(context.Background(), "version") + if err != nil { + return 0, err + } + return parseGitVersion(string(out)) +} + +func parseGitVersion(output string) (int, error) { + // e.g. "git version 2.43.0", "git version 2.39.2.windows.1" + fields := strings.Fields(output) + if len(fields) < 3 || fields[0] != "git" || fields[1] != "version" { + return 0, fmt.Errorf("unexpected git version output %q", strings.TrimSpace(output)) + } + parts := strings.SplitN(fields[2], ".", 3) + if len(parts) < 2 { + return 0, fmt.Errorf("unexpected git version %q", fields[2]) + } + major, err := strconv.Atoi(parts[0]) + if err != nil { + return 0, fmt.Errorf("unexpected git version %q", fields[2]) + } + minor, err := strconv.Atoi(parts[1]) + if err != nil { + return 0, fmt.Errorf("unexpected git version %q", fields[2]) + } + patch := 0 + if len(parts) == 3 { + if idx := strings.IndexFunc(parts[2], func(r rune) bool { return r < '0' || r > '9' }); idx >= 0 { + parts[2] = parts[2][:idx] + } + if parts[2] != "" { + patch, _ = strconv.Atoi(parts[2]) + } + } + return major*10000 + minor*100 + patch, nil +} + +func formatGitVersion(version int) string { + return fmt.Sprintf("%d.%d.%d", version/10000, version%10000/100, version%100) +} diff --git a/backend/plugins/gitextractor/parser/taskdata.go b/backend/plugins/gitextractor/parser/taskdata.go index bdfdbd2ae1b..a8bb4510d9a 100644 --- a/backend/plugins/gitextractor/parser/taskdata.go +++ b/backend/plugins/gitextractor/parser/taskdata.go @@ -49,4 +49,8 @@ type GitExtractorOptions struct { PluginName string `json:"pluginName" mapstructure:"pluginName,omitempty"` // Configured by upstream plugin (e.g., GitLab) to exclude file extensions from commit stats ExcludeFileExtensions []string `json:"excludeFileExtensions" mapstructure:"excludeFileExtensions"` + // UseGitCli selects the git command line collector. It is a temporary + // migration switch and will be removed once the library based collectors + // are gone. + UseGitCli bool `json:"useGitCli,omitempty" mapstructure:"useGitCli"` } diff --git a/backend/plugins/gitextractor/tasks/repo_cloner.go b/backend/plugins/gitextractor/tasks/repo_cloner.go index 76013e28e96..63e58527b54 100644 --- a/backend/plugins/gitextractor/tasks/repo_cloner.go +++ b/backend/plugins/gitextractor/tasks/repo_cloner.go @@ -71,7 +71,9 @@ func CloneGitRepo(subTaskCtx plugin.SubTaskContext) errors.Error { } // We have done comparison experiments for git2go and go-git, and the results show that git2go has better performance. var repoCollector parser.RepoCollector - if *taskData.Options.UseGoGit { + if taskData.Options.UseGitCli { + repoCollector, err = parser.NewGitcliRepoCollector(localDir, op.RepoId, storage, logger) + } else if *taskData.Options.UseGoGit { repoCollector, err = parser.NewGogitRepoCollector(localDir, op.RepoId, storage, logger) } else { repoCollector, err = parser.NewLibgit2RepoCollector(localDir, op.RepoId, storage, logger) From 9f5ae911e853e1dff9595253a9bf6eb2f030c5b2 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Sun, 1 Nov 2026 13:18:17 +0800 Subject: [PATCH 3/5] refactor(gitextractor): remove libgit2 and go-git Drop the libgit2 (git2go) and go-git collectors now that the git CLI collector passes the contract suite. The backend has no import "C" left and builds with CGO_ENABLED=0. - delete repo_libgit2.go, repo_gogit.go and the git2go sort adapter - drop the useGoGit option/flag and the temporary gitcli switch - remove the go-git and git2go modules - remove install-libgit2.sh, the libgit2 build/copy steps from the Dockerfiles and devcontainer, and the CI install steps - drop the git2go dependabot ignore and USE_GO_GIT_IN_GIT_EXTRACTOR - update the gitextractor README, test docs and agent notes Behaviour changes inherited from the new collector: only commits reachable from refs are collected (dangling ODB objects are ignored), a component row is written for every commit file, and shallow boundary commits are skipped. --- .devcontainer/Dockerfile | 5 +- .github/dependabot.yml | 3 - .github/workflows/golangci-lint.yml | 2 - .github/workflows/test-e2e.yml | 4 - .github/workflows/test.yml | 2 - .gitignore | 1 - AGENTS.md | 2 +- backend/DevelopmentManual.md | 2 +- backend/Dockerfile | 75 +- backend/Dockerfile.local | 27 +- backend/go.mod | 18 - backend/go.sum | 65 -- backend/plugins/gitextractor/README.md | 5 +- backend/plugins/gitextractor/gitextractor.go | 4 - backend/plugins/gitextractor/impl/impl.go | 5 - .../plugins/gitextractor/models/diffline.go | 39 - .../gitextractor/parser/clone_gitcli.go | 8 +- .../gitextractor/parser/clone_gitcli_test.go | 2 +- .../parser/collector_contract_test.go | 252 ++----- .../gitextractor/parser/commitparse_test.go | 2 +- .../gitextractor/parser/numstatparse.go | 2 +- .../plugins/gitextractor/parser/parse_test.go | 20 +- .../plugins/gitextractor/parser/patchparse.go | 4 +- .../gitextractor/parser/repo_gitcli.go | 33 +- .../plugins/gitextractor/parser/repo_gogit.go | 626 ---------------- .../gitextractor/parser/repo_libgit2.go | 698 ------------------ .../plugins/gitextractor/parser/taskdata.go | 5 - .../plugins/gitextractor/tasks/repo_cloner.go | 10 +- backend/plugins/kiro/impl/table_info_test.go | 7 +- backend/python/DevelopmentSetup.md | 2 +- backend/scripts/install-libgit2.sh | 126 ---- backend/test/Readme.md | 11 +- devops/docker/lake-builder/Dockerfile | 39 +- env.example | 3 - 34 files changed, 152 insertions(+), 1957 deletions(-) delete mode 100644 backend/plugins/gitextractor/models/diffline.go delete mode 100644 backend/plugins/gitextractor/parser/repo_gogit.go delete mode 100644 backend/plugins/gitextractor/parser/repo_libgit2.go delete mode 100755 backend/scripts/install-libgit2.sh diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b7099ace08b..e795825084f 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -15,10 +15,7 @@ FROM mcr.microsoft.com/devcontainers/go:2-1.27-bookworm -ENV PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/lib:/usr/local/lib/pkgconfig -ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib ENV DEVLAKE_PLUGINS=argocd,bamboo,bitbucket,circleci,customize,dora,gitextractor,github,github_graphql,gitlab,issue_trace,jenkins,jira,org,pagerduty,refdiff,slack,sonarqube,trello,webhook RUN apt-get update -y -RUN apt-get install pkg-config python3-dev default-libmysqlclient-dev build-essential libpq-dev cmake -y -RUN git clone -b v1.5.0 https://github.com/libgit2/libgit2.git && cd libgit2 && mkdir build && cd build && cmake .. && make && make install +RUN apt-get install python3-dev default-libmysqlclient-dev build-essential libpq-dev -y diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 41f2b2a725b..aaf59c2d52a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -58,9 +58,6 @@ updates: # The Go toolchain directive is coupled to the CI bootstrap version and # to the golang base images; it is raised deliberately, never by a bot. - dependency-name: go - # git2go is ABI-coupled to the libgit2 version installed in the builder - # image. A mismatch fails at link time, so this is always manual. - - dependency-name: github.com/libgit2/git2go/* # swag v2 is still a release candidate and requires regenerating every # swagger annotation. Tracked as an externally gated upgrade. - dependency-name: github.com/swaggo/swag diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 0df4d8308bc..a21ca46c31c 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -32,8 +32,6 @@ jobs: - uses: actions/checkout@v7 - name: Set git config run: git config --global --add safe.directory $(pwd) - - name: Install libgit2 1.5 for git2go v34 - run: backend/scripts/install-libgit2.sh - name: Install Go 1.27.3 run: backend/scripts/install-go.sh - name: Install mockery 3.8.1 diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 3499df44ed0..1512e7273ae 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -50,8 +50,6 @@ jobs: - name: Checkout code uses: actions/checkout@v7 - run: git config --global --add safe.directory $(pwd) - - name: Install libgit2 1.5 for git2go v34 - run: backend/scripts/install-libgit2.sh - name: Install Go 1.27.3 run: backend/scripts/install-go.sh - name: Build Python @@ -94,8 +92,6 @@ jobs: - name: Checkout code uses: actions/checkout@v7 - run: git config --global --add safe.directory $(pwd) - - name: Install libgit2 1.5 for git2go v34 - run: backend/scripts/install-libgit2.sh - name: Install Go 1.27.3 run: backend/scripts/install-go.sh - name: Build Python diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 86e2ea95e32..6c0ac07079c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,8 +47,6 @@ jobs: uses: actions/checkout@v7 - name: Set git config run: git config --global --add safe.directory $(pwd) - - name: Install libgit2 1.5 for git2go v34 - run: backend/scripts/install-libgit2.sh - name: Install Go 1.27.3 run: backend/scripts/install-go.sh - name: Check go.mod is tidy without generated mocks diff --git a/.gitignore b/.gitignore index 0877343e6bc..7097c0187d1 100644 --- a/.gitignore +++ b/.gitignore @@ -147,7 +147,6 @@ NOTES.md main vendor bin -libgit2 .air.toml # Playwright CLI snapshots/logs (local dev) diff --git a/AGENTS.md b/AGENTS.md index b896b7af69c..f558e08db3b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -129,4 +129,4 @@ Located in `backend/python/plugins/`. Use Poetry for dependencies. See [backend/ - Forgetting to add models to `GetTablesInfo()` fails `plugins/table_info_test.go` - Migration scripts must be added to `All()` in `register.go` - API changes require running `make swag` to update Swagger docs -- Python plugins require `libgit2` for gitextractor functionality +- `gitextractor` shells out to the `git` CLI; no git library or cgo is required diff --git a/backend/DevelopmentManual.md b/backend/DevelopmentManual.md index 26208d55ff5..96ba60958db 100644 --- a/backend/DevelopmentManual.md +++ b/backend/DevelopmentManual.md @@ -161,7 +161,7 @@ func TestDevlakeServer(t *testing.T) { Plugins: []plugin.PluginMeta{ // list of plugins to make available to the server. Just create an empty instance of your intended plugin structs (conventionally declared in their respective `impl.go`) github.Github{}, githubGraphql.GithubGraphql{}, - gitextractor.GitExtractor{}, // this requires that your host machine has libgit2 installed, otherwise you'll get compilation errors + gitextractor.GitExtractor{}, gitlab.Gitlab(""), jenkins.Jenkins{}, webhook.Webhook{}, diff --git a/backend/Dockerfile b/backend/Dockerfile index 877a1ac7b88..9ed02234a0a 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -23,14 +23,6 @@ #While incubation status is not necessarily a reflection of the completeness or stability of the code, #it does indicate that the project has yet to be fully endorsed by the ASF. -FROM --platform=linux/amd64 debian:bookworm as debian-amd64 -RUN apt-get update -RUN apt-get install -y libssh2-1-dev libssl-dev zlib1g-dev - -FROM --platform=linux/arm64 debian:bookworm as debian-arm64 -RUN apt-get update -RUN apt-get install -y libssh2-1-dev libssl-dev zlib1g-dev - FROM --platform=$BUILDPLATFORM golang:1.27-bookworm as builder # docker build --build-arg GOPROXY=https://goproxy.io,direct -t mericodev/lake . @@ -40,47 +32,11 @@ ARG HTTP_PROXY= ARG HTTPS_PROXY= RUN apt-get update -RUN apt-get install -y gcc binutils libfindbin-libs-perl cmake libssh2-1-dev libssl-dev zlib1g-dev - -RUN if [ "$(arch)" != "aarch64" ] ; then \ - apt-get install -y gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu ; \ - fi -RUN if [ "$(arch)" != "x86_64" ] ; then \ - apt-get install -y gcc-x86-64-linux-gnu binutils-x86-64-linux-gnu ; \ - fi +RUN apt-get install -y ca-certificates git RUN go install github.com/vektra/mockery/v3@v3.8.1 RUN go install github.com/swaggo/swag/cmd/swag@v1.16.6 -COPY --from=debian-amd64 /usr/include /rootfs-amd64/usr/include -COPY --from=debian-amd64 /usr/lib/x86_64-linux-gnu /rootfs-amd64/usr/lib/x86_64-linux-gnu -COPY --from=debian-amd64 /lib/x86_64-linux-gnu /rootfs-amd64/lib/x86_64-linux-gnu - -COPY --from=debian-arm64 /usr/include /rootfs-arm64/usr/include -COPY --from=debian-arm64 /usr/lib/aarch64-linux-gnu /rootfs-arm64/usr/lib/aarch64-linux-gnu -COPY --from=debian-arm64 /lib/aarch64-linux-gnu /rootfs-arm64/lib/aarch64-linux-gnu - - -# set -e: otherwise a build failure for one arch is silently skipped and the -# other arch's libgit2 is never installed under /usr/local/deps. -RUN set -e; \ - for arch in aarch64 x86_64 ; do \ - mkdir -p /tmp/build/${arch} && cd /tmp/build/${arch} && \ - wget https://github.com/libgit2/libgit2/archive/refs/tags/v1.5.0.tar.gz -O - | tar -xz && \ - cd libgit2-1.5.0 && \ - mkdir build && cd build && \ - if [ "$arch" = "aarch64" ] ; then \ - cmake .. -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc \ - -DBUILD_SHARED_LIBS=ON -DUSE_BUNDLED_ZLIB=ON -DCMAKE_SYSROOT=/rootfs-arm64 \ - -DCMAKE_INSTALL_PREFIX=/usr/local/deps/${arch} ; \ - elif [ "$arch" = "x86_64" ] ; then \ - cmake .. -DCMAKE_C_COMPILER=x86_64-linux-gnu-gcc \ - -DBUILD_SHARED_LIBS=ON -DUSE_BUNDLED_ZLIB=ON -DCMAKE_SYSROOT=/rootfs-amd64 \ - -DCMAKE_INSTALL_PREFIX=/usr/local/deps/${arch} ; \ - fi && \ - make -j install ; \ - done - FROM builder as build @@ -95,35 +51,20 @@ ARG GO_PLUGINS= RUN --mount=type=cache,target=/root/.cache/go-build \ if [ "$TARGETPLATFORM" = "linux/arm64" ] ; then \ - ln -s /usr/local/deps/aarch64 /usr/local/deps/target && \ - export CC=aarch64-linux-gnu-gcc && \ - export GOARCH=arm64 && \ - export CGO_LDFLAGS="-Wl,-rpath-link,/rootfs-arm64/usr/lib/aarch64-linux-gnu -Wl,-rpath-link,/rootfs-arm64/lib/aarch64-linux-gnu" ; \ + export GOARCH=arm64 ; \ else \ - ln -s /usr/local/deps/x86_64 /usr/local/deps/target && \ - export CC=x86_64-linux-gnu-gcc && \ - export GOARCH=amd64 && \ - export CGO_LDFLAGS="-Wl,-rpath-link,/rootfs-amd64/usr/lib/x86_64-linux-gnu -Wl,-rpath-link,/rootfs-amd64/lib/x86_64-linux-gnu" ; \ + export GOARCH=amd64 ; \ fi && \ - export PKG_CONFIG_PATH=/usr/local/deps/target/lib/pkgconfig && \ - export CGO_ENABLED=1 &&\ + export CGO_ENABLED=0 && \ DEVLAKE_PLUGINS="$GO_PLUGINS" make all -# remove symlink in lib, we will recreate in final image -RUN cd /usr/local/deps/target/lib && \ - for file in *.so* ; do \ - if [ -L $file ] ; then \ - unlink $file ; \ - fi \ - done - FROM python:3.11-slim-bookworm as base # apt-get upgrade picks up Debian security fixes that the base tag does not contain yet RUN apt-get update && \ apt-get upgrade -y && \ - apt-get install -y python3-dev python3-pip tar pkg-config curl libssh2-1 zlib1g libffi-dev default-libmysqlclient-dev libpq-dev tini git openssh-client corkscrew && \ + apt-get install -y python3-dev python3-pip tar pkg-config curl zlib1g libffi-dev default-libmysqlclient-dev libpq-dev tini git openssh-client corkscrew && \ apt-get clean && \ rm -fr /usr/share/doc/* \ /usr/share/info/* \ @@ -166,12 +107,6 @@ RUN /app/python/build.sh FROM base as devlake-base ARG DEBUG= -# libraries -ENV LD_LIBRARY_PATH=/app/libs -RUN mkdir -p /app/libs -COPY --from=build /usr/local/deps/target/lib/*.so* /app/libs -RUN ldconfig -vn /app/libs - # apps # The single binary already contains every plugin; the .so files that # `make build-plugin` produces stay in the builder stage and are never copied. diff --git a/backend/Dockerfile.local b/backend/Dockerfile.local index e2046db3bd2..bd9a16cf27a 100644 --- a/backend/Dockerfile.local +++ b/backend/Dockerfile.local @@ -31,21 +31,7 @@ ARG HTTPS_PROXY= RUN apt-get update && apt-get install -y \ gcc \ - binutils \ - libfindbin-libs-perl \ - cmake \ - libssh2-1-dev \ - libssl-dev \ - zlib1g-dev - -# Install libgit2 for native platform only -RUN mkdir -p /tmp/build && cd /tmp/build && \ - wget https://github.com/libgit2/libgit2/archive/refs/tags/v1.5.0.tar.gz -O - | tar -xz && \ - cd libgit2-1.5.0 && \ - mkdir build && cd build && \ - cmake .. -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local && \ - make -j$(nproc) install && \ - ldconfig + ca-certificates RUN go install github.com/vektra/mockery/v3@v3.8.1 RUN go install github.com/swaggo/swag/cmd/swag@v1.16.6 @@ -82,21 +68,14 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ FROM debian:bookworm-slim RUN apt-get update && apt-get install -y \ - libssh2-1 \ - libssl3 \ ca-certificates \ - # gitextractor shells out to the git CLI unless - # USE_GO_GIT_IN_GIT_EXTRACTOR is set, so the binary must be present or every - # clone fails with "git: executable file not found in $PATH". + # gitextractor shells out to the git CLI, so the binary must be present or + # every clone fails with "git: executable file not found in $PATH". # debian:bookworm-slim does not ship it; the official image inherits it from # its python base. git \ && rm -rf /var/lib/apt/lists/* -# Copy libgit2 -COPY --from=builder /usr/local/lib/libgit2.so* /usr/local/lib/ -RUN ldconfig - WORKDIR /app COPY --from=builder /app/bin/lake ./bin/lake diff --git a/backend/go.mod b/backend/go.mod index f1f382839f3..fa0b0cbc5da 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -13,7 +13,6 @@ require ( github.com/coreos/go-oidc/v3 v3.21.0 // github.com/gin-contrib/cors v1.6.0 github.com/gin-gonic/gin v1.12.0 - github.com/go-git/go-git/v5 v5.19.3 github.com/go-playground/validator/v10 v10.30.5 github.com/go-sql-driver/mysql v1.10.1 github.com/go-viper/mapstructure/v2 v2.5.0 @@ -21,7 +20,6 @@ require ( github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.11.0 - github.com/libgit2/git2go/v34 v34.0.0 github.com/magiconair/properties v1.18.12 github.com/merico-ai/graphql v0.0.0-20260206020408-b7fd267bcfac github.com/panjf2000/ants/v2 v2.12.1 @@ -48,11 +46,8 @@ require ( ) require ( - dario.cat/mergo v1.0.0 // indirect filippo.io/edwards25519 v1.2.0 // indirect github.com/KyleBanks/depth v1.2.1 // indirect - github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/ProtonMail/go-crypto v1.1.6 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.24 // indirect @@ -72,19 +67,14 @@ require ( github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect - github.com/cloudflare/circl v1.6.3 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect - github.com/cyphar/filepath-securejoin v0.6.1 // indirect - github.com/emirpasic/gods v1.18.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.15 // indirect github.com/getsentry/sentry-go v0.46.0 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-errors/errors v1.4.2 // indirect - github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.9.2 // indirect github.com/go-jose/go-jose/v4 v4.1.6 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect @@ -95,17 +85,14 @@ require ( github.com/goccy/go-json v0.10.5 // indirect github.com/goccy/go-yaml v1.19.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect @@ -115,13 +102,10 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect - github.com/pjbgf/sha1cd v0.7.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.59.1 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect - github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect - github.com/skeema/knownhosts v1.3.1 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/pflag v1.0.10 // indirect @@ -131,7 +115,6 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.1 // indirect - github.com/xanzy/ssh-agent v0.3.3 // indirect go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/arch v0.22.0 // indirect @@ -141,6 +124,5 @@ require ( golang.org/x/text v0.42.0 // indirect golang.org/x/tools v0.51.0 // indirect google.golang.org/protobuf v1.36.12 // indirect - gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/backend/go.sum b/backend/go.sum index c6a7886bf45..7e7964921b4 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -1,22 +1,11 @@ -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= -github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-sdk-go-v2 v1.47.4 h1:bTvk6UdSSSvaQsfLVj1JRDEEqj/PDRwGf1KanaRzau0= github.com/aws/aws-sdk-go-v2 v1.47.4/go.mod h1:XkrjAZvr705Nytv6PkPUhoSB2W9+vVHBp5oAWpkh8Ak= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.24 h1:t7FxqJ5KHDeKceyD4HtGaG6z7hMSeVGjs5/4c+L5ArM= @@ -63,8 +52,6 @@ github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiD github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/chainguard-dev/git-urls v1.0.2 h1:pSpT7ifrpc5X55n4aTTm7FFUE+ZQHKiqpiwNkJrVcKQ= github.com/chainguard-dev/git-urls v1.0.2/go.mod h1:rbGgj10OS7UgZlbzdUQIQpT0k/D4+An04HJY7Ol+Y/o= -github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= -github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/cockroachdb/errors v1.14.0 h1:EfdVEJpN3z8rPMo43Yit59LxoiIa470fSXpZXuEs+ZI= @@ -77,15 +64,9 @@ github.com/coreos/go-oidc/v3 v3.21.0 h1:wZo4Q9Pum8dYEj0eMUPrqR+kvuGkeUplbLpNCkBq github.com/coreos/go-oidc/v3 v3.21.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= -github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= -github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= -github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= -github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= @@ -100,18 +81,8 @@ github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= -github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= -github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= -github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.9.2 h1:OXFSRyz4g20upsGDJgQG9Bak1l/ZEv8GHVYB52O71sE= -github.com/go-git/go-billy/v5 v5.9.2/go.mod h1:ExsU+jcGwXTBOnyilvAnEM1wug1IxHr4yP2ZXsNRtV0= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.19.3 h1:qHttbZ+Am7wEjdTpR1XMQ4rl42FK9SIXzZ4o9x7s6M4= -github.com/go-git/go-git/v5 v5.19.3/go.mod h1:Ye4C8dVigkqrv9UsB4NMdSLV4yAIkLiIhW61gcOyhl4= github.com/go-jose/go-jose/v4 v4.1.6 h1:sNRkftxFJxwsILiQukqp5z83QBSmntVwmgRY7AqBDO4= github.com/go-jose/go-jose/v4 v4.1.6/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -150,13 +121,9 @@ github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0kt github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= -github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= -github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -169,8 +136,6 @@ github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= @@ -179,8 +144,6 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= -github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= @@ -194,8 +157,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.5.0 h1:pLqT2kq1zpHW/1D18QMjMpdtX7cekxqtJJjg5ANyWw0= github.com/leodido/go-urn v1.5.0/go.mod h1:9BORnCDhdPBJNDEX+w1bJisa8yOKYi116VeO96s4ifE= -github.com/libgit2/git2go/v34 v34.0.0 h1:UKoUaKLmiCRbOCD3PtUi2hD6hESSXzME/9OUZrGcgu8= -github.com/libgit2/git2go/v34 v34.0.0/go.mod h1:blVco2jDAw6YTXkErMMqzHLcAjKkwF0aWIRHBqiJkZ0= github.com/magiconair/properties v1.18.12 h1:sT9zQpvTB3B4gzrX0tmZNTEaGyg8Zw55MFYRE32Mr9I= github.com/magiconair/properties v1.18.12/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -217,16 +178,12 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= -github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/panjf2000/ants/v2 v2.12.1 h1:BWvU2wHpyXWxhhNXsGB6JXLCNbshyLd1QxvoAmZnu10= github.com/panjf2000/ants/v2 v2.12.1/go.mod h1:tSQuaNQ6r6NRhPt+IZVUevvDyFMTs+eS4ztZc52uJTY= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pjbgf/sha1cd v0.7.0 h1:ZRNPKHj+gfkLBf0KJv/p1Hmz+7bqCT5o311YX0nq+DA= -github.com/pjbgf/sha1cd v0.7.0/go.mod h1:pKR5Li+qTCo+ebqITZfwPlJGoCFCvSO00qxjbNtTUFQ= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -243,13 +200,8 @@ github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52Z github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= -github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= -github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo= github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q= -github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= -github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= @@ -269,9 +221,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -300,8 +250,6 @@ github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/viant/afs v1.30.1 h1:l+cIg7pVo5Cdia2GqaK+h3hwvj26QHg6VHmuatZ0CTM= github.com/viant/afs v1.30.1/go.mod h1:rScbFd9LJPGTM8HOI8Kjwee0AZ+MZMupAvFpPg+Qdj4= -github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= -github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -319,9 +267,7 @@ golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= @@ -337,7 +283,6 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= @@ -352,22 +297,16 @@ golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -395,18 +334,14 @@ google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fw google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/datatypes v1.2.7 h1:ww9GAhF1aGXZY3EB3cJPJ7//JiuQo7DlQA7NNlVaTdk= gorm.io/datatypes v1.2.7/go.mod h1:M2iO+6S3hhi4nAyYe444Pcb0dcIiOMJ7QHaUXxyiNZY= diff --git a/backend/plugins/gitextractor/README.md b/backend/plugins/gitextractor/README.md index 9d70677828a..e1e0266971b 100644 --- a/backend/plugins/gitextractor/README.md +++ b/backend/plugins/gitextractor/README.md @@ -16,8 +16,9 @@ limitations under the License. --> Please see details in the [Apache DevLake website](https://devlake.apache.org/docs/Plugins/gitextractor) +The plugin reads repositories with the `git` command line, so `git` (2.31 or +newer) must be installed where the server runs. No libgit2 or cgo is required. -Build with local compiled v1.5.0 libgit2 ``` -CGO_LDFLAGS="-L/usr/local/lib -lgit2" CGO_CFLAGS="-I/usr/local/include" go build ./plugins/gitextractor/... 2>&1 | tail -5 +go build ./plugins/gitextractor/... ``` \ No newline at end of file diff --git a/backend/plugins/gitextractor/gitextractor.go b/backend/plugins/gitextractor/gitextractor.go index 1261c3834fe..da8f9b9ca0c 100644 --- a/backend/plugins/gitextractor/gitextractor.go +++ b/backend/plugins/gitextractor/gitextractor.go @@ -35,8 +35,6 @@ func main() { // pk := cmd.Flags().StringP("privateKey", "k", "", "private key file") // pkPass := cmd.Flags().StringP("privateKeyPassPhrase", "P", "", "passphrase for private key") proxy := cmd.Flags().StringP("proxy", "x", "", "proxy") - useGoGit := cmd.Flags().BoolP("useGoGit", "g", false, "use go-git instead of libgit2") - useGitCli := cmd.Flags().BoolP("useGitCli", "", false, "use the git command line collector") skipCommitStat := cmd.Flags().BoolP("skipCommitStat", "S", false, "") skipCommitFiles := cmd.Flags().BoolP("skipCommitFiles", "F", true, "") noShallowClone := cmd.Flags().BoolP("noShallowClone", "A", false, "") @@ -53,8 +51,6 @@ func main() { // "privateKey": * // "passphrase" "proxy": *proxy, - "useGoGit": *useGoGit, - "useGitCli": *useGitCli, "skipCommitStat": skipCommitStat, "skipCommitFiles": skipCommitFiles, "noShallowClone": noShallowClone, diff --git a/backend/plugins/gitextractor/impl/impl.go b/backend/plugins/gitextractor/impl/impl.go index 746afee85b9..6b0aa1bc5e9 100644 --- a/backend/plugins/gitextractor/impl/impl.go +++ b/backend/plugins/gitextractor/impl/impl.go @@ -20,7 +20,6 @@ package impl import ( "fmt" "net/url" - "strings" "github.com/apache/devlake/core/dal" "github.com/apache/devlake/core/errors" @@ -116,12 +115,8 @@ func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options map[st } *optValue = &defValue } - loadBool(&op.UseGoGit, "UseGoGit", false) loadBool(&op.SkipCommitStat, "SKIP_COMMIT_STAT", false) loadBool(&op.SkipCommitFiles, "SKIP_COMMIT_FILES", true) - op.UseGitCli = op.UseGitCli || strings.EqualFold(cfg.GetString("GIT_EXTRACTOR_COLLECTOR"), "gitcli") - log.Info("UseGoGit: %v", *op.UseGoGit) - log.Info("UseGitCli: %v", op.UseGitCli) log.Info("SkipCommitStat: %v", *op.SkipCommitStat) log.Info("SkipCommitFiles: %v", *op.SkipCommitFiles) diff --git a/backend/plugins/gitextractor/models/diffline.go b/backend/plugins/gitextractor/models/diffline.go deleted file mode 100644 index 5375df29417..00000000000 --- a/backend/plugins/gitextractor/models/diffline.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package models - -import git "github.com/libgit2/git2go/v34" - -type DiffLines []git.DiffLine - -// Len return length of diffLines -func (diffLines DiffLines) Len() int { - return len(diffLines) -} - -// Less function -func (diffLines DiffLines) Less(i, j int) bool { - return diffLines[i].OldLineno > diffLines[j].OldLineno -} - -// Swap function -func (diffLines DiffLines) Swap(i, j int) { - temp := diffLines[i] - diffLines[i] = diffLines[j] - diffLines[j] = temp -} diff --git a/backend/plugins/gitextractor/parser/clone_gitcli.go b/backend/plugins/gitextractor/parser/clone_gitcli.go index 3c88899f7d1..06f1a7acb00 100644 --- a/backend/plugins/gitextractor/parser/clone_gitcli.go +++ b/backend/plugins/gitextractor/parser/clone_gitcli.go @@ -46,8 +46,6 @@ const maxExtraDeepenRounds = 10 // CloneRepoConfig is the configuration for the CloneRepo method // the subtask should run in Full Sync mode whenever the configuration is changed type CloneRepoConfig struct { - UseGoGit *bool - UseGitCli bool SkipCommitStat *bool SkipCommitFiles *bool NoShallowClone bool @@ -73,8 +71,6 @@ func NewGitcliCloner(ctx plugin.SubTaskContext, localDir string) (*GitcliCloner, SubTaskContext: ctx, Params: taskData.Options.GitExtractorApiParams, SubtaskConfig: CloneRepoConfig{ - UseGoGit: taskData.Options.UseGoGit, - UseGitCli: taskData.Options.UseGitCli, SkipCommitStat: taskData.Options.SkipCommitStat, SkipCommitFiles: taskData.Options.SkipCommitFiles, NoShallowClone: taskData.Options.NoShallowClone, @@ -97,7 +93,7 @@ func NewGitcliCloner(ctx plugin.SubTaskContext, localDir string) (*GitcliCloner, func (g *GitcliCloner) prepareSync() errors.Error { taskData := g.taskData // Disable automatic (detached) gc/maintenance after clone/fetch. Otherwise git may - // repack the repo in the background while libgit2 is already reading it, which + // repack the repo in the background while the collector is reading it, which // makes temporary pack files (.tmp--pack-*.idx) vanish and the collection fail. g.globalConfigArgs = append(g.globalConfigArgs, "-c", "gc.auto=0", @@ -203,7 +199,7 @@ func (g *GitcliCloner) CloneRepo() errors.Error { if g.taskData.Options.NoShallowClone { // data source does not support shallow clone // 1. perform a full clone to accommodate - // 2. perform a local shallow clone to reduce the libgit2 memory usage + // 2. perform a local shallow clone so the collector reads a smaller repo if err := g.doubleClone(); err != nil { return err } diff --git a/backend/plugins/gitextractor/parser/clone_gitcli_test.go b/backend/plugins/gitextractor/parser/clone_gitcli_test.go index 790980f64f6..83636b97e90 100644 --- a/backend/plugins/gitextractor/parser/clone_gitcli_test.go +++ b/backend/plugins/gitextractor/parser/clone_gitcli_test.go @@ -127,7 +127,7 @@ func TestIncrementalCloneKeepsCommitsBehindMergeBoundary(t *testing.T) { } // git fetch may start `git maintenance run --auto` / `git gc --auto` in the -// background. That repacks the repo while libgit2 is already reading it, so +// background. That repacks the repo while the collector is reading it, so // temporary pack files (.tmp--pack-*.idx) vanish under its feet and the // collection fails with "could not find ... to open". func TestCloneDoesNotStartAutoMaintenance(t *testing.T) { diff --git a/backend/plugins/gitextractor/parser/collector_contract_test.go b/backend/plugins/gitextractor/parser/collector_contract_test.go index 52b97ece95e..6e2bf8d07ce 100644 --- a/backend/plugins/gitextractor/parser/collector_contract_test.go +++ b/backend/plugins/gitextractor/parser/collector_contract_test.go @@ -19,6 +19,8 @@ package parser import ( "context" + "os" + "path/filepath" "sort" "testing" @@ -33,93 +35,20 @@ import ( "github.com/stretchr/testify/require" ) -// collectorCase describes one RepoCollector implementation under test. -// -// The assertions in this file describe the collector contract that a -// replacement implementation must satisfy. The deviations block records where -// an existing implementation is known to violate that contract; they are -// documented instead of being silently tolerated so that a replacement cannot -// inherit them. -type collectorCase struct { - name string - open func(t *testing.T, localDir, repoId string, store models.Store) RepoCollector - - deviations collectorDeviations -} - -type collectorDeviations struct { - // storesOnlyLastComponent reports that only the component of the last file - // of each commit is stored. - storesOnlyLastComponent bool - // countTagsIgnoresContext reports that CountTags ignores a cancelled context. - countTagsIgnoresContext bool - // collectsShallowBoundary reports that the shallow boundary commit is - // collected with a full-tree diff instead of being skipped. - collectsShallowBoundary bool - // failsOnDeletedFiles reports that CollectDiffLine fails when the history - // contains a deleted file. - failsOnDeletedFiles bool - // skipsLineChanges reports that CollectDiffLine does not store - // CommitLineChange rows. - skipsLineChanges bool - // failsOnEmptyRepo reports that the Count* helpers fail on a repository - // without commits. - failsOnEmptyRepo bool - // omitsZeroLineFiles reports that changed files without line changes - // (binary files, empty files) are not stored. - omitsZeroLineFiles bool -} - -func collectorCases() []collectorCase { - return []collectorCase{ - { - name: "gitcli", - open: func(t *testing.T, localDir, repoId string, store models.Store) RepoCollector { - t.Helper() - collector, err := NewGitcliRepoCollector(localDir, repoId, store, unithelper.DummyLogger()) - require.Nil(t, err) - return collector - }, - }, - { - name: "libgit2", - open: func(t *testing.T, localDir, repoId string, store models.Store) RepoCollector { - t.Helper() - collector, err := NewLibgit2RepoCollector(localDir, repoId, store, unithelper.DummyLogger()) - require.Nil(t, err) - return collector - }, - deviations: collectorDeviations{ - storesOnlyLastComponent: true, - countTagsIgnoresContext: true, - collectsShallowBoundary: true, - }, - }, - { - name: "gogit", - open: func(t *testing.T, localDir, repoId string, store models.Store) RepoCollector { - t.Helper() - collector, err := NewGogitRepoCollector(localDir, repoId, store, unithelper.DummyLogger()) - require.Nil(t, err) - return collector - }, - deviations: collectorDeviations{ - failsOnDeletedFiles: true, - skipsLineChanges: true, - failsOnEmptyRepo: true, - omitsZeroLineFiles: true, - }, - }, - } +// These tests describe the collector contract: what the git command line +// collector writes for a known repository. The fixtures are built with the git +// CLI itself, so the tests do not depend on any git library. + +func newTestCollector(t *testing.T, localDir, repoId string, store models.Store) RepoCollector { + t.Helper() + collector, err := NewGitcliRepoCollector(localDir, repoId, store, unithelper.DummyLogger()) + require.Nil(t, err) + return collector } -func forEachCollector(t *testing.T, fn func(t *testing.T, c collectorCase)) { +func forEachCollector(t *testing.T, fn func(t *testing.T)) { t.Helper() - for _, c := range collectorCases() { - t.Run(c.name, func(t *testing.T) { - fn(t, c) - }) - } + t.Run("gitcli", fn) } // recordStore is a models.Store that records everything the collectors write. @@ -275,9 +204,9 @@ func parentShas(parents []*code.CommitParent, commitSha string) []string { func TestCollectorCountsAndRefs(t *testing.T) { repo := buildCanonicalRepo(t) - forEachCollector(t, func(t *testing.T, c collectorCase) { + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, repo.Dir, "test-repo", store) + collector := newTestCollector(t, repo.Dir, "test-repo", store) ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) commits, err := collector.CountCommits(context.Background()) @@ -329,9 +258,9 @@ func TestCollectorCommits(t *testing.T) { repo := buildCanonicalRepo(t) repoId := "test-repo" - forEachCollector(t, func(t *testing.T, c collectorCase) { + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, repo.Dir, repoId, store) + collector := newTestCollector(t, repo.Dir, repoId, store) ctx := newCollectorContext(t, defaultOptions(repoId), nil) require.NoError(t, collector.CollectCommits(ctx)) @@ -392,20 +321,7 @@ func TestCollectorCommits(t *testing.T) { repo.C5: {"feature.txt": {1, 0}}, } for sha, want := range expectedFiles { - if !c.deviations.omitsZeroLineFiles { - assert.Equal(t, want, files[sha], sha) - continue - } - for path, stat := range want { - if stat != [2]int{0, 0} { - assert.Equal(t, stat, files[sha][path], "%s %s", sha, path) - } - } - for path, stat := range files[sha] { - if stat != [2]int{0, 0} { - assert.Contains(t, want, path, "%s %s", sha, path) - } - } + assert.Equal(t, want, files[sha], sha) } assert.Equal(t, 4, commits[repo.C1].Additions) @@ -421,11 +337,9 @@ func TestCollectorCommits(t *testing.T) { for _, file := range store.commitFiles { assert.Equal(t, genCommitFileId(file.CommitSha, file.FilePath), file.Id) } - if !c.deviations.storesOnlyLastComponent { - components := componentsById(store.components) - for _, file := range store.commitFiles { - assert.Equal(t, "Default", components[file.Id], file.FilePath) - } + components := componentsById(store.components) + for _, file := range store.commitFiles { + assert.Equal(t, "Default", components[file.Id], file.FilePath) } }) } @@ -433,9 +347,9 @@ func TestCollectorCommits(t *testing.T) { func TestCollectorUsesConfiguredComponents(t *testing.T) { repo := buildLinearRepo(t) - forEachCollector(t, func(t *testing.T, c collectorCase) { + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, repo.Dir, "test-repo", store) + collector := newTestCollector(t, repo.Dir, "test-repo", store) ctx := newCollectorContext(t, defaultOptions("test-repo"), []code.Component{ {RepoId: "test-repo", Name: "docs", PathRegex: `^c\.txt$`}, }) @@ -446,18 +360,16 @@ func TestCollectorUsesConfiguredComponents(t *testing.T) { files := filesByCommit(store.commitFiles)[repo.C4] require.Contains(t, files, "c.txt") assert.Equal(t, "docs", components[genCommitFileId(repo.C4, "c.txt")]) - if !c.deviations.storesOnlyLastComponent { - assert.Equal(t, "Default", components[genCommitFileId(repo.C4, "b.txt")]) - } + assert.Equal(t, "Default", components[genCommitFileId(repo.C4, "b.txt")]) }) } func TestCollectorSkipCommitStat(t *testing.T) { repo := buildCanonicalRepo(t) - forEachCollector(t, func(t *testing.T, c collectorCase) { + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, repo.Dir, "test-repo", store) + collector := newTestCollector(t, repo.Dir, "test-repo", store) opts := defaultOptions("test-repo") opts.SkipCommitStat = boolPtr(true) opts.SkipCommitFiles = boolPtr(true) @@ -478,9 +390,9 @@ func TestCollectorSkipCommitStat(t *testing.T) { func TestCollectorSkipCommitFiles(t *testing.T) { repo := buildCanonicalRepo(t) - forEachCollector(t, func(t *testing.T, c collectorCase) { + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, repo.Dir, "test-repo", store) + collector := newTestCollector(t, repo.Dir, "test-repo", store) opts := defaultOptions("test-repo") opts.SkipCommitFiles = boolPtr(true) ctx := newCollectorContext(t, opts, nil) @@ -498,9 +410,9 @@ func TestCollectorSkipCommitFiles(t *testing.T) { func TestCollectorExcludeFileExtensions(t *testing.T) { repo := buildCanonicalRepo(t) - forEachCollector(t, func(t *testing.T, c collectorCase) { + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, repo.Dir, "test-repo", store) + collector := newTestCollector(t, repo.Dir, "test-repo", store) opts := defaultOptions("test-repo") opts.ExcludeFileExtensions = []string{".DAT"} ctx := newCollectorContext(t, opts, nil) @@ -529,12 +441,9 @@ func TestCollectorSkipsShallowBoundaryCommit(t *testing.T) { dir := f.shallowBareClone() - forEachCollector(t, func(t *testing.T, c collectorCase) { - if c.deviations.collectsShallowBoundary { - t.Skip("known deviation: the shallow boundary is collected with a full-tree diff") - } + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, dir, "test-repo", store) + collector := newTestCollector(t, dir, "test-repo", store) ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) // The tip is a shallow boundary: its parent is not in the object @@ -546,15 +455,42 @@ func TestCollectorSkipsShallowBoundaryCommit(t *testing.T) { }) } +func TestCollectorSkipsShallowBoundaryEvenWhenItsParentExists(t *testing.T) { + f := newGitFixture(t) + f.writeFile("a.txt", "l1\n") + c1 := f.commit(testTimeC1, "c1") + f.writeFile("a.txt", "l1\nl2\n") + c2 := f.commit(testTimeC2, "c2") + // keep the parent reachable through another branch + f.git("branch", "old", c1) + + dir := f.bareClone() + // Mark the tip as a shallow boundary although its parent object is still + // in the object database. git then renders the tip as a root commit, so + // the collector must skip it based on the shallow file, not on the parent + // being missing. + require.NoError(t, os.WriteFile(filepath.Join(dir, "shallow"), []byte(c2+"\n"), 0o644)) + + forEachCollector(t, func(t *testing.T) { + store := &recordStore{} + collector := newTestCollector(t, dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + require.NoError(t, collector.CollectCommits(ctx)) + + commits := commitsBySha(store.commits) + assert.NotContains(t, commits, c2, "the shallow boundary must not be collected") + require.Contains(t, commits, c1) + assert.Equal(t, 1, commits[c1].Additions) + }) +} + func TestCollectorEmptyRepo(t *testing.T) { dir := newGitFixture(t).bareClone() - forEachCollector(t, func(t *testing.T, c collectorCase) { - if c.deviations.failsOnEmptyRepo { - t.Skip("known deviation: the Count* helpers need at least one commit") - } + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, dir, "test-repo", store) + collector := newTestCollector(t, dir, "test-repo", store) ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) commits, err := collector.CountCommits(context.Background()) @@ -578,39 +514,24 @@ func TestCollectorEmptyRepo(t *testing.T) { func TestCollectorHonoursCancelledContext(t *testing.T) { repo := buildCanonicalRepo(t) - forEachCollector(t, func(t *testing.T, c collectorCase) { + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, repo.Dir, "test-repo", store) + collector := newTestCollector(t, repo.Dir, "test-repo", store) ctx, cancel := context.WithCancel(context.Background()) cancel() _, err := collector.CountCommits(ctx) assert.ErrorIs(t, err, context.Canceled) - if !c.deviations.countTagsIgnoresContext { - _, err = collector.CountTags(ctx) - assert.ErrorIs(t, err, context.Canceled) - } + _, err = collector.CountTags(ctx) + assert.ErrorIs(t, err, context.Canceled) _, err = collector.CountBranches(ctx) assert.ErrorIs(t, err, context.Canceled) }) } func TestCollectorRejectsNonRepo(t *testing.T) { - dir := t.TempDir() - for _, c := range collectorCases() { - t.Run(c.name, func(t *testing.T) { - var err error - switch c.name { - case "gitcli": - _, err = NewGitcliRepoCollector(dir, "test-repo", nil, unithelper.DummyLogger()) - case "libgit2": - _, err = NewLibgit2RepoCollector(dir, "test-repo", nil, unithelper.DummyLogger()) - case "gogit": - _, err = NewGogitRepoCollector(dir, "test-repo", nil, unithelper.DummyLogger()) - } - assert.Error(t, err, "%s should reject a non-repository", c.name) - }) - } + _, err := NewGitcliRepoCollector(t.TempDir(), "test-repo", nil, unithelper.DummyLogger()) + assert.Error(t, err, "the collector should reject a non-repository") } func TestCollectorCountsWhenHeadIsNotDefault(t *testing.T) { @@ -623,9 +544,9 @@ func TestCollectorCountsWhenHeadIsNotDefault(t *testing.T) { require.NotEqual(t, c1, c2) dir := f.bareClone() - forEachCollector(t, func(t *testing.T, c collectorCase) { + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, dir, "test-repo", store) + collector := newTestCollector(t, dir, "test-repo", store) // HEAD points at feature, which is not counted but is the default ref. branches, err := collector.CountBranches(context.Background()) @@ -648,12 +569,9 @@ func TestCollectorDiffLineSnapshot(t *testing.T) { repo := buildLinearRepo(t) expected := blameAtHEAD(t, repo.Dir) - forEachCollector(t, func(t *testing.T, c collectorCase) { - if c.deviations.failsOnDeletedFiles { - t.Skip("known deviation: CollectDiffLine fails on a deleted file") - } + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, repo.Dir, "test-repo", store) + collector := newTestCollector(t, repo.Dir, "test-repo", store) ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) require.NoError(t, collector.CollectDiffLine(ctx)) @@ -672,21 +590,16 @@ func TestCollectorDiffLineSnapshot(t *testing.T) { assert.Equal(t, lines, got[file], file) } - if c.deviations.skipsLineChanges { - assert.Empty(t, store.lineChanges) - } + assert.NotEmpty(t, store.lineChanges) }) } func TestCollectorCollectAllAndClose(t *testing.T) { repo := buildCanonicalRepo(t) - forEachCollector(t, func(t *testing.T, c collectorCase) { - if c.deviations.failsOnDeletedFiles { - t.Skip("known deviation: CollectDiffLine fails on a deleted file") - } + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, repo.Dir, "test-repo", store) + collector := newTestCollector(t, repo.Dir, "test-repo", store) ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) cleanedUp := false @@ -698,11 +611,7 @@ func TestCollectorCollectAllAndClose(t *testing.T) { assert.Len(t, store.refs, 4) assert.NotEmpty(t, store.commitFiles) assert.NotEmpty(t, store.snapshots) - if c.deviations.skipsLineChanges { - assert.Empty(t, store.lineChanges) - } else { - assert.NotEmpty(t, store.lineChanges) - } + assert.NotEmpty(t, store.lineChanges) require.NoError(t, collector.Close(context.Background())) assert.True(t, store.closed) @@ -710,12 +619,11 @@ func TestCollectorCollectAllAndClose(t *testing.T) { }) } -func TestLibgit2CollectorCommitLineChanges(t *testing.T) { +func TestCollectorCommitLineChanges(t *testing.T) { repo := buildLinearRepo(t) store := &recordStore{} - collector, err := NewLibgit2RepoCollector(repo.Dir, "test-repo", store, unithelper.DummyLogger()) - require.Nil(t, err) + collector := newTestCollector(t, repo.Dir, "test-repo", store) ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) require.NoError(t, collector.CollectDiffLine(ctx)) diff --git a/backend/plugins/gitextractor/parser/commitparse_test.go b/backend/plugins/gitextractor/parser/commitparse_test.go index 473fc070c0f..fca02f31fb8 100644 --- a/backend/plugins/gitextractor/parser/commitparse_test.go +++ b/backend/plugins/gitextractor/parser/commitparse_test.go @@ -34,7 +34,7 @@ parent 2222222222222222222222222222222222222222 author Jane Doe 1704067200 +0800 committer DevLake Test 1704070800 -0530 gpgsig -----BEGIN PGP SIGNATURE----- - + fakesignature-line fakesignature -----END PGP SIGNATURE----- diff --git a/backend/plugins/gitextractor/parser/numstatparse.go b/backend/plugins/gitextractor/parser/numstatparse.go index 1d96dd6ae89..f7d3fc21ec3 100644 --- a/backend/plugins/gitextractor/parser/numstatparse.go +++ b/backend/plugins/gitextractor/parser/numstatparse.go @@ -117,7 +117,7 @@ func parseNumstatEntry(entry string) (commitFileStat, error) { } // parseNumstatCount parses a numstat count. Binary files are reported as "-" -// and count as zero changed lines, just like libgit2 reports them. +// and count as zero changed lines. func parseNumstatCount(value string) (int, error) { if value == "-" { return 0, nil diff --git a/backend/plugins/gitextractor/parser/parse_test.go b/backend/plugins/gitextractor/parser/parse_test.go index 23783c3296d..988a4b87d97 100644 --- a/backend/plugins/gitextractor/parser/parse_test.go +++ b/backend/plugins/gitextractor/parser/parse_test.go @@ -157,7 +157,7 @@ func TestParsePatchStream(t *testing.T) { // binary file: both paths are known, but there are no line changes assert.Equal(t, patchFile{OldPath: "bin.dat", NewPath: "bin.dat"}, c2.Files[1]) - // deleted file: libgit2 reports the path on both sides + // deleted file: the path is reported on both sides assert.Equal(t, patchFile{ OldPath: "empty.txt", NewPath: "empty.txt", @@ -211,15 +211,19 @@ func TestParseGitVersion(t *testing.T) { for _, tc := range []struct { output string want int + ok bool }{ - {"git version 2.43.0\n", 2*10000 + 43*100}, - {"git version 2.31.0", 2*10000 + 31*100}, - {"git version 2.39.2.windows.1", 2*10000 + 39*100 + 2}, - {"git version 2.56.0", 2*10000 + 56*100}, + {"git version 2.43.0\n", 2*10000 + 43*100, true}, + {"git version 2.31.0", 2*10000 + 31*100, true}, + {"git version 2.39.2.windows.1", 2*10000 + 39*100 + 2, true}, + {"git version 2.56.0", 2*10000 + 56*100, true}, + {"git version 2.30.2", 2*10000 + 30*100 + 2, false}, + {"git version 1.9.0", 1*10000 + 9*100, false}, } { got, err := parseGitVersion(tc.output) if assert.NoError(t, err, tc.output) { assert.Equal(t, tc.want, got, tc.output) + assert.Equal(t, tc.ok, got >= minGitVersion, tc.output) } } @@ -241,3 +245,9 @@ func TestUnquoteGitPath(t *testing.T) { assert.Equal(t, tc.want, unquoteGitPath(tc.in), tc.in) } } + +func TestResolveRepoPath(t *testing.T) { + assert.Equal(t, "/repo/.git/shallow", resolveRepoPath("/repo", ".git/shallow")) + assert.Equal(t, "/repo/shallow", resolveRepoPath("/repo", "shallow")) + assert.Equal(t, "/abs/shallow", resolveRepoPath("/repo", "/abs/shallow")) +} diff --git a/backend/plugins/gitextractor/parser/patchparse.go b/backend/plugins/gitextractor/parser/patchparse.go index e4d3c8dc8a9..3aa27a70f3b 100644 --- a/backend/plugins/gitextractor/parser/patchparse.go +++ b/backend/plugins/gitextractor/parser/patchparse.go @@ -203,8 +203,8 @@ func parseHunkRange(value string) (int, error) { func (p *patchParser) endFile() { if p.current != nil { - // libgit2 reports the same path on both sides for additions and - // deletions, so mirror that here. + // Report the same path on both sides for additions and deletions, so the + // commit file and line change rows always name the file. if p.current.OldPath == "" { p.current.OldPath = p.current.NewPath } diff --git a/backend/plugins/gitextractor/parser/repo_gitcli.go b/backend/plugins/gitextractor/parser/repo_gitcli.go index 4cf1fc0d1e3..f13256daae6 100644 --- a/backend/plugins/gitextractor/parser/repo_gitcli.go +++ b/backend/plugins/gitextractor/parser/repo_gitcli.go @@ -20,9 +20,12 @@ package parser import ( "bufio" "context" + "crypto/sha256" + "encoding/hex" "fmt" "io" "os" + "path/filepath" "regexp" "sort" "strconv" @@ -39,7 +42,7 @@ import ( // minGitVersion is the first release with --diff-merges=first-parent, which is // required to diff merge commits against their first parent only. -const minGitVersion = 2*10000 + 31 +const minGitVersion = 2*10000 + 31*100 // commitBatchSize bounds how many commits are fetched and diffed at a time so // that a repository with a long history does not need to be held in memory. @@ -47,8 +50,9 @@ const commitBatchSize = 1024 var _ RepoCollector = (*GitcliRepoCollector)(nil) -// GitcliRepoCollector collects git data with the git command line, so the -// server no longer needs libgit2 or a pure Go git implementation. +// GitcliRepoCollector collects git data with the git command line. It replaces +// the previous libgit2 and go-git implementations, so the server no longer +// needs a git library or cgo. type GitcliRepoCollector struct { id string logger log.Logger @@ -494,7 +498,7 @@ type snapshotChanges struct { added []patchLine } -// applySnapshotChanges mirrors the line bookkeeping of the libgit2 collector: +// applySnapshotChanges applies the changes of one file to the line snapshot: // deletions remove the old line numbers and additions insert the commit sha at // the new line numbers. func applySnapshotChanges(snapshot map[string]*models.FileBlame, filePath, commitSha string, changes *snapshotChanges) { @@ -588,7 +592,7 @@ func (r *GitcliRepoCollector) shallowCommits(ctx context.Context) (map[string]st if err != nil { return shallow, err } - content, err := os.ReadFile(strings.TrimSpace(string(out))) + content, err := os.ReadFile(resolveRepoPath(r.git.dir, strings.TrimSpace(string(out)))) if os.IsNotExist(err) { return shallow, nil } @@ -603,6 +607,16 @@ func (r *GitcliRepoCollector) shallowCommits(ctx context.Context) (map[string]st return shallow, nil } +// resolveRepoPath turns a git path that is relative to the repository into an +// absolute one. `git rev-parse --git-path` returns paths relative to the +// repository root, but the collector's working directory is not the repository. +func resolveRepoPath(repoDir, gitPath string) string { + if filepath.IsAbs(gitPath) { + return gitPath + } + return filepath.Join(repoDir, gitPath) +} + type tagRef struct { refname string sha string @@ -684,6 +698,15 @@ func sortedComponentNames(componentMap map[string]*regexp.Regexp) []string { return names } +// genCommitFileId builds the commit file primary key. With some long paths the +// varchar(255) columns were not enough for both the id and the file path, so +// the path is compressed with sha256. +func genCommitFileId(commitSha, filePath string) string { + shaFilePath := sha256.New() + shaFilePath.Write([]byte(filePath)) + return commitSha + ":" + hex.EncodeToString(shaFilePath.Sum(nil)) +} + func normalizeExtensions(extensions []string) map[string]struct{} { excluded := make(map[string]struct{}, len(extensions)) for _, extension := range extensions { diff --git a/backend/plugins/gitextractor/parser/repo_gogit.go b/backend/plugins/gitextractor/parser/repo_gogit.go deleted file mode 100644 index fe254e85935..00000000000 --- a/backend/plugins/gitextractor/parser/repo_gogit.go +++ /dev/null @@ -1,626 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package parser - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "regexp" - "strings" - - "github.com/apache/devlake/core/dal" - "github.com/apache/devlake/core/errors" - "github.com/apache/devlake/core/log" - "github.com/apache/devlake/core/models/domainlayer" - "github.com/apache/devlake/core/models/domainlayer/code" - "github.com/apache/devlake/core/plugin" - "github.com/apache/devlake/plugins/gitextractor/models" - gogit "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/go-git/go-git/v5/plumbing/storer" -) - -type GogitRepoCollector struct { - id string - logger log.Logger - store models.Store - repo *gogit.Repository - cleanUp func() -} - -func NewGogitRepoCollector(localDir string, repoId string, store models.Store, logger log.Logger) (*GogitRepoCollector, errors.Error) { - repo, err := gogit.PlainOpen(localDir) - if err != nil { - return nil, errors.Convert(err) - } - return &GogitRepoCollector{ - id: repoId, - logger: logger, - store: store, - repo: repo, - }, nil -} - -func (r *GogitRepoCollector) SetCleanUp(f func()) error { - if f != nil { - r.cleanUp = f - } - return nil -} - -func (r *GogitRepoCollector) Close(ctx context.Context) error { - if err := r.store.Close(); err != nil { - return err - } - if r.cleanUp != nil { - r.cleanUp() - } - return nil -} - -// CollectAll The main parser subtask -func (r *GogitRepoCollector) CollectAll(subtaskCtx plugin.SubTaskContext) error { - subtaskCtx.SetProgress(0, -1) - err := r.CollectTags(subtaskCtx) - if err != nil { - return err - } - err = r.CollectBranches(subtaskCtx) - if err != nil { - return err - } - err = r.CollectCommits(subtaskCtx) - if err != nil { - return err - } - return r.CollectDiffLine(subtaskCtx) -} - -// CountTags Count git tags subtask -func (r *GogitRepoCollector) CountTags(ctx context.Context) (int, error) { - iter, err := r.repo.Tags() - if err != nil { - return 0, err - } - var tagsCount int - if err := iter.ForEach(func(reference *plumbing.Reference) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - tagsCount += 1 - return nil - }); err != nil { - return 0, err - } - return tagsCount, nil -} - -// CountBranches count the number of branches in a git repo -func (r *GogitRepoCollector) CountBranches(ctx context.Context) (int, error) { - refIter, err := r.repo.Storer.IterReferences() - if err != nil { - return 0, err - } - branchIter := storer.NewReferenceFilteredIter( - func(r *plumbing.Reference) bool { - return r.Name().IsBranch() || r.Name().IsRemote() - }, refIter) - var branchesCount int - - headRef, err := r.repo.Head() - if err != nil { - return 0, err - } - if err := branchIter.ForEach(func(reference *plumbing.Reference) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - if reference.Name() != headRef.Name() { - branchesCount += 1 - } - return nil - }); err != nil { - return 0, err - } - return branchesCount, nil -} - -// CountCommits count the number of commits in a git repo -func (r *GogitRepoCollector) CountCommits(ctx context.Context) (int, error) { - iter, err := r.repo.CommitObjects() - if err != nil { - return 0, err - } - var count int - if err := iter.ForEach(func(commit *object.Commit) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - count += 1 - return nil - }); err != nil { - return 0, err - } - return count, nil -} - -// CollectTags Collect Tags data -func (r *GogitRepoCollector) CollectTags(subtaskCtx plugin.SubTaskContext) error { - tagIter, err := r.repo.Tags() - if err != nil { - return err - } - if err := tagIter.ForEach(func(ref *plumbing.Reference) error { - select { - case <-subtaskCtx.GetContext().Done(): - return subtaskCtx.GetContext().Err() - default: - } - tagCommit := ref.Hash().String() - _, err := r.repo.CommitObject(ref.Hash()) - if err != nil && errors.Is(err, plumbing.ErrObjectNotFound) { - h, err := r.repo.ResolveRevision(plumbing.Revision(ref.Name())) - if err != nil { - return err - } - tagCommit = h.String() - } - name := ref.Name().String() - if tagCommit != "" { - codeRef := &code.Ref{ - DomainEntityExtended: domainlayer.DomainEntityExtended{Id: fmt.Sprintf("%s:%s", r.id, name)}, - RepoId: r.id, - Name: name, - CommitSha: tagCommit, - RefType: TAG, - } - err = r.store.Refs(codeRef) - if err != nil { - return err - } - subtaskCtx.IncProgress(1) - } - return nil - }); err != nil { - return err - } - return nil -} - -// CollectBranches Collect branch data -func (r *GogitRepoCollector) CollectBranches(subtaskCtx plugin.SubTaskContext) error { - refIter, err := r.repo.Storer.IterReferences() - if err != nil { - return err - } - branchIter := storer.NewReferenceFilteredIter( - func(r *plumbing.Reference) bool { - return r.Name().IsBranch() || r.Name().IsRemote() - }, refIter) - headRef, err := r.repo.Head() - if err != nil { - return err - } - if err := branchIter.ForEach(func(ref *plumbing.Reference) error { - select { - case <-subtaskCtx.GetContext().Done(): - return subtaskCtx.GetContext().Err() - default: - } - name := ref.Name().Short() - sha := ref.Hash().String() - _, err := r.repo.CommitObject(ref.Hash()) - if err != nil && errors.Is(err, plumbing.ErrObjectNotFound) { - // handle commit sha like "0000000000000000000000000000000000000000" - h, err := r.repo.ResolveRevision(plumbing.Revision(ref.Name())) - if err != nil { - return err - } - sha = h.String() - } - codeRef := &code.Ref{ - DomainEntityExtended: domainlayer.DomainEntityExtended{Id: fmt.Sprintf("%s:%s", r.id, name)}, - RepoId: r.id, - Name: name, - CommitSha: sha, - RefType: BRANCH, - IsDefault: ref.Name() == headRef.Name(), - } - if err := r.store.Refs(codeRef); err != nil { - return err - } - subtaskCtx.IncProgress(1) - return nil - }); err != nil { - return err - } - return nil -} - -func (r *GogitRepoCollector) getComponentMap(subtaskCtx plugin.SubTaskContext) (map[string]*regexp.Regexp, error) { - db := subtaskCtx.GetDal() - components := make([]code.Component, 0) - err := db.All(&components, dal.From(components), dal.Where("repo_id= ?", r.id)) - if err != nil { - return nil, err - } - componentMap := make(map[string]*regexp.Regexp) - for _, component := range components { - componentMap[component.Name] = regexp.MustCompile(component.PathRegex) - } - return componentMap, nil -} - -// CollectCommits Collect data from each commit, we can also get the diff line -func (r *GogitRepoCollector) CollectCommits(subtaskCtx plugin.SubTaskContext) (err error) { - taskOpts := subtaskCtx.GetData().(*GitExtractorTaskData).Options - // check it first - componentMap, err := r.getComponentMap(subtaskCtx) - if err != nil { - return err - } - - repo := r.repo - store := r.store - - commitsObjectsIter, err := repo.CommitObjects() - if err != nil { - return err - } - - if err := commitsObjectsIter.ForEach(func(commit *object.Commit) error { - select { - case <-subtaskCtx.GetContext().Done(): - return subtaskCtx.GetContext().Err() - default: - } - commitSha := commit.Hash.String() - - if commit.NumParents() != 0 { - _, err := commit.Parents().Next() - if err != nil { - if err == plumbing.ErrObjectNotFound { - // Skip calculating commit statistics when there are parent commits, but the first one cannot be fetched from the ODB. - // This usually happens during a shallow clone for incremental collection. Otherwise, we might end up overwriting - // the correct addition/deletion data in the database with an absurdly large addition number. - r.logger.Info("skip commit %s because it has no parent commit", commitSha) - return nil - } - return err - } - } - codeCommit := &code.Commit{ - Sha: commitSha, - Message: commit.Message, - AuthorName: commit.Author.Name, - AuthorEmail: commit.Author.Email, - AuthorId: commit.Author.Email, - AuthoredDate: commit.Author.When, - CommitterName: commit.Committer.Name, - CommitterEmail: commit.Committer.Email, - CommitterId: commit.Committer.Email, - CommittedDate: commit.Committer.When, - } - if err = r.storeParentCommits(commitSha, commit); err != nil { - return err - } - - if !*taskOpts.SkipCommitStat { - stats, err := commit.StatsContext(subtaskCtx.GetContext()) - if err != nil { - return err - } else { - excluded := map[string]struct{}{} - for _, ext := range taskOpts.ExcludeFileExtensions { - e := strings.ToLower(strings.TrimSpace(ext)) - if e == "" { - continue - } - excluded[e] = struct{}{} - } - for _, stat := range stats { - nameLower := strings.ToLower(stat.Name) - skip := false - for ext := range excluded { - if strings.HasSuffix(nameLower, ext) { - skip = true - break - } - } - if skip { - continue - } - codeCommit.Additions += stat.Addition - // In some repos, deletion may be zero, which is different from git log --stat. - // It seems go-git doesn't get the correct changes. - // I have run object.DiffTreeWithOptions manually with different diff algorithms, - // but get the same result with StatsContext. - // I cannot reproduce it with another repo. - // A similar issue: https://github.com/go-git/go-git/issues/367 - codeCommit.Deletions += stat.Deletion - } - } - } - - err = store.Commits(codeCommit) - if err != nil { - return err - } - - codeRepoCommit := &code.RepoCommit{ - RepoId: r.id, - CommitSha: commitSha, - } - err = store.RepoCommits(codeRepoCommit) - if err != nil { - return err - } - if !*taskOpts.SkipCommitFiles { - if err := r.storeDiffCommitFilesComparedToParent(subtaskCtx, componentMap, commit, taskOpts.ExcludeFileExtensions); err != nil { - return err - } - } - subtaskCtx.IncProgress(1) - return nil - }); err != nil { - return err - } - return -} - -func (r *GogitRepoCollector) storeParentCommits(commitSha string, commit *object.Commit) error { - if commit == nil { - return nil - } - var commitParents []*code.CommitParent - for i := 0; i < commit.NumParents(); i++ { - parent, err := commit.Parent(i) - if err != nil { - // parent commit might not exist when repo is shallow cloned (tradeoff of supporting timeAfter parameter) - if err.Error() == "object not found" { - continue - } - return err - } - if parent != nil { - if parentCommitSha := parent.Hash.String(); parentCommitSha != "" { - commitParents = append(commitParents, &code.CommitParent{ - CommitSha: commitSha, - ParentCommitSha: parentCommitSha, - }) - } - } - } - return r.store.CommitParents(commitParents) -} - -func (r *GogitRepoCollector) getCurrentAndParentTree(ctx context.Context, commit *object.Commit) (*object.Tree, *object.Tree, error) { - if _, err := commit.Stats(); err != nil { - return nil, nil, err - } - commitTree, err := commit.Tree() - if err != nil { - return nil, nil, err - } - var firstParentTree *object.Tree - if commit.NumParents() > 0 { - firstParent, err := commit.Parents().Next() - if err != nil { - return nil, nil, err - } - firstParentTree, err = firstParent.Tree() - if err != nil { - return nil, nil, err - } - } - return commitTree, firstParentTree, nil -} - -func (r *GogitRepoCollector) storeDiffCommitFilesComparedToParent(subtaskCtx plugin.SubTaskContext, componentMap map[string]*regexp.Regexp, commit *object.Commit, excludeExts []string) (err error) { - commitTree, firstParentTree, err := r.getCurrentAndParentTree(subtaskCtx.GetContext(), commit) - if err != nil { - return err - } - // no parent, doesn't need to patch - patch, err := firstParentTree.PatchContext(subtaskCtx.GetContext(), commitTree) - if err != nil { - return err - } - // normalize exclusions - excluded := map[string]struct{}{} - for _, ext := range excludeExts { - e := strings.ToLower(strings.TrimSpace(ext)) - if e == "" { - continue - } - excluded[e] = struct{}{} - } - for _, p := range patch.Stats() { - commitFile := &code.CommitFile{ - CommitSha: commit.Hash.String(), - } - fileName := p.Name - commitFile.FilePath = fileName - if len(excluded) > 0 { - lower := strings.ToLower(fileName) - skip := false - for ext := range excluded { - if strings.HasSuffix(lower, ext) { - skip = true - break - } - } - if skip { - continue - } - } - commitFile.Id = genCommitFileId(commitFile.CommitSha, fileName) - commitFile.Deletions = p.Deletion - commitFile.Additions = p.Addition - if err := r.storeCommitFileComponents(subtaskCtx, componentMap, commitFile.Id, commitFile.FilePath); err != nil { - return err - } - err = r.store.CommitFiles(commitFile) - if err != nil { - r.logger.Error(err, "CommitFiles error") - return nil - } - } - return nil -} - -// With some long path,the varchar(255) was not enough both ID and file_path -// So we use the hash to compress the path in ID and add length of file_path. -// Use commitSha and the sha256 of FilePath to create id -func genCommitFileId(commitSha, filePath string) string { - shaFilePath := sha256.New() - shaFilePath.Write([]byte(filePath)) - return commitSha + ":" + hex.EncodeToString(shaFilePath.Sum(nil)) -} - -func (r *GogitRepoCollector) storeCommitFileComponents(subtaskCtx plugin.SubTaskContext, componentMap map[string]*regexp.Regexp, commitFileId string, commitFilePath string) error { - if commitFileId == "" || commitFilePath == "" { - return errors.Default.New("commit id r commit file path is empty") - } - commitFileComponent := &code.CommitFileComponent{ - CommitFileId: commitFileId, - ComponentName: "Default", - } - for component, reg := range componentMap { - if reg.MatchString(commitFilePath) { - commitFileComponent.ComponentName = component - break - } - } - return r.store.CommitFileComponents(commitFileComponent) -} - -// storeRepoSnapshot depends on commit list's order. -func (r *GogitRepoCollector) storeRepoSnapshot(subtaskCtx plugin.SubTaskContext, commitList []*object.Commit) error { - ctx := subtaskCtx.GetContext() - snapshot := make(map[string][]string) // {"filePathAndName": ["line1 commit sha", "line2 commit sha"]} - for _, commit := range commitList { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - commitTree, firstParentTree, err := r.getCurrentAndParentTree(ctx, commit) - if err != nil { - return err - } - patch, err := firstParentTree.PatchContext(subtaskCtx.GetContext(), commitTree) - if err != nil { - return err - } - for _, p := range patch.Stats() { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - fileName := p.Name - if _, ok := snapshot[fileName]; !ok { - snapshot[fileName] = []string{} - } - blameResults, err := gogit.Blame(commit, fileName) - if err != nil { - return err - } - var newBlames []string - for _, blameResult := range blameResults.Lines { - newBlames = append(newBlames, blameResult.Hash.String()) - } - snapshot[fileName] = newBlames - } - } - // store snapshots - for fileName, lineBlames := range snapshot { - for idx, lineBlameHash := range lineBlames { - lineNo := idx + 1 - repoSnapshot := &code.RepoSnapshot{ - RepoId: r.id, - CommitSha: lineBlameHash, - FilePath: fileName, - LineNo: lineNo, - } - if err := r.store.RepoSnapshot(repoSnapshot); err != nil { - r.logger.Error(err, "store RepoSnapshot error") - return err - } - } - } - return nil -} - -func (r *GogitRepoCollector) GetCommitList(subtaskCtx plugin.SubTaskContext) ([]*object.Commit, error) { - var commitList []*object.Commit - // get current head commit sha, default is master branch - // check branch, if not master, checkout to branch's head - commitOid, err := r.repo.Head() - if err != nil { - return nil, err - } - // get head commit object and add into commitList - commit, err := r.repo.CommitObject(commitOid.Hash()) - if err != nil { - return nil, err - } - commitList = append(commitList, commit) - // if current head has parents, get parent commit sha - for commit != nil && commit.NumParents() > 0 { - parentCommit, err := commit.Parent(0) - if err != nil { - return nil, err - } - commit, err = r.repo.CommitObject(parentCommit.Hash) - if err != nil { - return nil, err - } - commitList = append(commitList, commit) - } - // reverse commitList - // use slices.Reverse(commitList) in higher golang version. - for i, j := 0, len(commitList)-1; i < j; i, j = i+1, j-1 { - commitList[i], commitList[j] = commitList[j], commitList[i] - } - return commitList, nil -} - -func (r *GogitRepoCollector) CollectDiffLine(subtaskCtx plugin.SubTaskContext) error { - commitList, err := r.GetCommitList(subtaskCtx) - if err != nil { - return err - } - if err := r.storeRepoSnapshot(subtaskCtx, commitList); err != nil { - return err - } - // fixme: collecting CommitLineChange is not implemented. - // There is no way to get such information with go-git, and table commit_line_change is not used by any dashboards - // So we just ignore it. - return nil -} diff --git a/backend/plugins/gitextractor/parser/repo_libgit2.go b/backend/plugins/gitextractor/parser/repo_libgit2.go deleted file mode 100644 index ad9b515b4bf..00000000000 --- a/backend/plugins/gitextractor/parser/repo_libgit2.go +++ /dev/null @@ -1,698 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package parser - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "regexp" - "sort" - "strconv" - "strings" - - "github.com/apache/devlake/core/dal" - "github.com/apache/devlake/core/errors" - "github.com/apache/devlake/core/log" - "github.com/apache/devlake/core/models/domainlayer" - "github.com/apache/devlake/core/models/domainlayer/code" - "github.com/apache/devlake/core/plugin" - "github.com/apache/devlake/plugins/gitextractor/models" - - git "github.com/libgit2/git2go/v34" -) - -var _ RepoCollector = (*Libgit2RepoCollector)(nil) - -var TypeNotMatchError = "the requested type does not match the type in the ODB" - -type Libgit2RepoCollector struct { - id string - logger log.Logger - - store models.Store - repo *git.Repository - cleanup func() -} - -func NewLibgit2RepoCollector(localDir string, repoId string, store models.Store, logger log.Logger) (*Libgit2RepoCollector, errors.Error) { - repo, err := git.OpenRepository(localDir) - if err != nil { - return nil, errors.Convert(err) - } - return &Libgit2RepoCollector{ - id: repoId, - logger: logger, - store: store, - repo: repo, - }, nil -} - -func (r *Libgit2RepoCollector) SetCleanUp(f func()) error { - if f != nil { - r.cleanup = f - } - return nil -} - -// CollectAll The main parser subtask -func (r *Libgit2RepoCollector) CollectAll(subtaskCtx plugin.SubTaskContext) error { - subtaskCtx.SetProgress(0, -1) - err := r.CollectTags(subtaskCtx) - if err != nil { - return err - } - err = r.CollectBranches(subtaskCtx) - if err != nil { - return err - } - err = r.CollectCommits(subtaskCtx) - if err != nil { - return err - } - opt := subtaskCtx.GetData().(*GitExtractorTaskData).Options - if !*opt.SkipCommitStat { - return r.CollectDiffLine(subtaskCtx) - } - return nil -} - -// Close resources -func (r *Libgit2RepoCollector) Close(ctx context.Context) error { - defer func() { - if r.cleanup != nil { - r.cleanup() - } - }() - return r.store.Close() -} - -// CountTags Count git tags subtask -func (r *Libgit2RepoCollector) CountTags(ctx context.Context) (int, error) { - tags, err := r.repo.Tags.List() - if err != nil { - return 0, errors.Convert(err) - } - return len(tags), nil -} - -// CountBranches count the number of branches in a git repo -func (r *Libgit2RepoCollector) CountBranches(ctx context.Context) (int, error) { - var branchIter *git.BranchIterator - branchIter, err := r.repo.NewBranchIterator(git.BranchAll) - if err != nil { - return 0, errors.Convert(err) - } - count := 0 - err = branchIter.ForEach(func(branch *git.Branch, branchType git.BranchType) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - if branch.IsBranch() || branch.IsRemote() { - isHead, _ := branch.IsHead() - if !isHead { - count++ - } - } - return nil - }) - return count, errors.Convert(err) -} - -// CountCommits count the number of commits in a git repo -func (r *Libgit2RepoCollector) CountCommits(ctx context.Context) (int, error) { - odb, err := r.repo.Odb() - if err != nil { - return 0, errors.Convert(err) - } - count := 0 - err = odb.ForEach(func(id *git.Oid) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - commit, e := r.repo.LookupCommit(id) - if e != nil && e.Error() != TypeNotMatchError { - return errors.Convert(e) - } - if commit != nil { - count++ - } - return nil - }) - return count, errors.Convert(err) -} - -// CollectTags Collect Tags data -func (r *Libgit2RepoCollector) CollectTags(subtaskCtx plugin.SubTaskContext) error { - return errors.Convert(r.repo.Tags.Foreach(func(name string, id *git.Oid) error { - select { - case <-subtaskCtx.GetContext().Done(): - return subtaskCtx.GetContext().Err() - default: - } - var err1 error - var tag *git.Tag - var tagCommit string - tag, err1 = r.repo.LookupTag(id) - if err1 != nil && err1.Error() != TypeNotMatchError { - return errors.Convert(err1) - } - if tag != nil { - tagCommit = tag.TargetId().String() - } else { - tagCommit = id.String() - } - r.logger.Debug("tagCommit: %s", tagCommit) - if tagCommit != "" { - ref := &code.Ref{ - DomainEntityExtended: domainlayer.DomainEntityExtended{Id: fmt.Sprintf("%s:%s", r.id, name)}, - RepoId: r.id, - Name: name, - CommitSha: tagCommit, - RefType: TAG, - } - err1 = r.store.Refs(ref) - if err1 != nil { - return err1 - } - subtaskCtx.IncProgress(1) - } - return nil - })) -} - -// CollectBranches Collect branch data -func (r *Libgit2RepoCollector) CollectBranches(subtaskCtx plugin.SubTaskContext) error { - var repoInter *git.BranchIterator - repoInter, err := r.repo.NewBranchIterator(git.BranchAll) - if err != nil { - return errors.Convert(err) - } - return errors.Convert(repoInter.ForEach(func(branch *git.Branch, branchType git.BranchType) error { - select { - case <-subtaskCtx.GetContext().Done(): - return subtaskCtx.GetContext().Err() - default: - } - if branch.IsBranch() || branch.IsRemote() { - name, err1 := branch.Name() - if err1 != nil && err1.Error() != TypeNotMatchError { - return err1 - } - var sha string - if oid := branch.Target(); oid != nil { - sha = oid.String() - } - ref := &code.Ref{ - DomainEntityExtended: domainlayer.DomainEntityExtended{Id: fmt.Sprintf("%s:%s", r.id, name)}, - RepoId: r.id, - Name: name, - CommitSha: sha, - RefType: BRANCH, - } - // commit sha may be an empty string, it's unexpected. - ref.IsDefault, err1 = branch.IsHead() - if err1 != nil && err1.Error() != TypeNotMatchError { - return err1 - } - err1 = r.store.Refs(ref) - if err1 != nil && err1.Error() != TypeNotMatchError { - return err1 - } - subtaskCtx.IncProgress(1) - return nil - } - return nil - })) -} - -// CollectCommits Collect data from each commit, we can also get the diff line -func (r *Libgit2RepoCollector) CollectCommits(subtaskCtx plugin.SubTaskContext) error { - taskOpts := subtaskCtx.GetData().(*GitExtractorTaskData).Options - opts, err := getDiffOpts() - if err != nil { - return err - } - db := subtaskCtx.GetDal() - components := make([]code.Component, 0) - err = db.All(&components, dal.From(components), dal.Where("repo_id= ?", r.id)) - if err != nil { - return err - } - componentMap := make(map[string]*regexp.Regexp) - for _, component := range components { - componentMap[component.Name] = regexp.MustCompile(component.PathRegex) - } - odb, err := errors.Convert01(r.repo.Odb()) - if err != nil { - return err - } - return errors.Convert(odb.ForEach(func(id *git.Oid) error { - select { - case <-subtaskCtx.GetContext().Done(): - return subtaskCtx.GetContext().Err() - default: - } - commit, err1 := r.repo.LookupCommit(id) - if err1 != nil && err1.Error() != TypeNotMatchError { - return errors.Convert(err1) - } - if commit == nil { - return nil - } - var parent *git.Commit - if commit.ParentCount() > 0 { - parent = commit.Parent(0) - // Skip calculating commit statistics when there are parent commits, but the first one cannot be fetched from the ODB. - // This usually happens during a shallow clone for incremental collection. Otherwise, we might end up overwriting - // the correct addition/deletion data in the database with an absurdly large addition number. - if parent == nil { - r.logger.Info("skip commit %s because it has no parent commit", commit.Id().String()) - return nil - } - } - commitSha := commit.Id().String() - r.logger.Debug("process commit: %s", commitSha) - c := &code.Commit{ - Sha: commitSha, - Message: commit.Message(), - } - author := commit.Author() - if author != nil { - c.AuthorName = author.Name - c.AuthorEmail = author.Email - c.AuthorId = author.Email - c.AuthoredDate = author.When - } - committer := commit.Committer() - if committer != nil { - c.CommitterName = committer.Name - c.CommitterEmail = committer.Email - c.CommitterId = committer.Email - c.CommittedDate = committer.When - } - err = r.storeParentCommits(commitSha, commit) - if err != nil { - return err - } - - if !*taskOpts.SkipCommitStat { - var stats *git.DiffStats - var addIncluded, delIncluded int - if stats, addIncluded, delIncluded, err = r.getDiffComparedToParent(taskOpts, c.Sha, commit, parent, opts, componentMap); err != nil { - return err - } - r.logger.Debug("state: %#+v\n", stats.Deletions()) - c.Additions += addIncluded - c.Deletions += delIncluded - } - - err = r.store.Commits(c) - if err != nil { - return err - } - repoCommit := &code.RepoCommit{ - RepoId: r.id, - CommitSha: c.Sha, - } - err = r.store.RepoCommits(repoCommit) - if err != nil { - return err - } - subtaskCtx.IncProgress(1) - return nil - })) -} - -func (r *Libgit2RepoCollector) storeParentCommits(commitSha string, commit *git.Commit) errors.Error { - var commitParents []*code.CommitParent - for i := uint(0); i < commit.ParentCount(); i++ { - parent := commit.Parent(i) - if parent != nil { - if parentId := parent.Id(); parentId != nil { - commitParents = append(commitParents, &code.CommitParent{ - CommitSha: commitSha, - ParentCommitSha: parentId.String(), - }) - } - } - } - return r.store.CommitParents(commitParents) -} - -func (r *Libgit2RepoCollector) getDiffComparedToParent(taskOpts *GitExtractorOptions, commitSha string, commit *git.Commit, parent *git.Commit, opts *git.DiffOptions, componentMap map[string]*regexp.Regexp) (*git.DiffStats, int, int, errors.Error) { - var err error - var parentTree, tree *git.Tree - if parent != nil { - parentTree, err = parent.Tree() - } - if err != nil { - return nil, 0, 0, errors.Convert(err) - } - tree, err = commit.Tree() - if err != nil { - return nil, 0, 0, errors.Convert(err) - } - var diff *git.Diff - diff, err = r.repo.DiffTreeToTree(parentTree, tree, opts) - if err != nil { - return nil, 0, 0, errors.Convert(err) - } - // build excluded extension set - excluded := map[string]struct{}{} - for _, ext := range taskOpts.ExcludeFileExtensions { - e := strings.ToLower(strings.TrimSpace(ext)) - if e == "" { - continue - } - excluded[e] = struct{}{} - } - if !*taskOpts.SkipCommitFiles { - err = r.storeCommitFilesFromDiff(commitSha, diff, componentMap, excluded) - if err != nil { - return nil, 0, 0, errors.Convert(err) - } - } - var stats *git.DiffStats - stats, err = diff.Stats() - if err != nil { - return nil, 0, 0, errors.Convert(err) - } - // calculate included totals with exclusions - addIncluded := 0 - delIncluded := 0 - if len(excluded) == 0 { - addIncluded = stats.Insertions() - delIncluded = stats.Deletions() - return stats, addIncluded, delIncluded, nil - } - _ = diff.ForEach(func(file git.DiffDelta, progress float64) (git.DiffForEachHunkCallback, error) { - // choose path to check based on delta status; for deletions use old path - pathForCheck := file.NewFile.Path - if file.Status == git.DeltaDeleted || pathForCheck == "" { - pathForCheck = file.OldFile.Path - } - lower := strings.ToLower(pathForCheck) - for ext := range excluded { - if strings.HasSuffix(lower, ext) { - // skip all lines for excluded files - return func(hunk git.DiffHunk) (git.DiffForEachLineCallback, error) { - return func(line git.DiffLine) error { return nil }, nil - }, nil - } - } - return func(hunk git.DiffHunk) (git.DiffForEachLineCallback, error) { - return func(line git.DiffLine) error { - if line.Origin == git.DiffLineAddition { - addIncluded += line.NumLines - } - if line.Origin == git.DiffLineDeletion { - delIncluded += line.NumLines - } - return nil - }, nil - }, nil - }, git.DiffDetailLines) - return stats, addIncluded, delIncluded, nil -} - -func (r *Libgit2RepoCollector) storeCommitFilesFromDiff(commitSha string, diff *git.Diff, componentMap map[string]*regexp.Regexp, excluded map[string]struct{}) errors.Error { - var commitFile *code.CommitFile - var commitFileComponent *code.CommitFileComponent - var err error - err = diff.ForEach(func(file git.DiffDelta, progress float64) ( - git.DiffForEachHunkCallback, error) { - if commitFile != nil { - err = r.store.CommitFiles(commitFile) - if err != nil { - r.logger.Error(err, "CommitFiles error") - return nil, err - } - } - - // skip files by extension if configured - if len(excluded) > 0 { - pathForCheck := file.NewFile.Path - if file.Status == git.DeltaDeleted || pathForCheck == "" { - pathForCheck = file.OldFile.Path - } - lower := strings.ToLower(pathForCheck) - for ext := range excluded { - if strings.HasSuffix(lower, ext) { - // skip this file entirely - return func(hunk git.DiffHunk) (git.DiffForEachLineCallback, error) { - return func(line git.DiffLine) error { return nil }, nil - }, nil - } - } - } - - commitFile = new(code.CommitFile) - commitFile.CommitSha = commitSha - // prefer new path; for deletions fall back to old path - commitFile.FilePath = file.NewFile.Path - if commitFile.FilePath == "" { - commitFile.FilePath = file.OldFile.Path - } - - // With some long path,the varchar(255) was not enough both ID and file_path - // So we use the hash to compress the path in ID and add length of file_path. - // Use commitSha and the sha256 of FilePath to create id - shaFilePath := sha256.New() - shaFilePath.Write([]byte(commitFile.FilePath)) - commitFile.Id = commitSha + ":" + hex.EncodeToString(shaFilePath.Sum(nil)) - - commitFileComponent = new(code.CommitFileComponent) - for component, reg := range componentMap { - if reg.MatchString(commitFile.FilePath) { - commitFileComponent.ComponentName = component - break - } - } - commitFileComponent.CommitFileId = commitFile.Id - if commitFileComponent.ComponentName == "" { - commitFileComponent.ComponentName = "Default" - } - return func(hunk git.DiffHunk) (git.DiffForEachLineCallback, error) { - return func(line git.DiffLine) error { - if line.Origin == git.DiffLineAddition { - commitFile.Additions += line.NumLines - } - if line.Origin == git.DiffLineDeletion { - commitFile.Deletions += line.NumLines - } - return nil - }, nil - }, nil - }, git.DiffDetailLines) - if commitFileComponent != nil { - err = r.store.CommitFileComponents(commitFileComponent) - if err != nil { - r.logger.Error(err, "CommitFileComponents error") - } - } - if commitFile != nil { - err = r.store.CommitFiles(commitFile) - if err != nil { - r.logger.Error(err, "CommitFiles error") - } - } - return errors.Convert(err) -} - -// CollectDiffLine get line diff data from a specific branch -func (r *Libgit2RepoCollector) CollectDiffLine(subtaskCtx plugin.SubTaskContext) error { - //Using this subtask,we can get every line change in every commit. - //We maintain a snapshot structure to get which commit each deleted line belongs to - snapshot := make(map[string] /*file path*/ *models.FileBlame) - repo := r.repo - //step 1. get the reverse commit list - commitList := make([]git.Commit, 0) - // get current head commit sha, default is master branch - // check branch, if not master, checkout to branch's head - commitOid, err1 := repo.Head() - if err1 != nil && err1.Error() != TypeNotMatchError { - return errors.Convert(err1) - } - //get head commit object and add into commitList - commit, err1 := repo.LookupCommit(commitOid.Target()) - if err1 != nil && err1.Error() != TypeNotMatchError { - return errors.Convert(err1) - } - commitList = append(commitList, *commit) - // if current head has parents, get parent commitsha - for commit != nil && commit.ParentCount() > 0 { - pid := commit.ParentId(0) - commit, err1 = repo.LookupCommit(pid) - if err1 != nil && err1.Error() != TypeNotMatchError { - return errors.Convert(err1) - } - commitList = append(commitList, *commit) - } - // reverse commitList - for i, j := 0, len(commitList)-1; i < j; i, j = i+1, j-1 { - commitList[i], commitList[j] = commitList[j], commitList[i] - } - //step 2. get the diff of each commit - // for each commit, get the diff - for _, commitsha := range commitList { - curcommit, err := repo.LookupCommit(commitsha.Id()) - if err != nil { - return errors.Convert(err) - } - if curcommit.ParentCount() == 0 || curcommit.ParentCount() > 0 { - var parentTree, tree *git.Tree - tree, err = curcommit.Tree() - if err != nil { - return errors.Convert(err) - } - var diff *git.Diff - //FIXME error type convert - opts, err := git.DefaultDiffOptions() - opts.NotifyCallback = func(diffSoFar *git.Diff, delta git.DiffDelta, matchedPathSpec string) error { - return nil - } - if err != nil { - return errors.Convert(err) - } - if curcommit.ParentCount() > 0 { - parent := curcommit.Parent(0) - parentTree, err = parent.Tree() - } - diff, err = repo.DiffTreeToTree(parentTree, tree, &opts) - if err != nil { - return errors.Convert(err) - } - deleted := make(models.DiffLines, 0) - added := make(models.DiffLines, 0) - var lastFile string - lastFile = "" - err = diff.ForEach(func(file git.DiffDelta, progress float64) (git.DiffForEachHunkCallback, error) { - // if it doesn't exist in snapshot, create a new one - if _, ok := snapshot[file.OldFile.Path]; !ok { - fileBlame, err := models.NewFileBlame() - if err != nil { - r.logger.Info("Create FileBlame Error") - return nil, err - } - snapshot[file.OldFile.Path] = (*models.FileBlame)(fileBlame) - } - if lastFile == "" { - lastFile = file.NewFile.Path - } else if lastFile != file.NewFile.Path { - updateSnapshotFileBlame(curcommit, deleted, added, lastFile, snapshot) - // reset the deleted and added,last_file now is current file - deleted = make([]git.DiffLine, 0) - added = make([]git.DiffLine, 0) - lastFile = file.NewFile.Path - } - hunkNum := 0 - return func(hunk git.DiffHunk) (git.DiffForEachLineCallback, error) { - hunkNum++ - return func(line git.DiffLine) error { - commitLineChange := &code.CommitLineChange{} - commitLineChange.CommitSha = curcommit.Id().String() - commitLineChange.ChangedType = line.Origin.String() - commitLineChange.LineNoNew = line.NewLineno - commitLineChange.LineNoOld = line.OldLineno - commitLineChange.OldFilePath = file.OldFile.Path - commitLineChange.NewFilePath = file.NewFile.Path - commitLineChange.HunkNum = hunkNum - commitLineChange.Id = curcommit.Id().String() + ":" + file.NewFile.Path + ":" + strconv.Itoa(line.OldLineno) + ":" + strconv.Itoa(line.NewLineno) - if line.Origin == git.DiffLineAddition { - added = append(added, line) - } else if line.Origin == git.DiffLineDeletion { - fb := snapshot[file.OldFile.Path] - l := fb.Find(line.OldLineno) - if l != nil && l.Value != nil { - temp := snapshot[file.OldFile.Path].Find(line.OldLineno) - commitLineChange.PrevCommit = temp.Value.(string) - } else { - r.logger.Info("err", file.OldFile.Path, line.OldLineno, curcommit.Id().String()) - } - deleted = append(deleted, line) - } - err = r.store.CommitLineChange(commitLineChange) - if err != nil { - return errors.Convert(err) - } - return nil - }, nil - }, nil - }, git.DiffDetailLines) - if err != nil { - return errors.Convert(err) - } - //finally,process the last file in diff - updateSnapshotFileBlame(curcommit, deleted, added, lastFile, snapshot) - } - } - r.logger.Info("line change collect success") - db := subtaskCtx.GetDal() - err := db.Delete(&code.RepoSnapshot{}, dal.Where("repo_id= ?", r.id)) - if err != nil { - return errors.Convert(err) - } - for fp := range snapshot { - temp := snapshot[fp] - count := 0 - for e := temp.Lines.Front(); e != nil; e = e.Next() { - count++ - snapshotLine := &code.RepoSnapshot{} - snapshotLine.RepoId = r.id - snapshotLine.LineNo = count - snapshotLine.CommitSha = e.Value.(string) - snapshotLine.FilePath = fp - err := r.store.RepoSnapshot(snapshotLine) - if err != nil { - r.logger.Info("error") - return err - } - } - - } - - r.logger.Info("collect snapshot finished") - return nil -} - -func updateSnapshotFileBlame(currentCommit *git.Commit, deleted models.DiffLines, added models.DiffLines, lastFile string, snapshot map[string]*models.FileBlame) { - sort.Sort(deleted) - for _, line := range deleted { - snapshot[lastFile].RemoveLine(line.OldLineno) - } - for _, line := range added { - snapshot[lastFile].AddLine(line.NewLineno, currentCommit.Id().String()) - } -} - -func getDiffOpts() (*git.DiffOptions, errors.Error) { - opts, err := git.DefaultDiffOptions() - if err != nil { - return nil, errors.Convert(err) - } - opts.NotifyCallback = func(diffSoFar *git.Diff, delta git.DiffDelta, matchedPathSpec string) error { - return nil - } - return &opts, nil -} diff --git a/backend/plugins/gitextractor/parser/taskdata.go b/backend/plugins/gitextractor/parser/taskdata.go index a8bb4510d9a..707c27b640f 100644 --- a/backend/plugins/gitextractor/parser/taskdata.go +++ b/backend/plugins/gitextractor/parser/taskdata.go @@ -41,7 +41,6 @@ type GitExtractorOptions struct { PrivateKey string `json:"privateKey" mapstructure:"privateKey"` Passphrase string `json:"passphrase" mapstructure:"passphrase"` Proxy string `json:"proxy" mapstructure:"proxy"` - UseGoGit *bool `json:"useGoGit" mapstructure:"useGoGit"` SkipCommitStat *bool `json:"skipCommitStat" mapstructure:"skipCommitStat" comment:"skip all commit stat including added/deleted lines and commit files as well"` SkipCommitFiles *bool `json:"skipCommitFiles" mapstructure:"skipCommitFiles"` NoShallowClone bool `json:"noShallowClone" mapstructure:"noShallowClone"` @@ -49,8 +48,4 @@ type GitExtractorOptions struct { PluginName string `json:"pluginName" mapstructure:"pluginName,omitempty"` // Configured by upstream plugin (e.g., GitLab) to exclude file extensions from commit stats ExcludeFileExtensions []string `json:"excludeFileExtensions" mapstructure:"excludeFileExtensions"` - // UseGitCli selects the git command line collector. It is a temporary - // migration switch and will be removed once the library based collectors - // are gone. - UseGitCli bool `json:"useGitCli,omitempty" mapstructure:"useGitCli"` } diff --git a/backend/plugins/gitextractor/tasks/repo_cloner.go b/backend/plugins/gitextractor/tasks/repo_cloner.go index 63e58527b54..0c218e9d80d 100644 --- a/backend/plugins/gitextractor/tasks/repo_cloner.go +++ b/backend/plugins/gitextractor/tasks/repo_cloner.go @@ -69,15 +69,7 @@ func CloneGitRepo(subTaskCtx plugin.SubTaskContext) errors.Error { if repoCloner.IsIncremental() { storage.SetIncrementalMode(repoCloner.IsIncremental()) } - // We have done comparison experiments for git2go and go-git, and the results show that git2go has better performance. - var repoCollector parser.RepoCollector - if taskData.Options.UseGitCli { - repoCollector, err = parser.NewGitcliRepoCollector(localDir, op.RepoId, storage, logger) - } else if *taskData.Options.UseGoGit { - repoCollector, err = parser.NewGogitRepoCollector(localDir, op.RepoId, storage, logger) - } else { - repoCollector, err = parser.NewLibgit2RepoCollector(localDir, op.RepoId, storage, logger) - } + repoCollector, err := parser.NewGitcliRepoCollector(localDir, op.RepoId, storage, logger) if err != nil { return err } diff --git a/backend/plugins/kiro/impl/table_info_test.go b/backend/plugins/kiro/impl/table_info_test.go index 90598fd1e12..3291b12c3ef 100644 --- a/backend/plugins/kiro/impl/table_info_test.go +++ b/backend/plugins/kiro/impl/table_info_test.go @@ -24,10 +24,9 @@ import ( ) // The repo-wide plugins/table_info_test.go performs this same check for every -// plugin, but building that package requires gitextractor and therefore a -// specific libgit2. Running it here as well keeps the feedback local: a model -// added without registering it in GetTablesInfo fails immediately rather than -// only in CI. +// plugin. Running it here as well keeps the feedback local: a model added +// without registering it in GetTablesInfo fails immediately rather than only +// in CI. func TestKiroTableInfo(t *testing.T) { checker := unithelper.NewTableInfoChecker(unithelper.TableInfoCheckerConfig{}) checker.FeedIn("../models", Kiro{}.GetTablesInfo) diff --git a/backend/python/DevelopmentSetup.md b/backend/python/DevelopmentSetup.md index c831313c8de..c171bd81dbf 100644 --- a/backend/python/DevelopmentSetup.md +++ b/backend/python/DevelopmentSetup.md @@ -16,7 +16,7 @@ limitations under the License. --> # Python prerequisites for `make e2e-test` -`make e2e-test` also requires a test database and, for some service tests, `libgit2`. See +`make e2e-test` also requires a test database and `git` for the service tests. See [`backend/test/Readme.md`](../test/Readme.md) for the full local E2E test flow. For the Python plugin tests to run properly, the following steps must be taken: diff --git a/backend/scripts/install-libgit2.sh b/backend/scripts/install-libgit2.sh deleted file mode 100755 index 7f6ced6184c..00000000000 --- a/backend/scripts/install-libgit2.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eu - -LIBGIT2_VERSION=1.5.0 -LIBGIT2_SHA256=8de872a0f201b33d9522b817c92e14edb4efad18dae95cf156cf240b2efff93e -LIBGIT2_URL=https://github.com/libgit2/libgit2/archive/refs/tags/v${LIBGIT2_VERSION}.tar.gz - -current_version=$(pkg-config --modversion libgit2 2>/dev/null || true) -case "$current_version" in - 1.5.*) - echo "libgit2 $current_version is already installed" - exit 0 - ;; -esac - -if [ "$(id -u)" -ne 0 ]; then - echo "install-libgit2.sh must run as root" >&2 - exit 1 -fi - -build_dir=$(mktemp -d) -trap 'rm -rf "$build_dir"' EXIT HUP INT TERM - -export DEBIAN_FRONTEND=noninteractive - -# Debian releases are pulled from security.debian.org/deb.debian.org only -# while they have active (LTS) support. Once a release goes EOL, its -# packages move to archive.debian.org and the live mirrors 404. Repoint -# apt at the archive for releases we know are EOL so this keeps working -# without depending on the CI image being rebuilt on a newer base. -eol_codenames="bullseye" -codename=$(. /etc/os-release && echo "$VERSION_CODENAME") -case " $eol_codenames " in - *" $codename "*) - echo "Debian $codename is EOL; repointing apt sources at archive.debian.org" - for f in /etc/apt/sources.list /etc/apt/sources.list.d/*.list; do - [ -f "$f" ] || continue - sed -i \ - -e "s|http://deb.debian.org/debian|http://archive.debian.org/debian|g" \ - -e "s|http://security.debian.org/debian-security|http://archive.debian.org/debian-security|g" \ - "$f" - done - echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99no-check-valid-until - ;; -esac - -apt_update_and_install() { - apt-get update - apt-get install -y --no-install-recommends \ - ca-certificates \ - cmake \ - curl \ - gcc \ - libssh2-1-dev \ - libssl-dev \ - make \ - pkg-config \ - zlib1g-dev -} - -n=0 -until apt_update_and_install; do - n=$((n + 1)) - if [ "$n" -ge 3 ]; then - echo "apt-get update/install failed after $n attempts" >&2 - exit 1 - fi - echo "apt-get update/install failed, retrying ($n/3)..." >&2 - apt-get clean - sleep 5 -done - -archive="$build_dir/libgit2.tar.gz" -curl --fail --location --retry 3 "$LIBGIT2_URL" --output "$archive" -echo "$LIBGIT2_SHA256 $archive" | sha256sum --check --strict - -mkdir "$build_dir/source" -tar -xzf "$archive" -C "$build_dir/source" --strip-components=1 - -multiarch=$(gcc -print-multiarch) -libdir=lib -if [ -n "$multiarch" ]; then - libdir="lib/$multiarch" -fi - -cmake \ - -S "$build_dir/source" \ - -B "$build_dir/build" \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_TESTS=OFF \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_LIBDIR="$libdir" \ - -DCMAKE_INSTALL_PREFIX=/usr -cmake --build "$build_dir/build" --parallel "$(getconf _NPROCESSORS_ONLN)" -cmake --install "$build_dir/build" -ldconfig - -export PKG_CONFIG_PATH="/usr/$libdir/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" -installed_version=$(pkg-config --modversion libgit2) -if [ "$installed_version" != "$LIBGIT2_VERSION" ]; then - echo "expected libgit2 $LIBGIT2_VERSION, found $installed_version" >&2 - exit 1 -fi - -if [ -n "${GITHUB_ENV:-}" ]; then - echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH" >> "$GITHUB_ENV" -fi - -rm -rf /var/lib/apt/lists/* -echo "installed libgit2 $installed_version" diff --git a/backend/test/Readme.md b/backend/test/Readme.md index 87bfcfa30b2..70931816ee3 100644 --- a/backend/test/Readme.md +++ b/backend/test/Readme.md @@ -32,15 +32,8 @@ go run ./test/init.go make e2e-test ``` -The service E2E tests compile the `gitextractor` plugin, which links against `libgit2`. If `libgit2` is installed in -a non-standard location, expose it through `pkg-config` and add an rpath so the test binary can load the dynamic -library at runtime: - -```bash -export PKG_CONFIG_PATH=/path/to/libgit2/build -export CGO_LDFLAGS='-L/path/to/libgit2/build -Wl,-rpath,/path/to/libgit2/build' -make e2e-test -``` +The service E2E tests run the `gitextractor` plugin, which reads repositories +with the `git` command line. Make sure `git` (2.31 or newer) is installed. ### Planning-failure database regression test diff --git a/devops/docker/lake-builder/Dockerfile b/devops/docker/lake-builder/Dockerfile index 95cd0544eef..64c95bbebe3 100644 --- a/devops/docker/lake-builder/Dockerfile +++ b/devops/docker/lake-builder/Dockerfile @@ -14,51 +14,14 @@ # limitations under the License. # -# Debian 11 (bullseye) reached the end of its LTS period on 2026-08-31 and its -# security suite is no longer served, so `apt upgrade` fails there. All stages -# use Debian 12 (bookworm), matching the runtime image in backend/Dockerfile. -FROM --platform=linux/amd64 debian:bookworm AS debian-amd64 -RUN apt-get -y update && apt -y upgrade &&\ - apt-get install -y libssh2-1-dev libssl-dev zlib1g-dev - -# This stage only cross-builds libgit2, it does not need a Go toolchain. -FROM --platform=linux/amd64 debian:bookworm AS builder - -# Base dependencies -RUN apt-get -y update && apt -y upgrade &&\ - apt-get install -y ca-certificates wget gcc binutils libfindbin-libs-perl cmake libssh2-1-dev libssl-dev zlib1g-dev \ - gcc-x86-64-linux-gnu binutils-x86-64-linux-gnu - -COPY --from=debian-amd64 /usr/include /rootfs-amd64/usr/include -COPY --from=debian-amd64 /usr/lib/x86_64-linux-gnu /rootfs-amd64/usr/lib/x86_64-linux-gnu -COPY --from=debian-amd64 /lib/x86_64-linux-gnu /rootfs-amd64/lib/x86_64-linux-gnu - -## Build libs/headers needed for gitextractor plugin -RUN \ - mkdir -p /tmp/build/x86_64 && cd /tmp/build/x86_64 && \ - wget https://github.com/libgit2/libgit2/archive/refs/tags/v1.5.0.tar.gz -O - | tar -xz && \ - cd libgit2-1.5.0 && \ - mkdir build && cd build && \ - cmake .. -DCMAKE_C_COMPILER=x86_64-linux-gnu-gcc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTS=OFF -DCMAKE_SYSROOT=/rootfs-amd64 -DCMAKE_INSTALL_PREFIX=/usr/local/deps/x86_64 &&\ - make -j install &&\ - mkdir -p /tmp/deps &&\ - cp *libgit2* /tmp/deps/ &&\ - cp -r ../include /tmp/deps/include - # Same Python line as the runtime image in backend/Dockerfile. FROM python:3.11-slim-bookworm RUN apt -y update && apt -y upgrade && apt -y install tzdata make tar curl gcc g++ pkg-config git \ - libssh2-1 zlib1g libffi-dev \ + zlib1g libffi-dev \ default-libmysqlclient-dev \ libpq-dev -# Install Libs/Headers from previous stage -COPY --from=builder /tmp/deps/*.so* /usr/lib/ -COPY --from=builder /tmp/deps/*.pc /usr/lib/x86_64-linux-gnu/pkgconfig/ -COPY --from=builder /tmp/deps/include/ /usr/include/ -ENV PKG_CONFIG_PATH=/usr/lib/x86_64-linux-gnu/pkgconfig - # Install Golang # The previous installer was fetched from `git.io`, which GitHub shut down in # 2022, so the image could no longer be rebuilt. Fetch the official archive diff --git a/env.example b/env.example index 68be8d6d249..9a4acb0dc51 100755 --- a/env.example +++ b/env.example @@ -111,9 +111,6 @@ FORBID_REDIRECTION=false GITLAB_SERVER_COLLECT_ALL_USERS=true ########################## -# In plugin gitextractor, use go-git to collector repo's data -########################## -USE_GO_GIT_IN_GIT_EXTRACTOR=false # NOTE that COMMIT_FILES is part of the COMMIT_STAT SKIP_COMMIT_STAT=false SKIP_COMMIT_FILES=true From b78c827a84b1bf87da279838a52f1c87b5a41e54 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Sun, 1 Nov 2026 18:48:42 +0800 Subject: [PATCH 4/5] perf(gitextractor): only write component rows when components exist Nothing in DevLake populates the `components` table, so the component map is empty for normal repositories and every commit file got a "Default" row in `commit_file_components`. That is pure write amplification: 67,520 rows for the 8,383 commits of apache/devlake. Skip the component rows entirely when the repository has no components configured, and skip the component query when commit files are not collected. When components are configured, keep writing the correct per-file rows. The previous libgit2 collector only wrote a row for the last file of each commit, which was not a documented optimisation and assigns an arbitrary file's component; go-git already wrote one row per file. --- .../parser/collector_contract_test.go | 7 ++-- .../gitextractor/parser/repo_gitcli.go | 36 +++++++++++-------- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/backend/plugins/gitextractor/parser/collector_contract_test.go b/backend/plugins/gitextractor/parser/collector_contract_test.go index 6e2bf8d07ce..c2029961b0f 100644 --- a/backend/plugins/gitextractor/parser/collector_contract_test.go +++ b/backend/plugins/gitextractor/parser/collector_contract_test.go @@ -337,10 +337,9 @@ func TestCollectorCommits(t *testing.T) { for _, file := range store.commitFiles { assert.Equal(t, genCommitFileId(file.CommitSha, file.FilePath), file.Id) } - components := componentsById(store.components) - for _, file := range store.commitFiles { - assert.Equal(t, "Default", components[file.Id], file.FilePath) - } + // This repository has no components configured, so every file would + // only get a meaningless "Default" component row. + assert.Empty(t, store.components) }) } diff --git a/backend/plugins/gitextractor/parser/repo_gitcli.go b/backend/plugins/gitextractor/parser/repo_gitcli.go index f13256daae6..60ef6b70975 100644 --- a/backend/plugins/gitextractor/parser/repo_gitcli.go +++ b/backend/plugins/gitextractor/parser/repo_gitcli.go @@ -230,9 +230,12 @@ func (r *GitcliRepoCollector) CollectCommits(subtaskCtx plugin.SubTaskContext) ( if err != nil { return err } - componentMap, err := r.componentMap(subtaskCtx) - if err != nil { - return err + var componentMap map[string]*regexp.Regexp + if !*taskOpts.SkipCommitFiles { + componentMap, err = r.componentMap(subtaskCtx) + if err != nil { + return err + } } excluded := normalizeExtensions(taskOpts.ExcludeFileExtensions) @@ -344,6 +347,9 @@ func (r *GitcliRepoCollector) storeParentCommits(commit *parsedCommit, present m return r.store.CommitParents(parents) } +// storeCommitFile stores one changed file of a commit. Component rows are only +// written when the repository has components configured: without them every row +// would be "Default" for every changed file, which is pure write amplification. func (r *GitcliRepoCollector) storeCommitFile(commitSha string, file commitFileStat, componentMap map[string]*regexp.Regexp) errors.Error { commitFile := &code.CommitFile{ CommitSha: commitSha, @@ -353,18 +359,20 @@ func (r *GitcliRepoCollector) storeCommitFile(commitSha string, file commitFileS } commitFile.Id = genCommitFileId(commitFile.CommitSha, commitFile.FilePath) - componentName := "Default" - for _, name := range sortedComponentNames(componentMap) { - if componentMap[name].MatchString(commitFile.FilePath) { - componentName = name - break + if len(componentMap) > 0 { + componentName := "Default" + for _, name := range sortedComponentNames(componentMap) { + if componentMap[name].MatchString(commitFile.FilePath) { + componentName = name + break + } + } + if err := r.store.CommitFileComponents(&code.CommitFileComponent{ + CommitFileId: commitFile.Id, + ComponentName: componentName, + }); err != nil { + return err } - } - if err := r.store.CommitFileComponents(&code.CommitFileComponent{ - CommitFileId: commitFile.Id, - ComponentName: componentName, - }); err != nil { - return err } return r.store.CommitFiles(commitFile) } From c8e5bddf6f4c775d9572176882d63504adc5d898 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Sun, 1 Nov 2026 19:09:54 +0800 Subject: [PATCH 5/5] fix(gitextractor): drop unused line scanner helper --- backend/plugins/gitextractor/parser/gitrunner.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/backend/plugins/gitextractor/parser/gitrunner.go b/backend/plugins/gitextractor/parser/gitrunner.go index f3bb8e3d6dd..4d547f8a570 100644 --- a/backend/plugins/gitextractor/parser/gitrunner.go +++ b/backend/plugins/gitextractor/parser/gitrunner.go @@ -18,7 +18,6 @@ limitations under the License. package parser import ( - "bufio" "bytes" "context" "fmt" @@ -142,14 +141,3 @@ func splitLines(out []byte) []string { } return strings.Split(text, "\n") } - -func scanLines(reader io.Reader, fn func(line string) error) error { - scanner := bufio.NewScanner(reader) - scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) - for scanner.Scan() { - if err := fn(scanner.Text()); err != nil { - return err - } - } - return scanner.Err() -}