diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go index b2ed4ce688b4..8f8ea6e1780d 100644 --- a/pkg/sources/gitlab/gitlab.go +++ b/pkg/sources/gitlab/gitlab.go @@ -8,6 +8,7 @@ import ( "slices" "strings" "sync" + "time" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" @@ -471,14 +472,25 @@ func (s *Source) newClient() (*gitlab.Client, error) { // Initialize a new api instance. switch s.authMethod { case "OAUTH": - apiClient, err := gitlab.NewOAuthClient(s.token, gitlab.WithBaseURL(s.url)) + apiClient, err := gitlab.NewOAuthClient( + s.token, + gitlab.WithBaseURL(s.url), + gitlab.WithCustomRetryWaitMinMax(time.Second, 5*time.Second), + gitlab.WithCustomRetryMax(3), + ) if err != nil { return nil, fmt.Errorf("could not create Gitlab OAUTH client for %q: %w", s.url, err) } return apiClient, nil case "BASIC_AUTH": - apiClient, err := gitlab.NewBasicAuthClient(s.user, s.password, gitlab.WithBaseURL(s.url)) + apiClient, err := gitlab.NewBasicAuthClient( + s.user, + s.password, + gitlab.WithBaseURL(s.url), + gitlab.WithCustomRetryWaitMinMax(time.Second, 5*time.Second), + gitlab.WithCustomRetryMax(3), + ) if err != nil { return nil, fmt.Errorf("could not create Gitlab BASICAUTH client for %q: %w", s.url, err) } @@ -491,7 +503,12 @@ func (s *Source) newClient() (*gitlab.Client, error) { } fallthrough case "TOKEN": - apiClient, err := gitlab.NewOAuthClient(s.token, gitlab.WithBaseURL(s.url)) + apiClient, err := gitlab.NewOAuthClient( + s.token, + gitlab.WithBaseURL(s.url), + gitlab.WithCustomRetryWaitMinMax(time.Second, 5*time.Second), + gitlab.WithCustomRetryMax(3), + ) if err != nil { return nil, fmt.Errorf("could not create Gitlab TOKEN client for %q: %w", s.url, err) } @@ -711,62 +728,79 @@ func (s *Source) getAllProjectReposV2( "list_options", listOpts, "all_available", *projectQueryOptions.Membership) - // https://pkg.go.dev/gitlab.com/gitlab-org/api/client-go#Scan2 - projectsIter := gitlab.Scan2(func(p gitlab.PaginationOptionFunc) ([]*gitlab.Project, *gitlab.Response, error) { - return apiClient.Projects.ListProjects(projectQueryOptions, p, gitlab.WithContext(ctx)) - }) - + // totalCount tracks the total number of projects processed by this enumeration. + // It includes all projects fetched from the API, even those later skipped by ignore rules. totalCount := 0 - // process each project - for project, projectErr := range projectsIter { - if projectErr != nil { - err := fmt.Errorf("error during project enumeration: %w", projectErr) + requestOptions := []gitlab.RequestOptionFunc{gitlab.WithContext(ctx)} - if reportErr := reporter.UnitErr(ctx, err); reportErr != nil { - return reportErr + // Pagination loop: Continue fetching pages until the API indicates there are no more. + for { + // Fetch a page of projects from the GitLab API using the current query options. + projects, resp, err := apiClient.Projects.ListProjects(projectQueryOptions, requestOptions...) + if err != nil { + err = fmt.Errorf("received error on listing projects, you might not have permissions to do that: %w", err) + if err := reporter.UnitErr(ctx, err); err != nil { + return err } - - continue + // break on error as with error we will not have any response and no next page + break } - totalCount++ + // Log the batch size for debugging and monitoring. + ctx.Logger().V(3).Info("listed projects batch", "batch_size", len(projects), "running_total", totalCount) + // Process each project in the current page. + for _, project := range projects { + projCtx := context.WithValues(ctx, + "project_id", project.ID, + "project_name", project.NameWithNamespace) - projCtx := context.WithValues(ctx, - "project_id", project.ID, - "project_name", project.NameWithNamespace) + totalCount++ - // skip projects configured to be ignored. - if ignoreRepo(project.PathWithNamespace) { - projCtx.Logger().V(3).Info("skipping project", "reason", "ignored in config") + // skip projects configured to be ignored. + if ignoreRepo(project.PathWithNamespace) { + projCtx.Logger().V(3).Info("skipping project", "reason", "ignored in config") - continue - } + continue + } - // report an error if we could not convert the project into a URL. - if _, err := url.Parse(project.HTTPURLToRepo); err != nil { - projCtx.Logger().V(3).Info("skipping project", - "reason", "URL parse failure", - "url", project.HTTPURLToRepo, - "parse_error", err) + // report an error if we could not convert the project into a URL. + if _, err := url.Parse(project.HTTPURLToRepo); err != nil { + projCtx.Logger().V(3).Info("skipping project", + "reason", "URL parse failure", + "url", project.HTTPURLToRepo, + "parse_error", err) - err = fmt.Errorf("could not parse url %q given by project: %w", project.HTTPURLToRepo, err) - if err := reporter.UnitErr(ctx, err); err != nil { - return err + err = fmt.Errorf("could not parse url %q given by project: %w", project.HTTPURLToRepo, err) + if err := reporter.UnitErr(ctx, err); err != nil { + return err + } + + continue } - continue - } + // report the unit. + projCtx.Logger().V(3).Info("accepting project") - // report the unit. - projCtx.Logger().V(3).Info("accepting project") + s.cacheGitlabProject(project) + unit := git.SourceUnit{Kind: git.UnitRepo, ID: project.HTTPURLToRepo} + gitlabReposEnumerated.WithLabelValues(s.name).Inc() - s.cacheGitlabProject(project) - unit := git.SourceUnit{Kind: git.UnitRepo, ID: project.HTTPURLToRepo} - gitlabReposEnumerated.WithLabelValues(s.name).Inc() + if err := reporter.UnitOk(ctx, unit); err != nil { + return err + } + } - if err := reporter.UnitOk(ctx, unit); err != nil { - return err + // if next page is empty, break the loop + if resp == nil || resp.NextLink == "" { + // No more pages to fetch. This is the normal loop exit condition. + // It also acts as a safety stop if the current request failed. + break + } + // Only update the token for the next page if we have a valid, non-empty link. + requestOptions = []gitlab.RequestOptionFunc{ + gitlab.WithContext(ctx), + gitlab.WithKeysetPaginationParameters(resp.NextLink), } }