diff --git a/modules/har/pkg/har/migrate/adapter/har/adapter.go b/modules/har/pkg/har/migrate/adapter/har/adapter.go index 7503a6ec..525d0b75 100644 --- a/modules/har/pkg/har/migrate/adapter/har/adapter.go +++ b/modules/har/pkg/har/migrate/adapter/har/adapter.go @@ -112,12 +112,16 @@ func (a *harAdapter) UploadFile( err = a.client.uploadSwiftFile(registry, f.Name, file, artifactName, version) case types.DART: err = a.client.uploadDartFile(registry, artifactName, version, f, file) - case types.RAW: + case types.RAW, types.CRAN: err = a.client.uploadRawFile(registry, f, file) case types.DEBIAN: err = a.client.uploadDebianFile(registry, f, file, metadata) case types.PUPPET: err = a.client.uploadPuppetFile(registry, f, file) + case types.RUBY: + err = a.client.uploadRubyFile(registry, f, file) + case types.TERRAFORM: + err = a.client.uploadTerraformFile(registry, f, artifactName, version, file) case types.CONAN: err = a.client.uploadConanFile(registry, file, metadata) default: @@ -157,7 +161,7 @@ func (a *harAdapter) VersionExists(ctx context.Context, p types.Package, registr } func (a *harAdapter) FileExists(ctx context.Context, registryRef, pkg, version string, file *types.File, artifactType types.ArtifactType) (bool, error) { - if artifactType == types.RAW { + if artifactType == types.RAW || artifactType == types.CRAN { return a.client.headRawFile(registryRef, file.Uri) } return a.client.artifactFileExists(ctx, registryRef, pkg, version, file, artifactType) diff --git a/modules/har/pkg/har/migrate/adapter/har/adapter_terraform_test.go b/modules/har/pkg/har/migrate/adapter/har/adapter_terraform_test.go new file mode 100644 index 00000000..9eeabbf1 --- /dev/null +++ b/modules/har/pkg/har/migrate/adapter/har/adapter_terraform_test.go @@ -0,0 +1,65 @@ +package har + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/harness/cli/modules/har/pkg/har/migrate/types" + + "github.com/rs/zerolog" +) + +// TestUploadFileTerraformRouting verifies that adapter.UploadFile dispatches +// TERRAFORM to uploadTerraformFile (module and provider paths). +func TestUploadFileTerraformRouting(t *testing.T) { + tests := []struct { + name string + fileName string + pkg string + version string + wantPathHas string + }{ + { + name: "module routed correctly", + fileName: "vpc-1.0.0.tar.gz", + pkg: "hashicorp/vpc/aws", + version: "1.0.0", + wantPathHas: "/hashicorp/vpc/aws/1.0.0", + }, + { + name: "provider routed correctly", + fileName: "terraform-provider-aws_2.0.0_linux_amd64.zip", + pkg: "hashicorp/aws", + version: "2.0.0", + wantPathHas: "/hashicorp/aws/2.0.0", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + w.WriteHeader(http.StatusCreated) + })) + defer srv.Close() + + c := newTestClient(t, srv.URL) + a := &harAdapter{client: c, logger: zerolog.Nop()} + + f := &types.File{Name: tt.fileName, Uri: "/" + tt.fileName} + body := io.NopCloser(strings.NewReader("bytes")) + + err := a.UploadFile("reg1", body, f, http.Header{}, tt.pkg, tt.version, types.TERRAFORM, nil) + if err != nil { + t.Fatalf("UploadFile error: %v", err) + } + if !strings.Contains(gotPath, tt.wantPathHas) { + t.Errorf("path = %q, want to contain %q", gotPath, tt.wantPathHas) + } + }) + } +} diff --git a/modules/har/pkg/har/migrate/adapter/har/arpkg/client_gen.go b/modules/har/pkg/har/migrate/adapter/har/arpkg/client_gen.go index dce9dcc8..4c50b04d 100644 --- a/modules/har/pkg/har/migrate/adapter/har/arpkg/client_gen.go +++ b/modules/har/pkg/har/migrate/adapter/har/arpkg/client_gen.go @@ -238,8 +238,17 @@ type ClientInterface interface { // UploadRpmPackageWithBody request with any body UploadRpmPackageWithBody(ctx context.Context, accountId string, registry string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + // UploadRubyPackageWithBody request with any body + UploadRubyPackageWithBody(ctx context.Context, accountId string, registry string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + // UploadSwiftPackageWithBody request with any body UploadSwiftPackageWithBody(ctx context.Context, accountId string, registry string, scope string, name string, version string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + // UploadTerraformModuleWithBody request with any body + UploadTerraformModuleWithBody(ctx context.Context, accountId string, registry string, namespace string, name string, provider string, version string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + // UploadTerraformProviderWithBody request with any body + UploadTerraformProviderWithBody(ctx context.Context, accountId string, registry string, namespace string, pType string, version string, filename string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) } func (c *Client) UploadCargoPackageWithBody(ctx context.Context, accountId string, registry string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { @@ -470,6 +479,18 @@ func (c *Client) UploadRpmPackageWithBody(ctx context.Context, accountId string, return c.Client.Do(req) } +func (c *Client) UploadRubyPackageWithBody(ctx context.Context, accountId string, registry string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewUploadRubyPackageRequestWithBody(c.Server, accountId, registry, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) UploadSwiftPackageWithBody(ctx context.Context, accountId string, registry string, scope string, name string, version string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewUploadSwiftPackageRequestWithBody(c.Server, accountId, registry, scope, name, version, contentType, body) if err != nil { @@ -482,6 +503,30 @@ func (c *Client) UploadSwiftPackageWithBody(ctx context.Context, accountId strin return c.Client.Do(req) } +func (c *Client) UploadTerraformModuleWithBody(ctx context.Context, accountId string, registry string, namespace string, name string, provider string, version string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewUploadTerraformModuleRequestWithBody(c.Server, accountId, registry, namespace, name, provider, version, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) UploadTerraformProviderWithBody(ctx context.Context, accountId string, registry string, namespace string, pType string, version string, filename string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewUploadTerraformProviderRequestWithBody(c.Server, accountId, registry, namespace, pType, version, filename, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + // NewUploadCargoPackageRequestWithBody generates requests for UploadCargoPackage with any type of body func NewUploadCargoPackageRequestWithBody(server string, accountId string, registry string, contentType string, body io.Reader) (*http.Request, error) { var err error @@ -1452,6 +1497,49 @@ func NewUploadRpmPackageRequestWithBody(server string, accountId string, registr return req, nil } +// NewUploadRubyPackageRequestWithBody generates requests for UploadRubyPackage with any type of body +func NewUploadRubyPackageRequestWithBody(server string, accountId string, registry string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "accountId", runtime.ParamLocationPath, accountId) + if err != nil { + return nil, err + } + + var pathParam1 string + + pathParam1, err = runtime.StyleParamWithLocation("simple", false, "registry", runtime.ParamLocationPath, registry) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/pkg/%s/%s/ruby/api/v1/gems", pathParam0, pathParam1) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + // NewUploadSwiftPackageRequestWithBody generates requests for UploadSwiftPackage with any type of body func NewUploadSwiftPackageRequestWithBody(server string, accountId string, registry string, scope string, name string, version string, contentType string, body io.Reader) (*http.Request, error) { var err error @@ -1516,6 +1604,148 @@ func NewUploadSwiftPackageRequestWithBody(server string, accountId string, regis return req, nil } +// NewUploadTerraformModuleRequestWithBody generates requests for UploadTerraformModule with any type of body +func NewUploadTerraformModuleRequestWithBody(server string, accountId string, registry string, namespace string, name string, provider string, version string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "accountId", runtime.ParamLocationPath, accountId) + if err != nil { + return nil, err + } + + var pathParam1 string + + pathParam1, err = runtime.StyleParamWithLocation("simple", false, "registry", runtime.ParamLocationPath, registry) + if err != nil { + return nil, err + } + + var pathParam2 string + + pathParam2, err = runtime.StyleParamWithLocation("simple", false, "namespace", runtime.ParamLocationPath, namespace) + if err != nil { + return nil, err + } + + var pathParam3 string + + pathParam3, err = runtime.StyleParamWithLocation("simple", false, "name", runtime.ParamLocationPath, name) + if err != nil { + return nil, err + } + + var pathParam4 string + + pathParam4, err = runtime.StyleParamWithLocation("simple", false, "provider", runtime.ParamLocationPath, provider) + if err != nil { + return nil, err + } + + var pathParam5 string + + pathParam5, err = runtime.StyleParamWithLocation("simple", false, "version", runtime.ParamLocationPath, version) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/pkg/%s/%s/terraform/v1/modules/%s/%s/%s/%s", pathParam0, pathParam1, pathParam2, pathParam3, pathParam4, pathParam5) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("PUT", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + +// NewUploadTerraformProviderRequestWithBody generates requests for UploadTerraformProvider with any type of body +func NewUploadTerraformProviderRequestWithBody(server string, accountId string, registry string, namespace string, pType string, version string, filename string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "accountId", runtime.ParamLocationPath, accountId) + if err != nil { + return nil, err + } + + var pathParam1 string + + pathParam1, err = runtime.StyleParamWithLocation("simple", false, "registry", runtime.ParamLocationPath, registry) + if err != nil { + return nil, err + } + + var pathParam2 string + + pathParam2, err = runtime.StyleParamWithLocation("simple", false, "namespace", runtime.ParamLocationPath, namespace) + if err != nil { + return nil, err + } + + var pathParam3 string + + pathParam3, err = runtime.StyleParamWithLocation("simple", false, "type", runtime.ParamLocationPath, pType) + if err != nil { + return nil, err + } + + var pathParam4 string + + pathParam4, err = runtime.StyleParamWithLocation("simple", false, "version", runtime.ParamLocationPath, version) + if err != nil { + return nil, err + } + + var pathParam5 string + + pathParam5, err = runtime.StyleParamWithLocation("simple", false, "filename", runtime.ParamLocationPath, filename) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/pkg/%s/%s/terraform/v1/providers/%s/%s/%s/%s", pathParam0, pathParam1, pathParam2, pathParam3, pathParam4, pathParam5) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("PUT", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + func (c *Client) applyEditors(ctx context.Context, req *http.Request, additionalEditors []RequestEditorFn) error { for _, r := range c.RequestEditors { if err := r(ctx, req); err != nil { @@ -1616,8 +1846,17 @@ type ClientWithResponsesInterface interface { // UploadRpmPackageWithBodyWithResponse request with any body UploadRpmPackageWithBodyWithResponse(ctx context.Context, accountId string, registry string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UploadRpmPackageResp, error) + // UploadRubyPackageWithBodyWithResponse request with any body + UploadRubyPackageWithBodyWithResponse(ctx context.Context, accountId string, registry string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UploadRubyPackageResp, error) + // UploadSwiftPackageWithBodyWithResponse request with any body UploadSwiftPackageWithBodyWithResponse(ctx context.Context, accountId string, registry string, scope string, name string, version string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UploadSwiftPackageResp, error) + + // UploadTerraformModuleWithBodyWithResponse request with any body + UploadTerraformModuleWithBodyWithResponse(ctx context.Context, accountId string, registry string, namespace string, name string, provider string, version string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UploadTerraformModuleResp, error) + + // UploadTerraformProviderWithBodyWithResponse request with any body + UploadTerraformProviderWithBodyWithResponse(ctx context.Context, accountId string, registry string, namespace string, pType string, version string, filename string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UploadTerraformProviderResp, error) } type UploadCargoPackageResp struct { @@ -2019,6 +2258,27 @@ func (r UploadRpmPackageResp) StatusCode() int { return 0 } +type UploadRubyPackageResp struct { + Body []byte + HTTPResponse *http.Response +} + +// Status returns HTTPResponse.Status +func (r UploadRubyPackageResp) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r UploadRubyPackageResp) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type UploadSwiftPackageResp struct { Body []byte HTTPResponse *http.Response @@ -2040,6 +2300,48 @@ func (r UploadSwiftPackageResp) StatusCode() int { return 0 } +type UploadTerraformModuleResp struct { + Body []byte + HTTPResponse *http.Response +} + +// Status returns HTTPResponse.Status +func (r UploadTerraformModuleResp) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r UploadTerraformModuleResp) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +type UploadTerraformProviderResp struct { + Body []byte + HTTPResponse *http.Response +} + +// Status returns HTTPResponse.Status +func (r UploadTerraformProviderResp) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r UploadTerraformProviderResp) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + // UploadCargoPackageWithBodyWithResponse request with arbitrary body returning *UploadCargoPackageResp func (c *ClientWithResponses) UploadCargoPackageWithBodyWithResponse(ctx context.Context, accountId string, registry string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UploadCargoPackageResp, error) { rsp, err := c.UploadCargoPackageWithBody(ctx, accountId, registry, contentType, body, reqEditors...) @@ -2211,6 +2513,15 @@ func (c *ClientWithResponses) UploadRpmPackageWithBodyWithResponse(ctx context.C return ParseUploadRpmPackageResp(rsp) } +// UploadRubyPackageWithBodyWithResponse request with arbitrary body returning *UploadRubyPackageResp +func (c *ClientWithResponses) UploadRubyPackageWithBodyWithResponse(ctx context.Context, accountId string, registry string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UploadRubyPackageResp, error) { + rsp, err := c.UploadRubyPackageWithBody(ctx, accountId, registry, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseUploadRubyPackageResp(rsp) +} + // UploadSwiftPackageWithBodyWithResponse request with arbitrary body returning *UploadSwiftPackageResp func (c *ClientWithResponses) UploadSwiftPackageWithBodyWithResponse(ctx context.Context, accountId string, registry string, scope string, name string, version string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UploadSwiftPackageResp, error) { rsp, err := c.UploadSwiftPackageWithBody(ctx, accountId, registry, scope, name, version, contentType, body, reqEditors...) @@ -2220,6 +2531,24 @@ func (c *ClientWithResponses) UploadSwiftPackageWithBodyWithResponse(ctx context return ParseUploadSwiftPackageResp(rsp) } +// UploadTerraformModuleWithBodyWithResponse request with arbitrary body returning *UploadTerraformModuleResp +func (c *ClientWithResponses) UploadTerraformModuleWithBodyWithResponse(ctx context.Context, accountId string, registry string, namespace string, name string, provider string, version string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UploadTerraformModuleResp, error) { + rsp, err := c.UploadTerraformModuleWithBody(ctx, accountId, registry, namespace, name, provider, version, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseUploadTerraformModuleResp(rsp) +} + +// UploadTerraformProviderWithBodyWithResponse request with arbitrary body returning *UploadTerraformProviderResp +func (c *ClientWithResponses) UploadTerraformProviderWithBodyWithResponse(ctx context.Context, accountId string, registry string, namespace string, pType string, version string, filename string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UploadTerraformProviderResp, error) { + rsp, err := c.UploadTerraformProviderWithBody(ctx, accountId, registry, namespace, pType, version, filename, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseUploadTerraformProviderResp(rsp) +} + // ParseUploadCargoPackageResp parses an HTTP response from a UploadCargoPackageWithResponse call func ParseUploadCargoPackageResp(rsp *http.Response) (*UploadCargoPackageResp, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -2524,6 +2853,22 @@ func ParseUploadRpmPackageResp(rsp *http.Response) (*UploadRpmPackageResp, error return response, nil } +// ParseUploadRubyPackageResp parses an HTTP response from a UploadRubyPackageWithResponse call +func ParseUploadRubyPackageResp(rsp *http.Response) (*UploadRubyPackageResp, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &UploadRubyPackageResp{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + return response, nil +} + // ParseUploadSwiftPackageResp parses an HTTP response from a UploadSwiftPackageWithResponse call func ParseUploadSwiftPackageResp(rsp *http.Response) (*UploadSwiftPackageResp, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -2539,3 +2884,35 @@ func ParseUploadSwiftPackageResp(rsp *http.Response) (*UploadSwiftPackageResp, e return response, nil } + +// ParseUploadTerraformModuleResp parses an HTTP response from a UploadTerraformModuleWithResponse call +func ParseUploadTerraformModuleResp(rsp *http.Response) (*UploadTerraformModuleResp, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &UploadTerraformModuleResp{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + return response, nil +} + +// ParseUploadTerraformProviderResp parses an HTTP response from a UploadTerraformProviderWithResponse call +func ParseUploadTerraformProviderResp(rsp *http.Response) (*UploadTerraformProviderResp, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &UploadTerraformProviderResp{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + return response, nil +} diff --git a/modules/har/pkg/har/migrate/adapter/har/client.go b/modules/har/pkg/har/migrate/adapter/har/client.go index a2e1f53e..179084c6 100644 --- a/modules/har/pkg/har/migrate/adapter/har/client.go +++ b/modules/har/pkg/har/migrate/adapter/har/client.go @@ -1051,6 +1051,114 @@ func (c *client) uploadPuppetFile( return nil } +// uploadRubyFile streams a Ruby gem (.gem) to HAR via the Ruby push API. +// Do not send checksum headers; the server generates sidecar metadata on push. +func (c *client) uploadRubyFile( + registry string, + f *types.File, + file io.ReadCloser, +) error { + if c.pkgClient == nil { + return fmt.Errorf("ruby upload: pkg client is not configured") + } + + resp, err := c.pkgClient.UploadRubyPackageWithBodyWithResponse( + context.Background(), + c.accountID, + registry, + "application/octet-stream", + file, + ) + if err != nil { + return fmt.Errorf("failed to upload Ruby gem '%s': %w", f.Name, err) + } + + switch resp.StatusCode() { + case http2.StatusOK, http2.StatusCreated: + return nil + case http2.StatusConflict: + return types.ErrArtifactAlreadyExists + default: + return fmt.Errorf("failed to upload Ruby gem '%s', status code: %d, response: %s", + f.Name, resp.StatusCode(), string(resp.Body)) + } +} + +// uploadTerraformFile routes a Terraform file to the correct HAR endpoint. +// Modules (.tar.gz/.tgz) go to PUT /terraform/v1/modules/{ns}/{name}/{provider}/{ver}. +// Providers (.zip) go to PUT /terraform/v1/providers/{ns}/{type}/{ver}/{filename}. +// The pkg argument is "ns/name/provider" for modules or "ns/type" for providers. +func (c *client) uploadTerraformFile( + registry string, + f *types.File, + pkg string, + version string, + file io.ReadCloser, +) error { + ctx := context.Background() + lower := strings.ToLower(f.Name) + + // Route by pkg segment count: modules have "ns/name/provider" (3 parts), + // providers have "ns/type" (2 parts). This handles both Layout A (.tar.gz/.tgz) + // and Layout B (.zip) modules unambiguously. + pkgParts := strings.SplitN(pkg, "/", 3) + isModule := len(pkgParts) == 3 + + if isModule && (strings.HasSuffix(lower, ".tar.gz") || strings.HasSuffix(lower, ".tgz") || strings.HasSuffix(lower, ".zip")) { + // Module upload: pkg = "ns/name/provider" + ns, name, provider := pkgParts[0], pkgParts[1], pkgParts[2] + resp, err := c.pkgClient.UploadTerraformModuleWithBodyWithResponse( + ctx, + c.accountID, + registry, + ns, name, provider, version, + "application/octet-stream", + file, + ) + if err != nil { + return fmt.Errorf("failed to upload Terraform module '%s': %w", f.Name, err) + } + if resp.StatusCode() == http2.StatusConflict { + return types.ErrArtifactAlreadyExists + } + if resp.StatusCode() < 200 || resp.StatusCode() > 299 { + return fmt.Errorf("failed to upload Terraform module '%s', status: %d, response: %s", + f.Name, resp.StatusCode(), string(resp.Body)) + } + return nil + } + + if !isModule && strings.HasSuffix(lower, ".zip") { + // Provider upload: pkg = "ns/type" + parts := strings.SplitN(pkg, "/", 2) + if len(parts) != 2 { + return fmt.Errorf("terraform provider package name must be ns/type, got: %s", pkg) + } + ns, typeName := parts[0], parts[1] + resp, err := c.pkgClient.UploadTerraformProviderWithBodyWithResponse( + ctx, + c.accountID, + registry, + ns, typeName, version, f.Name, + "application/octet-stream", + file, + ) + if err != nil { + return fmt.Errorf("failed to upload Terraform provider '%s': %w", f.Name, err) + } + if resp.StatusCode() == http2.StatusConflict { + return types.ErrArtifactAlreadyExists + } + if resp.StatusCode() < 200 || resp.StatusCode() > 299 { + return fmt.Errorf("failed to upload Terraform provider '%s', status: %d, response: %s", + f.Name, resp.StatusCode(), string(resp.Body)) + } + return nil + } + + return fmt.Errorf("unsupported Terraform file extension for '%s': must be .tar.gz, .tgz, or .zip", f.Name) +} + func (c *client) uploadConanFile( registry string, file io.ReadCloser, diff --git a/modules/har/pkg/har/migrate/adapter/har/client_test.go b/modules/har/pkg/har/migrate/adapter/har/client_test.go new file mode 100644 index 00000000..209a52c8 --- /dev/null +++ b/modules/har/pkg/har/migrate/adapter/har/client_test.go @@ -0,0 +1,130 @@ +package har + +import ( + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/harness/cli/modules/har/pkg/har/migrate/adapter/har/arpkg" + "github.com/harness/cli/modules/har/pkg/har/migrate/types" +) + +// newTestClient builds a har client whose generated pkg client points at the +// given test server URL. +func newTestClient(t *testing.T, serverURL string) *client { + t.Helper() + pc, err := arpkg.NewClientWithResponses(serverURL) + if err != nil { + t.Fatalf("new pkg client: %v", err) + } + return &client{pkgClient: pc, url: serverURL, accountID: "acct1"} +} + +// TestUploadTerraformFile covers module (.tar.gz) and provider (.zip) upload paths +// including conflict (409 → ErrArtifactAlreadyExists) and non-2xx error handling. +func TestUploadTerraformFile(t *testing.T) { + tests := []struct { + name string + fileName string + pkg string + version string + status int + wantConflict bool + wantErr bool + wantPathHas string + }{ + { + name: "module upload success", + fileName: "vpc-1.0.0.tar.gz", pkg: "hashicorp/vpc/aws", version: "1.0.0", + status: http.StatusCreated, wantPathHas: "/hashicorp/vpc/aws/1.0.0", + }, + { + name: "module upload tgz extension", + fileName: "vpc-1.0.0.tgz", pkg: "hashicorp/vpc/aws", version: "1.0.0", + status: http.StatusCreated, wantPathHas: "/hashicorp/vpc/aws/1.0.0", + }, + { + name: "module upload conflict", + fileName: "vpc-1.0.0.tar.gz", pkg: "hashicorp/vpc/aws", version: "1.0.0", + status: http.StatusConflict, wantConflict: true, + }, + { + name: "module upload error", + fileName: "vpc-1.0.0.tar.gz", pkg: "hashicorp/vpc/aws", version: "1.0.0", + status: http.StatusBadRequest, wantErr: true, + }, + { + name: "provider upload success", + fileName: "terraform-provider-aws_2.0.0_linux_amd64.zip", pkg: "hashicorp/aws", version: "2.0.0", + status: http.StatusCreated, wantPathHas: "/hashicorp/aws/2.0.0", + }, + { + name: "provider upload conflict", + fileName: "terraform-provider-aws_2.0.0_linux_amd64.zip", pkg: "hashicorp/aws", version: "2.0.0", + status: http.StatusConflict, wantConflict: true, + }, + { + name: "provider upload error", + fileName: "terraform-provider-aws_2.0.0_linux_amd64.zip", pkg: "hashicorp/aws", version: "2.0.0", + status: http.StatusInternalServerError, wantErr: true, + }, + { + name: "unsupported extension", + fileName: "terraform-something.txt", pkg: "hashicorp/aws", version: "2.0.0", + status: http.StatusCreated, wantErr: true, + }, + { + name: "module bad pkg format", + fileName: "vpc-1.0.0.tar.gz", pkg: "hashicorp", version: "1.0.0", + status: http.StatusCreated, wantErr: true, + }, + { + name: "provider bad pkg format", + fileName: "terraform-provider-aws_2.0.0_linux_amd64.zip", pkg: "hashicorp", version: "2.0.0", + status: http.StatusCreated, wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + if tt.status >= 400 { + w.WriteHeader(tt.status) + _, _ = w.Write([]byte("rejected")) + return + } + w.WriteHeader(tt.status) + })) + defer srv.Close() + + c := newTestClient(t, srv.URL) + f := &types.File{Name: tt.fileName, Uri: "/" + tt.fileName} + body := io.NopCloser(strings.NewReader("bytes")) + + err := c.uploadTerraformFile("reg1", f, tt.pkg, tt.version, body) + + switch { + case tt.wantConflict: + if !errors.Is(err, types.ErrArtifactAlreadyExists) { + t.Fatalf("expected ErrArtifactAlreadyExists, got %v", err) + } + case tt.wantErr: + if err == nil { + t.Fatalf("expected error, got nil") + } + default: + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tt.wantPathHas != "" && !strings.Contains(gotPath, tt.wantPathHas) { + t.Errorf("path = %q, want to contain %q", gotPath, tt.wantPathHas) + } + } + }) + } +} diff --git a/modules/har/pkg/har/migrate/adapter/jfrog/adapter.go b/modules/har/pkg/har/migrate/adapter/jfrog/adapter.go index 4d0b7c7f..938eb3c2 100644 --- a/modules/har/pkg/har/migrate/adapter/jfrog/adapter.go +++ b/modules/har/pkg/har/migrate/adapter/jfrog/adapter.go @@ -457,6 +457,39 @@ func (a *adapter) GetPackages(registry string, artifactType types.ArtifactType, } log.Info().Msgf("Found %d PUPPET packages", len(packages)) return packages, nil + } else if artifactType == types.RUBY { + // Ruby gems in JFrog may live at any path depth; identity comes from the + // .gem filename ({name}-{version}.gem or {name}-{version}-{platform}.gem). + files, err := tree.GetAllFiles(root) + if err != nil { + return nil, fmt.Errorf("get all files: %w", err) + } + + pkgMap := make(map[string]bool) + for _, file := range files { + if file.Folder { + continue + } + if !strings.HasSuffix(file.Name, ".gem") { + continue + } + meta, ok := util.ParseRubyGemFileNameWithPath(file.Uri) + if !ok { + continue + } + pkgMap[meta.Name] = true + } + + for pkgName := range pkgMap { + packages = append(packages, types.Package{ + Registry: registry, + Path: "/", + Name: pkgName, + Size: -1, + }) + } + log.Info().Msgf("Found %d RUBY packages", len(packages)) + return packages, nil } else if artifactType == types.CONAN { files, err := tree.GetAllFiles(root) if err != nil { @@ -465,6 +498,43 @@ func (a *adapter) GetPackages(registry string, artifactType types.ArtifactType, packages = append(packages, util.GetConanPackages(files, registry)...) log.Info().Msgf("Found %d CONAN packages", len(packages)) return packages, nil + } else if artifactType == types.TERRAFORM { + // Terraform modules: /////-.tar.gz + // Terraform providers: ////terraform-provider-__.zip + // One logical package per (ns/name/provider) for modules and (ns/type) for providers. + files, err := tree.GetAllFiles(root) + if err != nil { + return nil, fmt.Errorf("get all files: %w", err) + } + + pkgMap := make(map[string]bool) + for _, file := range files { + if file.Folder { + continue + } + if util.IsTerraformModule(file.Uri) { + ns, name, provider, _, ok := util.ParseTerraformModulePath(file.Uri) + if ok { + pkgMap[ns+"/"+name+"/"+provider] = true + } + } else if util.IsTerraformProvider(file.Uri) { + ns, typeName, _, _, _, _, ok := util.ParseTerraformProviderPath(file.Uri) + if ok { + pkgMap[ns+"/"+typeName] = true + } + } + } + + for pkgName := range pkgMap { + packages = append(packages, types.Package{ + Registry: registry, + Path: "/", + Name: pkgName, + Size: -1, + }) + } + log.Info().Msgf("Found %d TERRAFORM packages", len(packages)) + return packages, nil } else if artifactType == types.HELM_HTTP { seenPath := make(map[string]bool) files, err := tree.GetAllFiles(root) @@ -589,6 +659,41 @@ func (a *adapter) GetPackages(registry string, artifactType types.ArtifactType, } } return packages, nil + } else if artifactType == types.CRAN { + files, err := tree.GetAllFiles(root) + if err != nil { + return nil, fmt.Errorf("get all files: %w", err) + } + + pkgMap := make(map[string]bool) + skippedUnrecognized := 0 + for _, file := range files { + if file.Folder { + continue + } + if util.IsCranIndexFile(file.Uri) { + continue + } + pkgName, _, ok := util.ParseCranFileNameWithPath(file.Uri) + if !ok { + skippedUnrecognized++ + continue + } + pkgMap[pkgName] = true + } + + for pkgName := range pkgMap { + packages = append(packages, types.Package{ + Registry: registry, + Path: "/", + Name: pkgName, + Size: -1, + }) + } + if skippedUnrecognized > 0 { + log.Warn().Msgf("Skipped %d unrecognized CRAN paths in %s", skippedUnrecognized, registry) + } + log.Info().Msgf("Found %d CRAN packages", len(packages)) } else { return []types.Package{}, errors.New("unknown artifact type") } @@ -1044,6 +1149,79 @@ func (a *adapter) GetVersions( return versions, nil } + if artifactType == types.RUBY { + files, err := tree.GetAllFiles(node) + if err != nil { + return nil, fmt.Errorf("get all files: %w", err) + } + + versionMap := make(map[string]bool) + for _, file := range files { + if file.Folder { + continue + } + if !strings.HasSuffix(file.Name, ".gem") { + continue + } + meta, ok := util.ParseRubyGemFileNameWithPath(file.Uri) + if !ok || meta.Name != pkg { + continue + } + versionMap[meta.Version] = true + } + + var versions []types.Version + for version := range versionMap { + versions = append(versions, types.Version{ + Registry: registry, + Pkg: pkg, + Path: "/", + Name: version, + Size: -1, + }) + } + log.Info().Msgf("Found %d versions for RUBY package %s", len(versions), pkg) + return versions, nil + } + + if artifactType == types.TERRAFORM { + files, err := tree.GetAllFiles(node) + if err != nil { + return nil, fmt.Errorf("get all files: %w", err) + } + + versionMap := make(map[string]bool) + for _, file := range files { + if file.Folder { + continue + } + if util.IsTerraformModule(file.Uri) { + ns, name, provider, version, ok := util.ParseTerraformModulePath(file.Uri) + if ok && ns+"/"+name+"/"+provider == pkg { + versionMap[version] = true + } + } else if util.IsTerraformProvider(file.Uri) { + ns, typeName, version, _, _, _, ok := util.ParseTerraformProviderPath(file.Uri) + if ok && ns+"/"+typeName == pkg { + versionMap[version] = true + } + } + } + + var versions []types.Version + for version := range versionMap { + versions = append(versions, types.Version{ + Registry: registry, + Pkg: pkg, + Path: "/", + Name: version, + Size: -1, + }) + } + log.Info().Msgf("Found %d versions for TERRAFORM package %s", len(versions), pkg) + return versions, nil + } + if artifactType == types.PYTHON { var versions []types.Version indexPath := fmt.Sprintf(".pypi/%s/%s.html", pkg, pkg) diff --git a/modules/har/pkg/har/migrate/adapter/jfrog/adapter_cran_test.go b/modules/har/pkg/har/migrate/adapter/jfrog/adapter_cran_test.go new file mode 100644 index 00000000..e7fecc89 --- /dev/null +++ b/modules/har/pkg/har/migrate/adapter/jfrog/adapter_cran_test.go @@ -0,0 +1,101 @@ +package jfrog + +import ( + "testing" + + "github.com/harness/cli/modules/har/pkg/har/migrate/tree" + "github.com/harness/cli/modules/har/pkg/har/migrate/types" +) + +func makeCranFiles(uris ...string) []types.File { + files := make([]types.File, len(uris)) + for i, uri := range uris { + parts := splitCranPath(uri) + name := parts[len(parts)-1] + files[i] = types.File{Name: name, Uri: uri, Folder: false} + } + return files +} + +func splitCranPath(uri string) []string { + var parts []string + cur := "" + for _, c := range uri { + if c == '/' { + if cur != "" { + parts = append(parts, cur) + cur = "" + } + } else { + cur += string(c) + } + } + if cur != "" { + parts = append(parts, cur) + } + return parts +} + +func TestGetPackagesCRANDirect(t *testing.T) { + files := makeCranFiles( + "/src/contrib/jsonlite_1.8.0.tar.gz", + "/src/contrib/Archive/jsonlite/1.7.0/jsonlite_1.7.0.tar.gz", + "/src/contrib/PACKAGES", + "/src/contrib/PACKAGES.gz", + "/bin/windows/contrib/4.4/jsonlite_1.8.0.zip", + "/bin/big-sur-arm64/contrib/4.4/data.table_1.14.0.tgz", + "/unrelated/readme.txt", + ) + root := tree.TransformToTree(files) + + a := &adapter{} + pkgs, err := a.GetPackages("cran-local", types.CRAN, root) + if err != nil { + t.Fatalf("GetPackages: %v", err) + } + + byName := make(map[string]bool) + for _, p := range pkgs { + byName[p.Name] = true + if p.Name == "PACKAGES" || p.Name == "PACKAGES.gz" { + t.Errorf("index file enumerated as package: %+v", p) + } + } + if len(byName) != 2 { + t.Fatalf("expected 2 packages, got %d: %v", len(byName), byName) + } + for _, name := range []string{"jsonlite", "data.table"} { + if !byName[name] { + t.Errorf("missing package %q", name) + } + } +} + +func TestGetPackagesCRANEmpty(t *testing.T) { + root := tree.TransformToTree(nil) + a := &adapter{} + pkgs, err := a.GetPackages("cran-local", types.CRAN, root) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(pkgs) != 0 { + t.Errorf("expected 0 packages, got %v", pkgs) + } +} + +func TestGetPackagesCRANSkipsFoldersAndIndexes(t *testing.T) { + files := []types.File{ + {Name: "contrib", Uri: "/src/contrib", Folder: true}, + {Name: "PACKAGES", Uri: "/src/contrib/PACKAGES", Folder: false}, + {Name: "jsonlite_1.8.0.tar.gz", Uri: "/src/contrib/jsonlite_1.8.0.tar.gz", Folder: false}, + } + root := tree.TransformToTree(files) + a := &adapter{} + pkgs, err := a.GetPackages("cran-local", types.CRAN, root) + if err != nil { + t.Fatalf("GetPackages: %v", err) + } + if len(pkgs) != 1 || pkgs[0].Name != "jsonlite" { + t.Errorf("got %v, want [jsonlite]", pkgs) + } +} diff --git a/modules/har/pkg/har/migrate/adapter/jfrog/adapter_ruby_test.go b/modules/har/pkg/har/migrate/adapter/jfrog/adapter_ruby_test.go new file mode 100644 index 00000000..0868ac9a --- /dev/null +++ b/modules/har/pkg/har/migrate/adapter/jfrog/adapter_ruby_test.go @@ -0,0 +1,77 @@ +package jfrog + +import ( + "testing" + + "github.com/harness/cli/modules/har/pkg/har/migrate/tree" + "github.com/harness/cli/modules/har/pkg/har/migrate/types" +) + +func rubyTestAdapter() *adapter { + return &adapter{reg: types.RegistryConfig{Type: types.JFROG}} +} + +func rubyTestTree() *types.TreeNode { + files := []types.File{ + {Registry: "gems-local", Name: "rails-8.0.2.gem", Uri: "/gems/rails-8.0.2.gem", Size: 1024}, + {Registry: "gems-local", Name: "rails-8.0.2-x86_64-linux.gem", Uri: "/gems/rails-8.0.2-x86_64-linux.gem", Size: 2048}, + {Registry: "gems-local", Name: "nokogiri-1.15.0.gem", Uri: "/gems/nokogiri-1.15.0.gem", Size: 4096}, + {Registry: "gems-local", Name: "rails-8.0.2.gem.md5", Uri: "/gems/rails-8.0.2.gem.md5", Size: 32}, + } + return tree.TransformToTree(files) +} + +func TestGetPackagesRubyDedupesGemNames(t *testing.T) { + const registry = "gems-local" + a := rubyTestAdapter() + root := rubyTestTree() + + pkgs, err := a.GetPackages(registry, types.RUBY, root) + if err != nil { + t.Fatalf("GetPackages: %v", err) + } + + want := map[string]bool{"rails": true, "nokogiri": true} + if len(pkgs) != len(want) { + t.Fatalf("expected %d packages, got %d: %+v", len(want), len(pkgs), pkgs) + } + for _, p := range pkgs { + if !want[p.Name] { + t.Errorf("unexpected package %q", p.Name) + } + if p.Path != "/" || p.Registry != registry { + t.Errorf("package %q metadata = %+v", p.Name, p) + } + } +} + +func TestGetVersionsRubyGroupsPlatformVariants(t *testing.T) { + const registry = "gems-local" + a := rubyTestAdapter() + root := rubyTestTree() + + pkg := types.Package{Registry: registry, Name: "rails", Path: "/"} + versions, err := a.GetVersions(pkg, root, registry, pkg.Name, types.RUBY) + if err != nil { + t.Fatalf("GetVersions: %v", err) + } + if len(versions) != 1 { + t.Fatalf("expected 1 version for rails (both platform variants), got %d: %+v", len(versions), versions) + } + if versions[0].Name != "8.0.2" { + t.Fatalf("version name = %q, want 8.0.2", versions[0].Name) + } +} + +func TestGetVersionsRubyUnknownPackage(t *testing.T) { + a := rubyTestAdapter() + root := rubyTestTree() + pkg := types.Package{Name: "missing-gem"} + versions, err := a.GetVersions(pkg, root, "gems-local", pkg.Name, types.RUBY) + if err != nil { + t.Fatalf("GetVersions: %v", err) + } + if len(versions) != 0 { + t.Fatalf("expected 0 versions, got %+v", versions) + } +} diff --git a/modules/har/pkg/har/migrate/adapter/jfrog/adapter_terraform_test.go b/modules/har/pkg/har/migrate/adapter/jfrog/adapter_terraform_test.go new file mode 100644 index 00000000..3abab1a0 --- /dev/null +++ b/modules/har/pkg/har/migrate/adapter/jfrog/adapter_terraform_test.go @@ -0,0 +1,167 @@ +package jfrog + +import ( + "testing" + + "github.com/harness/cli/modules/har/pkg/har/migrate/tree" + "github.com/harness/cli/modules/har/pkg/har/migrate/types" +) + +func makeFiles(uris ...string) []types.File { + files := make([]types.File, len(uris)) + for i, uri := range uris { + parts := splitPath(uri) + name := parts[len(parts)-1] + files[i] = types.File{Name: name, Uri: uri, Folder: false} + } + return files +} + +func splitPath(uri string) []string { + var parts []string + cur := "" + for _, c := range uri { + if c == '/' { + if cur != "" { + parts = append(parts, cur) + cur = "" + } + } else { + cur += string(c) + } + } + if cur != "" { + parts = append(parts, cur) + } + return parts +} + +func TestGetPackagesTerraformDirect(t *testing.T) { + files := makeFiles( + "/hashicorp/vpc/aws/1.0.0/vpc-1.0.0.tar.gz", + "/hashicorp/vpc/aws/1.1.0/vpc-1.1.0.tar.gz", + "/hashicorp/aws/2.0.0/terraform-provider-aws_2.0.0_linux_amd64.zip", + "/hashicorp/aws/2.0.0/terraform-provider-aws_2.0.0_darwin_arm64.zip", + ) + root := tree.TransformToTree(files) + + a := &adapter{} + pkgs, err := a.GetPackages("terraform-local", types.TERRAFORM, root) + if err != nil { + t.Fatalf("GetPackages: %v", err) + } + + pkgSet := make(map[string]bool) + for _, p := range pkgs { + pkgSet[p.Name] = true + } + + expected := map[string]bool{ + "hashicorp/vpc/aws": true, + "hashicorp/aws": true, + } + if len(pkgSet) != len(expected) { + t.Fatalf("got %d packages %v, want %d", len(pkgSet), pkgSet, len(expected)) + } + for k := range expected { + if !pkgSet[k] { + t.Errorf("missing package %q", k) + } + } +} + +func TestGetPackagesTerraformDeduplicates(t *testing.T) { + // Two files for the same module package — should still emit one package. + files := makeFiles( + "/hashicorp/vpc/aws/1.0.0/vpc-1.0.0.tar.gz", + "/hashicorp/vpc/aws/1.1.0/vpc-1.1.0.tar.gz", + ) + root := tree.TransformToTree(files) + + a := &adapter{} + pkgs, err := a.GetPackages("terraform-local", types.TERRAFORM, root) + if err != nil { + t.Fatalf("GetPackages: %v", err) + } + if len(pkgs) != 1 || pkgs[0].Name != "hashicorp/vpc/aws" { + t.Errorf("got %v, want [hashicorp/vpc/aws]", pkgs) + } +} + +func TestGetPackagesTerraformEmpty(t *testing.T) { + root := tree.TransformToTree(nil) + a := &adapter{} + pkgs, err := a.GetPackages("terraform-local", types.TERRAFORM, root) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(pkgs) != 0 { + t.Errorf("expected 0 packages, got %v", pkgs) + } +} + +func TestGetVersionsTerraformModule(t *testing.T) { + files := makeFiles( + "/hashicorp/vpc/aws/1.0.0/vpc-1.0.0.tar.gz", + "/hashicorp/vpc/aws/1.1.0/vpc-1.1.0.tar.gz", + // Different module — should not appear. + "/hashicorp/other/aws/1.0.0/other-1.0.0.tar.gz", + ) + root := tree.TransformToTree(files) + pkg := types.Package{Name: "hashicorp/vpc/aws"} + + a := &adapter{} + versions, err := a.GetVersions(pkg, root, "terraform-local", "hashicorp/vpc/aws", types.TERRAFORM) + if err != nil { + t.Fatalf("GetVersions: %v", err) + } + + vSet := make(map[string]bool) + for _, v := range versions { + vSet[v.Name] = true + } + expected := map[string]bool{"1.0.0": true, "1.1.0": true} + if len(vSet) != len(expected) { + t.Fatalf("got %v, want %v", vSet, expected) + } + for k := range expected { + if !vSet[k] { + t.Errorf("missing version %q", k) + } + } +} + +func TestGetVersionsTerraformProvider(t *testing.T) { + files := makeFiles( + "/hashicorp/aws/2.0.0/terraform-provider-aws_2.0.0_linux_amd64.zip", + "/hashicorp/aws/2.0.0/terraform-provider-aws_2.0.0_darwin_arm64.zip", + // Different provider — should not appear. + "/hashicorp/google/3.0.0/terraform-provider-google_3.0.0_linux_amd64.zip", + ) + root := tree.TransformToTree(files) + pkg := types.Package{Name: "hashicorp/aws"} + + a := &adapter{} + versions, err := a.GetVersions(pkg, root, "terraform-local", "hashicorp/aws", types.TERRAFORM) + if err != nil { + t.Fatalf("GetVersions: %v", err) + } + + if len(versions) != 1 || versions[0].Name != "2.0.0" { + t.Errorf("got %v, want [2.0.0]", versions) + } +} + +func TestGetVersionsTerraformEmpty(t *testing.T) { + root := tree.TransformToTree(nil) + pkg := types.Package{Name: "hashicorp/vpc/aws"} + + a := &adapter{} + versions, err := a.GetVersions(pkg, root, "terraform-local", "hashicorp/vpc/aws", types.TERRAFORM) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(versions) != 0 { + t.Errorf("expected 0 versions, got %v", versions) + } +} diff --git a/modules/har/pkg/har/migrate/adapter/mock_jfrog/cran_test.go b/modules/har/pkg/har/migrate/adapter/mock_jfrog/cran_test.go new file mode 100644 index 00000000..5a8d0bc1 --- /dev/null +++ b/modules/har/pkg/har/migrate/adapter/mock_jfrog/cran_test.go @@ -0,0 +1,46 @@ +package mock_jfrog + +import ( + "testing" + + "github.com/harness/cli/modules/har/pkg/har/migrate/adapter/jfrog" + "github.com/harness/cli/modules/har/pkg/har/migrate/tree" + "github.com/harness/cli/modules/har/pkg/har/migrate/types" +) + +// TestGetPackagesCRAN exercises CRAN enumeration through the mock client: live +// contrib archives, Archive/ superseded versions, and a windows binary. PACKAGES +// index files must not appear as packages. +func TestGetPackagesCRAN(t *testing.T) { + const registry = "cran-local" + adapter := jfrog.NewAdapterWithClient(types.RegistryConfig{Type: types.MOCK_JFROG}, NewMockClient()) + + files, err := adapter.GetFiles(registry) + if err != nil { + t.Fatalf("GetFiles: %v", err) + } + root := tree.TransformToTree(files) + + pkgs, err := adapter.GetPackages(registry, types.CRAN, root) + if err != nil { + t.Fatalf("GetPackages: %v", err) + } + + byName := make(map[string]types.Package) + for _, p := range pkgs { + byName[p.Name] = p + } + if len(pkgs) != 2 { + t.Fatalf("expected 2 packages (jsonlite, data.table), got %d: %+v", len(pkgs), pkgs) + } + for _, name := range []string{"jsonlite", "data.table"} { + if _, ok := byName[name]; !ok { + t.Errorf("missing package %q in %+v", name, byName) + } + } + for _, p := range pkgs { + if p.Name == "PACKAGES" || p.Name == "PACKAGES.gz" { + t.Errorf("index file enumerated as package: %+v", p) + } + } +} diff --git a/modules/har/pkg/har/migrate/adapter/mock_jfrog/mock_client.go b/modules/har/pkg/har/migrate/adapter/mock_jfrog/mock_client.go index 71fecda2..6b2c5fcb 100644 --- a/modules/har/pkg/har/migrate/adapter/mock_jfrog/mock_client.go +++ b/modules/har/pkg/har/migrate/adapter/mock_jfrog/mock_client.go @@ -242,6 +242,28 @@ func (c *mockClient) loadBinaryContent() { } c.binaryContent["nuget-local/foo/company.grpc.pkg/2.0.0/company.grpc.pkg.2.0.0.snupkg"] = createNugetPackageNupkg("company.grpc.pkg", "2.0.0") + + // Terraform modules: /////-.tar.gz + tfModules := []struct{ ns, name, provider, version string }{ + {"hashicorp", "vpc", "aws", "1.0.0"}, + {"hashicorp", "vpc", "aws", "1.1.0"}, + } + for _, m := range tfModules { + key := fmt.Sprintf("terraform-local/%s/%s/%s/%s/%s-%s.tar.gz", + m.ns, m.name, m.provider, m.version, m.name, m.version) + c.binaryContent[key] = createTerraformModuleTarGz(m.ns, m.name, m.provider, m.version) + } + + // Terraform providers: ////terraform-provider-___.zip + tfProviders := []struct{ ns, typeName, version, osName, arch string }{ + {"hashicorp", "aws", "2.0.0", "linux", "amd64"}, + {"hashicorp", "aws", "2.0.0", "darwin", "arm64"}, + } + for _, p := range tfProviders { + key := fmt.Sprintf("terraform-local/%s/%s/%s/terraform-provider-%s_%s_%s_%s.zip", + p.ns, p.typeName, p.version, p.typeName, p.version, p.osName, p.arch) + c.binaryContent[key] = createTerraformProviderZip(p.typeName, p.version, p.osName, p.arch) + } } func (c *mockClient) GetRegistries() ([]jfrog.JFrogRepository, error) { @@ -412,3 +434,33 @@ func createNpmPackageTgz(packageName, version, description string) []byte { return buf.Bytes() } + +// createTerraformModuleTarGz creates a minimal .tar.gz archive containing a +// main.tf file, satisfying HAR's "at least one .tf at root" validation. +func createTerraformModuleTarGz(ns, name, provider, version string) []byte { + var buf bytes.Buffer + gzWriter := gzip.NewWriter(&buf) + tarWriter := tar.NewWriter(gzWriter) + + content := fmt.Sprintf("# Terraform module %s/%s/%s v%s\n", ns, name, provider, version) + _ = tarWriter.WriteHeader(&tar.Header{ + Name: "main.tf", + Mode: 0644, + Size: int64(len(content)), + }) + tarWriter.Write([]byte(content)) + tarWriter.Close() + gzWriter.Close() + return buf.Bytes() +} + +// createTerraformProviderZip creates a minimal .zip archive for a provider binary. +func createTerraformProviderZip(typeName, version, osName, arch string) []byte { + var buf bytes.Buffer + w := zip.NewWriter(&buf) + content := fmt.Sprintf("mock provider binary: %s %s %s_%s\n", typeName, version, osName, arch) + f, _ := w.Create(fmt.Sprintf("terraform-provider-%s_%s_%s_%s", typeName, version, osName, arch)) + f.Write([]byte(content)) + w.Close() + return buf.Bytes() +} diff --git a/modules/har/pkg/har/migrate/adapter/mock_jfrog/terraform_test.go b/modules/har/pkg/har/migrate/adapter/mock_jfrog/terraform_test.go new file mode 100644 index 00000000..bc79638a --- /dev/null +++ b/modules/har/pkg/har/migrate/adapter/mock_jfrog/terraform_test.go @@ -0,0 +1,108 @@ +package mock_jfrog + +import ( + "testing" + + "github.com/harness/cli/modules/har/pkg/har/migrate/adapter/jfrog" + "github.com/harness/cli/modules/har/pkg/har/migrate/tree" + "github.com/harness/cli/modules/har/pkg/har/migrate/types" +) + +const terraformRegistry = "terraform-local" + +func TestGetPackagesTerraform(t *testing.T) { + adapter := jfrog.NewAdapterWithClient(types.RegistryConfig{Type: types.MOCK_JFROG}, NewMockClient()) + + files, err := adapter.GetFiles(terraformRegistry) + if err != nil { + t.Fatalf("GetFiles: %v", err) + } + root := tree.TransformToTree(files) + + pkgs, err := adapter.GetPackages(terraformRegistry, types.TERRAFORM, root) + if err != nil { + t.Fatalf("GetPackages: %v", err) + } + + pkgSet := make(map[string]bool) + for _, p := range pkgs { + pkgSet[p.Name] = true + } + + // hashicorp/vpc/aws = module; hashicorp/aws = provider + expected := map[string]bool{ + "hashicorp/vpc/aws": true, + "hashicorp/aws": true, + } + + if len(pkgSet) != len(expected) { + t.Fatalf("expected %d packages, got %d: %+v", len(expected), len(pkgSet), pkgs) + } + for pkg := range expected { + if !pkgSet[pkg] { + t.Errorf("missing expected package %q", pkg) + } + } +} + +func TestGetVersionsTerraformModule(t *testing.T) { + adapter := jfrog.NewAdapterWithClient(types.RegistryConfig{Type: types.MOCK_JFROG}, NewMockClient()) + + files, err := adapter.GetFiles(terraformRegistry) + if err != nil { + t.Fatalf("GetFiles: %v", err) + } + root := tree.TransformToTree(files) + + pkg := types.Package{Name: "hashicorp/vpc/aws"} + versions, err := adapter.GetVersions(pkg, root, terraformRegistry, "hashicorp/vpc/aws", types.TERRAFORM) + if err != nil { + t.Fatalf("GetVersions: %v", err) + } + + versionSet := make(map[string]bool) + for _, v := range versions { + versionSet[v.Name] = true + } + + expected := map[string]bool{"1.0.0": true, "1.1.0": true} + if len(versionSet) != len(expected) { + t.Fatalf("expected %d versions, got %d: %+v", len(expected), len(versionSet), versions) + } + for v := range expected { + if !versionSet[v] { + t.Errorf("missing expected version %q", v) + } + } +} + +func TestGetVersionsTerraformProvider(t *testing.T) { + adapter := jfrog.NewAdapterWithClient(types.RegistryConfig{Type: types.MOCK_JFROG}, NewMockClient()) + + files, err := adapter.GetFiles(terraformRegistry) + if err != nil { + t.Fatalf("GetFiles: %v", err) + } + root := tree.TransformToTree(files) + + pkg := types.Package{Name: "hashicorp/aws"} + versions, err := adapter.GetVersions(pkg, root, terraformRegistry, "hashicorp/aws", types.TERRAFORM) + if err != nil { + t.Fatalf("GetVersions: %v", err) + } + + versionSet := make(map[string]bool) + for _, v := range versions { + versionSet[v.Name] = true + } + + expected := map[string]bool{"2.0.0": true} + if len(versionSet) != len(expected) { + t.Fatalf("expected %d versions, got %d: %+v", len(expected), len(versionSet), versions) + } + for v := range expected { + if !versionSet[v] { + t.Errorf("missing expected version %q", v) + } + } +} diff --git a/modules/har/pkg/har/migrate/adapter/mock_jfrog/testdata/files/cran-local.json b/modules/har/pkg/har/migrate/adapter/mock_jfrog/testdata/files/cran-local.json new file mode 100644 index 00000000..0715ecba --- /dev/null +++ b/modules/har/pkg/har/migrate/adapter/mock_jfrog/testdata/files/cran-local.json @@ -0,0 +1,74 @@ +[ + { + "Registry": "cran-local", + "Name": "jsonlite_1.8.0.tar.gz", + "Uri": "/src/contrib/jsonlite_1.8.0.tar.gz", + "Folder": false, + "Size": 1024, + "LastModified": "2023-01-01T00:00:00.000Z", + "SHA1": "da39a3ee5e6b4b0d3255bfef95601890afd80730" + }, + { + "Registry": "cran-local", + "Name": "jsonlite_1.7.0.tar.gz", + "Uri": "/src/contrib/Archive/jsonlite/jsonlite_1.7.0.tar.gz", + "Folder": false, + "Size": 900, + "LastModified": "2022-06-01T00:00:00.000Z", + "SHA1": "da39a3ee5e6b4b0d3255bfef95601890afd80731" + }, + { + "Registry": "cran-local", + "Name": "data.table_1.14.8.tar.gz", + "Uri": "/src/contrib/data.table_1.14.8.tar.gz", + "Folder": false, + "Size": 2048, + "LastModified": "2023-03-01T00:00:00.000Z", + "SHA1": "da39a3ee5e6b4b0d3255bfef95601890afd80732" + }, + { + "Registry": "cran-local", + "Name": "jsonlite_1.8.0.zip", + "Uri": "/bin/windows/contrib/4.4/jsonlite_1.8.0.zip", + "Folder": false, + "Size": 1500, + "LastModified": "2023-04-01T00:00:00.000Z", + "SHA1": "da39a3ee5e6b4b0d3255bfef95601890afd80733" + }, + { + "Registry": "cran-local", + "Name": "jsonlite_1.8.0.tgz", + "Uri": "/bin/macosx/big-sur-arm64/contrib/4.4/jsonlite_1.8.0.tgz", + "Folder": false, + "Size": 1600, + "LastModified": "2023-04-05T00:00:00.000Z", + "SHA1": "da39a3ee5e6b4b0d3255bfef95601890afd80736" + }, + { + "Registry": "cran-local", + "Name": "data.table_1.14.8.tgz", + "Uri": "/bin/big-sur-arm64/contrib/4.4/data.table_1.14.8.tgz", + "Folder": false, + "Size": 2200, + "LastModified": "2023-04-06T00:00:00.000Z", + "SHA1": "da39a3ee5e6b4b0d3255bfef95601890afd80737" + }, + { + "Registry": "cran-local", + "Name": "PACKAGES", + "Uri": "/src/contrib/PACKAGES", + "Folder": false, + "Size": 128, + "LastModified": "2023-05-01T00:00:00.000Z", + "SHA1": "da39a3ee5e6b4b0d3255bfef95601890afd80734" + }, + { + "Registry": "cran-local", + "Name": "PACKAGES.gz", + "Uri": "/src/contrib/PACKAGES.gz", + "Folder": false, + "Size": 64, + "LastModified": "2023-05-01T00:00:00.000Z", + "SHA1": "da39a3ee5e6b4b0d3255bfef95601890afd80735" + } +] diff --git a/modules/har/pkg/har/migrate/adapter/mock_jfrog/testdata/files/terraform-local.json b/modules/har/pkg/har/migrate/adapter/mock_jfrog/testdata/files/terraform-local.json new file mode 100644 index 00000000..59f4bce6 --- /dev/null +++ b/modules/har/pkg/har/migrate/adapter/mock_jfrog/testdata/files/terraform-local.json @@ -0,0 +1,38 @@ +[ + { + "Registry": "terraform-local", + "Name": "vpc-1.0.0.tar.gz", + "Uri": "/hashicorp/vpc/aws/1.0.0/vpc-1.0.0.tar.gz", + "Folder": false, + "Size": 2048, + "LastModified": "2024-01-01T00:00:00.000Z", + "SHA1": "da39a3ee5e6b4b0d3255bfef95601890afd80740" + }, + { + "Registry": "terraform-local", + "Name": "vpc-1.1.0.tar.gz", + "Uri": "/hashicorp/vpc/aws/1.1.0/vpc-1.1.0.tar.gz", + "Folder": false, + "Size": 2100, + "LastModified": "2024-02-01T00:00:00.000Z", + "SHA1": "da39a3ee5e6b4b0d3255bfef95601890afd80741" + }, + { + "Registry": "terraform-local", + "Name": "terraform-provider-aws_2.0.0_linux_amd64.zip", + "Uri": "/hashicorp/aws/2.0.0/terraform-provider-aws_2.0.0_linux_amd64.zip", + "Folder": false, + "Size": 50000, + "LastModified": "2024-01-15T00:00:00.000Z", + "SHA1": "da39a3ee5e6b4b0d3255bfef95601890afd80742" + }, + { + "Registry": "terraform-local", + "Name": "terraform-provider-aws_2.0.0_darwin_arm64.zip", + "Uri": "/hashicorp/aws/2.0.0/terraform-provider-aws_2.0.0_darwin_arm64.zip", + "Folder": false, + "Size": 48000, + "LastModified": "2024-01-15T00:00:00.000Z", + "SHA1": "da39a3ee5e6b4b0d3255bfef95601890afd80743" + } +] diff --git a/modules/har/pkg/har/migrate/adapter/mock_jfrog/testdata/registries.json b/modules/har/pkg/har/migrate/adapter/mock_jfrog/testdata/registries.json index b5544ef4..7ca021b2 100644 --- a/modules/har/pkg/har/migrate/adapter/mock_jfrog/testdata/registries.json +++ b/modules/har/pkg/har/migrate/adapter/mock_jfrog/testdata/registries.json @@ -75,5 +75,19 @@ "url": "http://localhost:8081/artifactory/swift-local", "description": "Mock Swift Local Repository", "packageType": "swift" + }, + "cran-local": { + "key": "cran-local", + "type": "LOCAL", + "url": "http://localhost:8081/artifactory/cran-local", + "description": "Mock CRAN Local Repository", + "packageType": "cran" + }, + "terraform-local": { + "key": "terraform-local", + "type": "LOCAL", + "url": "http://localhost:8081/artifactory/terraform-local", + "description": "Mock Terraform Local Repository", + "packageType": "terraform" } } diff --git a/modules/har/pkg/har/migrate/migratable/file.go b/modules/har/pkg/har/migrate/migratable/file.go index 25a8b3f3..e61c7929 100644 --- a/modules/har/pkg/har/migrate/migratable/file.go +++ b/modules/har/pkg/har/migrate/migratable/file.go @@ -10,6 +10,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "net/mail" @@ -132,6 +133,7 @@ func (r *File) Pre(ctx context.Context) error { Uri: r.file.Uri, Size: int64(r.file.Size), Status: types.StatusSkip, + Reason: types.SkipReasonAlreadyExists, } r.stats.FileStats = append(r.stats.FileStats, stat) } @@ -172,7 +174,46 @@ func (r *File) Migrate(ctx context.Context) error { return fmt.Errorf("OCI migrate file is not supported") } - if r.artifactType == types.GENERIC || r.artifactType == types.RAW || r.artifactType == types.MAVEN || r.artifactType == types.NUGET || r.artifactType == types.PUPPET { + if r.artifactType == types.TERRAFORM { + downloadFile, header, err := r.srcAdapter.DownloadFile(r.srcRegistry, r.file.Uri) + if err != nil { + logger.Error().Err(err).Msg("Failed to download Terraform file") + return fmt.Errorf("download terraform file failed: %w", err) + } + defer downloadFile.Close() + + title := fmt.Sprintf("%s (%s)", r.file.Name, sizeutil.GetSize(int64(r.file.Size))) + pterm.Info.Println(fmt.Sprintf("Copying Terraform file %s from %s to %s", r.file.Name, r.srcRegistry, r.destRegistry)) + + err = r.destAdapter.UploadFile(r.destRegistry, downloadFile, r.file, header, r.pkg.Name, r.version.Name, + r.artifactType, nil) + + stat := types.FileStat{ + Name: r.file.Name, + Registry: r.srcRegistry, + Uri: r.file.Uri, + Size: int64(r.file.Size), + Status: types.StatusSuccess, + } + if err != nil { + if errors.Is(err, types.ErrArtifactAlreadyExists) { + stat.Status = types.StatusSkip + stat.Reason = types.SkipReasonAlreadyExists + pterm.Info.Println(fmt.Sprintf("%s already exists, skipping", title)) + } else { + logger.Error().Err(err).Msg("Failed to upload Terraform file") + stat.Status = types.StatusFail + stat.Error = err.Error() + pterm.Error.Println(title) + } + } else { + pterm.Success.Println(title) + } + r.stats.FileStats = append(r.stats.FileStats, stat) + return nil + } + + if r.artifactType == types.GENERIC || r.artifactType == types.RAW || r.artifactType == types.MAVEN || r.artifactType == types.NUGET || r.artifactType == types.PUPPET || r.artifactType == types.RUBY { downloadFile, header, err := r.srcAdapter.DownloadFile(r.srcRegistry, r.file.Uri) defer downloadFile.Close() if err != nil { @@ -193,10 +234,16 @@ func (r *File) Migrate(ctx context.Context) error { Status: types.StatusSuccess, } if err != nil { - logger.Error().Err(err).Msg("Failed to upload file") - stat.Status = types.StatusFail - stat.Error = err.Error() - pterm.Error.Println(title) + if errors.Is(err, types.ErrArtifactAlreadyExists) { + stat.Status = types.StatusSkip + stat.Reason = types.SkipReasonAlreadyExists + pterm.Info.Println(fmt.Sprintf("%s already exists, skipping", title)) + } else { + logger.Error().Err(err).Msg("Failed to upload file") + stat.Status = types.StatusFail + stat.Error = err.Error() + pterm.Error.Println(title) + } } else { pterm.Success.Println(title) } diff --git a/modules/har/pkg/har/migrate/migratable/file_terraform_test.go b/modules/har/pkg/har/migrate/migratable/file_terraform_test.go new file mode 100644 index 00000000..4cd6eb2e --- /dev/null +++ b/modules/har/pkg/har/migrate/migratable/file_terraform_test.go @@ -0,0 +1,204 @@ +package migratable + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + "testing" + + "github.com/harness/cli/modules/har/pkg/har/migrate/types" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/rs/zerolog" +) + +// noopTerraformAdapter provides zero-value implementations of every +// adapter.Adapter method so tests only need to override what they exercise. +type noopTerraformAdapter struct{} + +func (noopTerraformAdapter) GetKeyChain(string) (authn.Keychain, error) { return nil, nil } +func (noopTerraformAdapter) GetConfig() types.RegistryConfig { return types.RegistryConfig{} } +func (noopTerraformAdapter) ValidateCredentials() (bool, error) { return false, nil } +func (noopTerraformAdapter) GetRegistry(context.Context, string) (types.RegistryInfo, error) { + return types.RegistryInfo{}, nil +} +func (noopTerraformAdapter) CreateRegistryIfDoesntExist(string) (bool, error) { return false, nil } +func (noopTerraformAdapter) GetPackages(string, types.ArtifactType, *types.TreeNode) ([]types.Package, error) { + return nil, nil +} +func (noopTerraformAdapter) GetVersions(types.Package, *types.TreeNode, string, string, types.ArtifactType) ([]types.Version, error) { + return nil, nil +} +func (noopTerraformAdapter) GetFiles(string) ([]types.File, error) { return nil, nil } +func (noopTerraformAdapter) DownloadFile(string, string) (io.ReadCloser, http.Header, error) { + return nil, nil, fmt.Errorf("not implemented") +} +func (noopTerraformAdapter) UploadFile(string, io.ReadCloser, *types.File, http.Header, string, string, types.ArtifactType, map[string]interface{}) error { + return fmt.Errorf("not implemented") +} +func (noopTerraformAdapter) GetOCIImagePath(string, string, string) (string, error) { return "", nil } +func (noopTerraformAdapter) AddNPMTag(string, string, string, string) error { return nil } +func (noopTerraformAdapter) VersionExists(context.Context, types.Package, string, string, string, types.ArtifactType) (bool, error) { + return false, nil +} +func (noopTerraformAdapter) FileExists(context.Context, string, string, string, *types.File, types.ArtifactType) (bool, error) { + return false, nil +} +func (noopTerraformAdapter) GetAllFilesForVersion(context.Context, string, string, string) ([]string, error) { + return nil, nil +} +func (noopTerraformAdapter) CreateVersion(string, string, string, types.ArtifactType, []*types.PackageFiles, map[string]interface{}) error { + return nil +} +func (noopTerraformAdapter) SearchFiles(string) ([]types.SearchedFile, error) { return nil, nil } +func (noopTerraformAdapter) BuildExistingIndex(context.Context, string, int) (*types.ExistingIndex, error) { + return nil, nil +} + +// terraformFileFakeSrc serves file content keyed by URI. +type terraformFileFakeSrc struct { + noopTerraformAdapter + content map[string][]byte + failURI string // if set, DownloadFile returns an error for this URI +} + +func (s *terraformFileFakeSrc) DownloadFile(_ string, uri string) (io.ReadCloser, http.Header, error) { + if s.failURI != "" && s.failURI == uri { + return nil, nil, errors.New("download error") + } + b, ok := s.content[uri] + if !ok { + return nil, nil, errors.New("not found") + } + return io.NopCloser(strings.NewReader(string(b))), http.Header{}, nil +} + +// terraformFileFakeDest records uploads and can be configured to return specific errors. +type terraformFileFakeDest struct { + noopTerraformAdapter + uploaded []string + uploadErr error +} + +func (d *terraformFileFakeDest) UploadFile( + _ string, + file io.ReadCloser, + f *types.File, + _ http.Header, + _ string, + _ string, + _ types.ArtifactType, + _ map[string]interface{}, +) error { + if file != nil { + _, _ = io.Copy(io.Discard, file) + _ = file.Close() + } + if d.uploadErr != nil { + return d.uploadErr + } + d.uploaded = append(d.uploaded, f.Name) + return nil +} + +func newTerraformFileJob(src *terraformFileFakeSrc, dest *terraformFileFakeDest, f *types.File, pkg, version string, stats *types.TransferStats) *File { + return &File{ + srcRegistry: "src-reg", + destRegistry: "dst-reg", + srcAdapter: src, + destAdapter: dest, + artifactType: types.TERRAFORM, + logger: zerolog.Nop(), + pkg: types.Package{Name: pkg}, + version: types.Version{Name: version}, + file: f, + stats: stats, + config: &types.Config{Concurrency: 1, DryRun: false}, + mapping: &types.RegistryMapping{}, + } +} + +func TestFileMigrateTerraformModuleSuccess(t *testing.T) { + f := &types.File{Name: "vpc-1.0.0.tar.gz", Uri: "/hashicorp/vpc/aws/1.0.0/vpc-1.0.0.tar.gz", Size: 10} + src := &terraformFileFakeSrc{content: map[string][]byte{f.Uri: []byte("module")}} + dest := &terraformFileFakeDest{} + stats := &types.TransferStats{} + + job := newTerraformFileJob(src, dest, f, "hashicorp/vpc/aws", "1.0.0", stats) + if err := job.Migrate(context.Background()); err != nil { + t.Fatalf("Migrate() error: %v", err) + } + if len(dest.uploaded) != 1 || dest.uploaded[0] != f.Name { + t.Errorf("uploaded = %v, want [%s]", dest.uploaded, f.Name) + } + if len(stats.FileStats) != 1 || stats.FileStats[0].Status != types.StatusSuccess { + t.Errorf("stat = %+v, want StatusSuccess", stats.FileStats) + } +} + +func TestFileMigrateTerraformProviderSuccess(t *testing.T) { + f := &types.File{Name: "terraform-provider-aws_2.0.0_linux_amd64.zip", Uri: "/hashicorp/aws/2.0.0/terraform-provider-aws_2.0.0_linux_amd64.zip", Size: 20} + src := &terraformFileFakeSrc{content: map[string][]byte{f.Uri: []byte("provider")}} + dest := &terraformFileFakeDest{} + stats := &types.TransferStats{} + + job := newTerraformFileJob(src, dest, f, "hashicorp/aws", "2.0.0", stats) + if err := job.Migrate(context.Background()); err != nil { + t.Fatalf("Migrate() error: %v", err) + } + if len(dest.uploaded) != 1 { + t.Errorf("uploaded = %v, want 1 file", dest.uploaded) + } + if stats.FileStats[0].Status != types.StatusSuccess { + t.Errorf("stat status = %v, want StatusSuccess", stats.FileStats[0].Status) + } +} + +func TestFileMigrateTerraformAlreadyExists(t *testing.T) { + f := &types.File{Name: "vpc-1.0.0.tar.gz", Uri: "/hashicorp/vpc/aws/1.0.0/vpc-1.0.0.tar.gz", Size: 10} + src := &terraformFileFakeSrc{content: map[string][]byte{f.Uri: []byte("module")}} + dest := &terraformFileFakeDest{uploadErr: types.ErrArtifactAlreadyExists} + stats := &types.TransferStats{} + + job := newTerraformFileJob(src, dest, f, "hashicorp/vpc/aws", "1.0.0", stats) + if err := job.Migrate(context.Background()); err != nil { + t.Fatalf("Migrate() error: %v", err) + } + if len(stats.FileStats) != 1 || stats.FileStats[0].Status != types.StatusSkip { + t.Errorf("stat = %+v, want StatusSkip", stats.FileStats) + } + if stats.FileStats[0].Reason != types.SkipReasonAlreadyExists { + t.Errorf("Reason = %q, want %q", stats.FileStats[0].Reason, types.SkipReasonAlreadyExists) + } +} + +func TestFileMigrateTerraformUploadError(t *testing.T) { + f := &types.File{Name: "vpc-1.0.0.tar.gz", Uri: "/hashicorp/vpc/aws/1.0.0/vpc-1.0.0.tar.gz", Size: 10} + src := &terraformFileFakeSrc{content: map[string][]byte{f.Uri: []byte("module")}} + dest := &terraformFileFakeDest{uploadErr: errors.New("server error")} + stats := &types.TransferStats{} + + job := newTerraformFileJob(src, dest, f, "hashicorp/vpc/aws", "1.0.0", stats) + if err := job.Migrate(context.Background()); err != nil { + t.Fatalf("Migrate() error: %v", err) + } + if len(stats.FileStats) != 1 || stats.FileStats[0].Status != types.StatusFail { + t.Errorf("stat = %+v, want StatusFail", stats.FileStats) + } +} + +func TestFileMigrateTerraformDownloadError(t *testing.T) { + f := &types.File{Name: "vpc-1.0.0.tar.gz", Uri: "/hashicorp/vpc/aws/1.0.0/vpc-1.0.0.tar.gz", Size: 10} + src := &terraformFileFakeSrc{content: map[string][]byte{}, failURI: f.Uri} + dest := &terraformFileFakeDest{} + stats := &types.TransferStats{} + + job := newTerraformFileJob(src, dest, f, "hashicorp/vpc/aws", "1.0.0", stats) + err := job.Migrate(context.Background()) + if err == nil { + t.Fatal("expected error from download failure, got nil") + } +} diff --git a/modules/har/pkg/har/migrate/migratable/package.go b/modules/har/pkg/har/migrate/migratable/package.go index f56af051..7dd5c08d 100644 --- a/modules/har/pkg/har/migrate/migratable/package.go +++ b/modules/har/pkg/har/migrate/migratable/package.go @@ -55,6 +55,7 @@ type Package struct { dryRunStats *types.DryRunStats unfilteredRoot *types.TreeNode existingIndex *types.ExistingIndex + files []types.File } func NewPackageJob( @@ -73,6 +74,7 @@ func NewPackageJob( dryRunStats *types.DryRunStats, unfilteredRoot *types.TreeNode, existingIndex *types.ExistingIndex, + files []types.File, ) engine.Job { jobID := uuid.New().String() @@ -101,6 +103,7 @@ func NewPackageJob( dryRunStats: dryRunStats, unfilteredRoot: unfilteredRoot, existingIndex: existingIndex, + files: files, } } @@ -148,6 +151,7 @@ func (r *Package) Pre(ctx context.Context) error { Uri: r.pkg.Version, Size: int64(r.pkg.Size), Status: types.StatusSkip, + Reason: types.SkipReasonAlreadyExists, } r.stats.FileStats = append(r.stats.FileStats, stat) return nil @@ -203,6 +207,7 @@ func (r *Package) Pre(ctx context.Context) error { Uri: r.pkg.Name + ":" + tag, Size: 0, Status: types.StatusSkip, + Reason: types.SkipReasonAlreadyExists, } r.stats.FileStats = append(r.stats.FileStats, stat) } @@ -307,6 +312,8 @@ func (r *Package) Migrate(ctx context.Context) error { r.migrateComposer(ctx) } else if r.artifactType == types.SWIFT { r.migrateSwift(ctx) + } else if r.artifactType == types.CRAN { + r.migrateCran(ctx) } else { versions, err := r.srcAdapter.GetVersions(r.pkg, r.node, r.srcRegistry, r.pkg.Name, r.artifactType) if err != nil { @@ -314,6 +321,21 @@ func (r *Package) Migrate(ctx context.Context) error { return fmt.Errorf("get versions failed: %w", err) } + // Apply the opt-in version selector (packageFilters[].versions). When the + // current package is named in packageFilters with a non-empty versions list, + // only the named versions are migrated. + if sel, hasFilters, matched := util.SelectorForPackage(r.mapping, r.pkg.Name); hasFilters && matched && len(sel.Versions) > 0 { + originalCount := len(versions) + filtered := versions[:0:0] + for _, v := range versions { + if util.VersionSelectedBySelector(sel, v.Name) { + filtered = append(filtered, v) + } + } + versions = filtered + logger.Info().Msgf("Version selector filter for package %s: %d -> %d versions", r.pkg.Name, originalCount, len(versions)) + } + var jobs []engine.Job for _, version := range versions { versionNode, err := tree.GetNodeForPath(r.node, version.Path) @@ -532,6 +554,7 @@ func (r *Package) migrateComposerVersion(ctx context.Context, v types.Version) e if err != nil { if errors.Is(err, types.ErrArtifactAlreadyExists) { stat.Status = types.StatusSkip + stat.Reason = types.SkipReasonAlreadyExists pterm.Info.Println(fmt.Sprintf("%s already exists, skipping", title)) } else { r.logger.Error().Err(err).Msg("Failed to upload file") @@ -777,6 +800,7 @@ func (r *Package) migrateHelmHTTP(ctx context.Context) error { if err != nil { if errors.Is(err, types.ErrArtifactAlreadyExists) { stat.Status = types.StatusSkip + stat.Reason = types.SkipReasonAlreadyExists pterm.Info.Println(fmt.Sprintf("%s already exists, skipping", title)) r.stats.FileStats = append(r.stats.FileStats, stat) return nil @@ -825,6 +849,7 @@ func (r *Package) migrateHelmHTTPProv(ctx context.Context) { if err != nil { if errors.Is(err, types.ErrArtifactAlreadyExists) { stat.Status = types.StatusSkip + stat.Reason = types.SkipReasonAlreadyExists pterm.Info.Println(fmt.Sprintf("Provenance %s already exists, skipping", provName)) } else { r.logger.Error().Err(err).Msgf("Failed to upload provenance %s", provName) @@ -1015,6 +1040,7 @@ func (r *Package) migrateConan(ctx context.Context) error { if ulErr != nil { if errors.Is(ulErr, types.ErrArtifactAlreadyExists) { stat.Status = types.StatusSkip + stat.Reason = types.SkipReasonAlreadyExists pterm.Info.Println(fmt.Sprintf("%s already exists, skipping", title)) } else { r.logger.Error().Err(ulErr).Msgf("Failed to upload Conan file %s", entry.FileName) @@ -1030,6 +1056,104 @@ func (r *Package) migrateConan(ctx context.Context) error { return nil } +// migrateCran uploads every archive for a single R package (source and platform +// binaries). One version string can map to multiple paths, so migration stays at +// package scope rather than spawning per-version jobs. +func (r *Package) migrateCran(ctx context.Context) error { + if r.config.DryRun { + r.logger.Info().Msgf("Dry-run: skipping CRAN migration for package %s", r.pkg.Name) + return nil + } + + for _, file := range r.files { + if file.Folder || util.IsCranIndexFile(file.Uri) { + continue + } + _, version, ok := util.ParseCranFileNameWithPath(file.Uri) + if !ok { + continue + } + + destPath, ok := util.CranHarUploadPath(file.Uri) + if !ok { + r.logger.Error().Msgf("Failed to remap CRAN path %s for package %s", file.Uri, r.pkg.Name) + r.stats.FileStats = append(r.stats.FileStats, types.FileStat{ + Name: file.Name, + Registry: r.srcRegistry, + Uri: file.Uri, + Size: int64(file.Size), + Status: types.StatusFail, + Error: fmt.Sprintf("remap CRAN path %s for upload", file.Uri), + }) + continue + } + + if !r.config.Overwrite { + checkFile := &types.File{Uri: destPath} + exists, headErr := r.destAdapter.FileExists(ctx, r.registry.Path, r.pkg.Name, version, checkFile, types.CRAN) + if headErr != nil { + r.logger.Warn().Err(headErr).Msgf("Failed to HEAD file %s, will proceed with migration", destPath) + } else if exists { + r.logger.Info().Msgf("Skipping file %s as it already exists in destination (HEAD 200)", destPath) + r.stats.FileStats = append(r.stats.FileStats, types.FileStat{ + Name: file.Name, + Registry: r.srcRegistry, + Uri: file.Uri, + Size: int64(file.Size), + Status: types.StatusSkip, + Reason: types.SkipReasonAlreadyExists, + }) + continue + } + } + + downloadFile, header, err := r.srcAdapter.DownloadFile(r.srcRegistry, file.Uri) + if err != nil { + r.logger.Error().Err(err).Msgf("Failed to download CRAN file %s", file.Uri) + r.stats.FileStats = append(r.stats.FileStats, types.FileStat{ + Name: file.Name, + Registry: r.srcRegistry, + Uri: file.Uri, + Size: int64(file.Size), + Status: types.StatusFail, + Error: err.Error(), + }) + continue + } + + uploadFile := &types.File{Name: file.Name, Uri: destPath, Size: file.Size} + title := fmt.Sprintf("%s (%s)", file.Name, sizeutil.GetSize(int64(file.Size))) + pterm.Info.Println(fmt.Sprintf("Copying file %s from %s to %s", file.Name, r.srcRegistry, r.destRegistry)) + err = r.destAdapter.UploadFile(r.destRegistry, downloadFile, uploadFile, header, r.pkg.Name, version, types.CRAN, nil) + _ = downloadFile.Close() + + stat := types.FileStat{ + Name: file.Name, + Registry: r.srcRegistry, + Uri: file.Uri, + Size: int64(file.Size), + Status: types.StatusSuccess, + } + if err != nil { + if errors.Is(err, types.ErrArtifactAlreadyExists) { + stat.Status = types.StatusSkip + stat.Reason = types.SkipReasonAlreadyExists + pterm.Info.Println(fmt.Sprintf("%s already exists, skipping", title)) + } else { + r.logger.Error().Err(err).Msgf("Failed to upload CRAN file %s", file.Name) + stat.Status = types.StatusFail + stat.Error = err.Error() + pterm.Error.Println(fmt.Sprintf("%s — %v", title, err)) + } + } else { + pterm.Success.Println(title) + } + r.stats.FileStats = append(r.stats.FileStats, stat) + } + + return nil +} + // extractUpstreamVersion strips the Debian revision and epoch from a version string. // e.g. "2:1.5.0-1ubuntu2" -> "1.5.0" func extractUpstreamVersion(version string) string { diff --git a/modules/har/pkg/har/migrate/migratable/package_cran_test.go b/modules/har/pkg/har/migrate/migratable/package_cran_test.go new file mode 100644 index 00000000..f294c8c1 --- /dev/null +++ b/modules/har/pkg/har/migrate/migratable/package_cran_test.go @@ -0,0 +1,298 @@ +// Copyright © 2026 Harness Inc. +// SPDX-License-Identifier: Apache-2.0 + +package migratable + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "path" + "strings" + "testing" + + "github.com/harness/cli/modules/har/pkg/har/migrate/tree" + "github.com/harness/cli/modules/har/pkg/har/migrate/types" + "github.com/harness/cli/modules/har/pkg/har/migrate/util" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/rs/zerolog" +) + +// noopCranAdapter provides zero-value implementations of every +// adapter.Adapter method so tests only need to override what they exercise. +type noopCranAdapter struct{} + +func (noopCranAdapter) GetKeyChain(string) (authn.Keychain, error) { return nil, nil } +func (noopCranAdapter) GetConfig() types.RegistryConfig { return types.RegistryConfig{} } +func (noopCranAdapter) ValidateCredentials() (bool, error) { return false, nil } +func (noopCranAdapter) GetRegistry(context.Context, string) (types.RegistryInfo, error) { + return types.RegistryInfo{}, nil +} +func (noopCranAdapter) CreateRegistryIfDoesntExist(string) (bool, error) { return false, nil } +func (noopCranAdapter) GetPackages(string, types.ArtifactType, *types.TreeNode) ([]types.Package, error) { + return nil, nil +} +func (noopCranAdapter) GetVersions(types.Package, *types.TreeNode, string, string, types.ArtifactType) ([]types.Version, error) { + return nil, nil +} +func (noopCranAdapter) GetFiles(string) ([]types.File, error) { return nil, nil } +func (noopCranAdapter) DownloadFile(string, string) (io.ReadCloser, http.Header, error) { + return nil, nil, fmt.Errorf("not implemented") +} +func (noopCranAdapter) UploadFile(string, io.ReadCloser, *types.File, http.Header, string, string, types.ArtifactType, map[string]interface{}) error { + return fmt.Errorf("not implemented") +} +func (noopCranAdapter) GetOCIImagePath(string, string, string) (string, error) { return "", nil } +func (noopCranAdapter) AddNPMTag(string, string, string, string) error { return nil } +func (noopCranAdapter) VersionExists(context.Context, types.Package, string, string, string, types.ArtifactType) (bool, error) { + return false, nil +} +func (noopCranAdapter) FileExists(context.Context, string, string, string, *types.File, types.ArtifactType) (bool, error) { + return false, nil +} +func (noopCranAdapter) GetAllFilesForVersion(context.Context, string, string, string) ([]string, error) { + return nil, nil +} +func (noopCranAdapter) CreateVersion(string, string, string, types.ArtifactType, []*types.PackageFiles, map[string]interface{}) error { + return nil +} +func (noopCranAdapter) SearchFiles(string) ([]types.SearchedFile, error) { return nil, nil } +func (noopCranAdapter) BuildExistingIndex(context.Context, string, int) (*types.ExistingIndex, error) { + return nil, nil +} + +type cranFakeSrc struct { + noopCranAdapter + content map[string][]byte +} + +func (s *cranFakeSrc) DownloadFile(_ string, uri string) (io.ReadCloser, http.Header, error) { + b, ok := s.content[uri] + if !ok { + return nil, nil, fmt.Errorf("download %q: not found", uri) + } + return io.NopCloser(strings.NewReader(string(b))), http.Header{}, nil +} + +type cranFakeDest struct { + noopCranAdapter + uploadedUris []string + uploadErr error + exists bool + existsErr error + headURI string +} + +func (d *cranFakeDest) UploadFile( + _ string, + file io.ReadCloser, + f *types.File, + _ http.Header, + _ string, + _ string, + _ types.ArtifactType, + _ map[string]interface{}, +) error { + if file != nil { + _, _ = io.Copy(io.Discard, file) + _ = file.Close() + } + if d.uploadErr != nil { + return d.uploadErr + } + d.uploadedUris = append(d.uploadedUris, f.Uri) + return nil +} + +func (d *cranFakeDest) FileExists( + _ context.Context, + _, _, _ string, + file *types.File, + _ types.ArtifactType, +) (bool, error) { + if file != nil { + d.headURI = file.Uri + } + return d.exists, d.existsErr +} + +func cranFileTree(uris ...string) *types.TreeNode { + files := make([]types.File, 0, len(uris)) + for _, u := range uris { + files = append(files, types.File{Name: path.Base(u), Uri: u, Size: 10}) + } + return tree.TransformToTree(files) +} + +func filesForCranPackage(node *types.TreeNode, pkgName string) []types.File { + all, err := tree.GetAllFiles(node) + if err != nil { + return nil + } + flat := make([]types.File, 0, len(all)) + for _, f := range all { + if f != nil { + flat = append(flat, *f) + } + } + return util.BuildCranPackageFilesMap(flat)[pkgName] +} + +func newCranPackageJob(src *cranFakeSrc, dest *cranFakeDest, node *types.TreeNode, stats *types.TransferStats) *Package { + pkgName := "jsonlite" + return &Package{ + srcRegistry: "src-reg", + destRegistry: "dst-reg", + srcAdapter: src, + destAdapter: dest, + artifactType: types.CRAN, + logger: zerolog.Nop(), + pkg: types.Package{Name: pkgName, Path: "/"}, + node: node, + files: filesForCranPackage(node, pkgName), + stats: stats, + config: &types.Config{Concurrency: 1, DryRun: false, Overwrite: false}, + mapping: &types.RegistryMapping{}, + registry: types.RegistryInfo{Path: "cran-dest"}, + } +} + +func TestPackageMigrateCRANRemapsArchivePath(t *testing.T) { + srcURI := "/src/contrib/Archive/jsonlite/1.7.0/jsonlite_1.7.0.tar.gz" + node := cranFileTree(srcURI) + src := &cranFakeSrc{content: map[string][]byte{srcURI: []byte("pkg")}} + dest := &cranFakeDest{} + stats := &types.TransferStats{} + + job := newCranPackageJob(src, dest, node, stats) + if err := job.migrateCran(context.Background()); err != nil { + t.Fatalf("migrateCran() error: %v", err) + } + if len(dest.uploadedUris) != 1 || dest.uploadedUris[0] != "src/contrib/jsonlite_1.7.0.tar.gz" { + t.Errorf("uploaded = %v, want [src/contrib/jsonlite_1.7.0.tar.gz]", dest.uploadedUris) + } +} + +func TestPackageMigrateCRANUploadsAllPlatformsForSameVersion(t *testing.T) { + srcTar := "/src/contrib/jsonlite_1.8.0.tar.gz" + winZip := "/bin/windows/contrib/4.4/jsonlite_1.8.0.zip" + node := cranFileTree(srcTar, winZip) + src := &cranFakeSrc{content: map[string][]byte{ + srcTar: []byte("src"), + winZip: []byte("win"), + }} + dest := &cranFakeDest{} + stats := &types.TransferStats{} + + job := newCranPackageJob(src, dest, node, stats) + if err := job.migrateCran(context.Background()); err != nil { + t.Fatalf("migrateCran() error: %v", err) + } + if len(dest.uploadedUris) != 2 { + t.Fatalf("uploaded = %v, want 2 files for version 1.8.0", dest.uploadedUris) + } +} + +func TestPackageMigrateCRANSkipsIndexAndOtherPackages(t *testing.T) { + node := cranFileTree( + "/src/contrib/jsonlite_1.8.0.tar.gz", + "/src/contrib/Archive/jsonlite/1.7.0/jsonlite_1.7.0.tar.gz", + "/src/contrib/data.table_1.14.0.tar.gz", + "/src/contrib/PACKAGES", + ) + src := &cranFakeSrc{content: map[string][]byte{ + "/src/contrib/jsonlite_1.8.0.tar.gz": []byte("live"), + }} + dest := &cranFakeDest{} + stats := &types.TransferStats{} + + job := newCranPackageJob(src, dest, node, stats) + if err := job.migrateCran(context.Background()); err != nil { + t.Fatalf("migrateCran() error: %v", err) + } + if len(dest.uploadedUris) != 1 { + t.Errorf("uploaded count = %d, want 1 (jsonlite 1.8.0 only)", len(dest.uploadedUris)) + } +} + +func TestPackageMigrateCRANAlreadyExists(t *testing.T) { + srcURI := "/src/contrib/jsonlite_1.8.0.tar.gz" + node := cranFileTree(srcURI) + src := &cranFakeSrc{content: map[string][]byte{srcURI: []byte("pkg")}} + dest := &cranFakeDest{uploadErr: types.ErrArtifactAlreadyExists} + stats := &types.TransferStats{} + + job := newCranPackageJob(src, dest, node, stats) + if err := job.migrateCran(context.Background()); err != nil { + t.Fatalf("migrateCran() error: %v", err) + } + if len(stats.FileStats) != 1 || stats.FileStats[0].Status != types.StatusSkip { + t.Errorf("stat = %+v, want StatusSkip", stats.FileStats) + } + if stats.FileStats[0].Reason != types.SkipReasonAlreadyExists { + t.Errorf("Reason = %q, want %q", stats.FileStats[0].Reason, types.SkipReasonAlreadyExists) + } +} + +func TestPackageMigrateCRANSkipsUnrecognizedPaths(t *testing.T) { + srcURI := "/not/a/cran/path.txt" + node := cranFileTree(srcURI) + src := &cranFakeSrc{content: map[string][]byte{srcURI: []byte("x")}} + dest := &cranFakeDest{} + stats := &types.TransferStats{} + + job := newCranPackageJob(src, dest, node, stats) + if err := job.migrateCran(context.Background()); err != nil { + t.Fatalf("migrateCran() error: %v", err) + } + if len(dest.uploadedUris) != 0 { + t.Errorf("expected no uploads for unrecognized path, got %v", dest.uploadedUris) + } + if len(stats.FileStats) != 0 { + t.Errorf("stats = %+v, want no entries for skipped unrecognized path", stats.FileStats) + } +} + +func TestPackageMigrateCRANSkipsWhenHeadExists(t *testing.T) { + srcURI := "/src/contrib/Archive/jsonlite/1.7.0/jsonlite_1.7.0.tar.gz" + node := cranFileTree(srcURI) + src := &cranFakeSrc{content: map[string][]byte{srcURI: []byte("pkg")}} + dest := &cranFakeDest{exists: true} + stats := &types.TransferStats{} + + job := newCranPackageJob(src, dest, node, stats) + if err := job.migrateCran(context.Background()); err != nil { + t.Fatalf("migrateCran() error: %v", err) + } + if dest.headURI != "src/contrib/jsonlite_1.7.0.tar.gz" { + t.Errorf("HEAD uri = %q, want remapped contrib path", dest.headURI) + } + if len(dest.uploadedUris) != 0 { + t.Errorf("expected no uploads after HEAD skip, got %v", dest.uploadedUris) + } + if len(stats.FileStats) != 1 || stats.FileStats[0].Status != types.StatusSkip { + t.Errorf("stat = %+v, want StatusSkip", stats.FileStats) + } + if stats.FileStats[0].Reason != types.SkipReasonAlreadyExists { + t.Errorf("Reason = %q, want %q", stats.FileStats[0].Reason, types.SkipReasonAlreadyExists) + } +} + +func TestPackageMigrateCRANProceedsWhenHeadErrors(t *testing.T) { + srcURI := "/src/contrib/jsonlite_1.8.0.tar.gz" + node := cranFileTree(srcURI) + src := &cranFakeSrc{content: map[string][]byte{srcURI: []byte("pkg")}} + dest := &cranFakeDest{existsErr: errors.New("head failed")} + stats := &types.TransferStats{} + + job := newCranPackageJob(src, dest, node, stats) + if err := job.migrateCran(context.Background()); err != nil { + t.Fatalf("migrateCran() error: %v", err) + } + if len(dest.uploadedUris) != 1 { + t.Errorf("expected upload after HEAD error, got %v", dest.uploadedUris) + } +} diff --git a/modules/har/pkg/har/migrate/migratable/registry.go b/modules/har/pkg/har/migrate/migratable/registry.go index 2617d533..2c8657a6 100644 --- a/modules/har/pkg/har/migrate/migratable/registry.go +++ b/modules/har/pkg/har/migrate/migratable/registry.go @@ -233,8 +233,11 @@ func (r *Registry) Migrate(ctx context.Context) error { // For PYTHON (IsAtomicVersionArtifact), build an unfilteredRoot from the // original file list so version.go can recover distributions that were pruned // by date or pattern filters. Other types pass nil. + // TERRAFORM also gets the unfilteredRoot: its provider versions are atomic + // multi-file versions too (all platform zips per version), but discovery + // happens inside Version.Migrate, which swaps to this tree — see version.go. var unfilteredRoot *types.TreeNode - if dateFilterActive && util.IsAtomicVersionArtifact(currArtifactType) { + if dateFilterActive && (util.IsAtomicVersionArtifact(currArtifactType) || currArtifactType == types.TERRAFORM) { recoveryFiles := originalFiles if util.IsFileLevelFilterableArtifact(currArtifactType) && (len(r.mapping.IncludePatterns) > 0 || len(r.mapping.ExcludePatterns) > 0) { @@ -272,13 +275,22 @@ func (r *Registry) Migrate(ctx context.Context) error { } } + // Apply the opt-in package selector allow-list (packageFilters). When set, + // only the named packages are migrated; runs before the destination index is + // built so the index only covers selected work. No-op when unset. + if len(r.mapping.PackageFilters) > 0 { + originalCount := len(pkgs) + pkgs = util.FilterPackagesBySelectors(pkgs, r.mapping.PackageFilters) + logger.Info().Msgf("Package selector filter: %d -> %d packages", originalCount, len(pkgs)) + } + if r.artifactType == types.COMPOSER && composerPkgsBeforeFilters > 0 && len(pkgs) == 0 && - (len(r.mapping.IncludePatterns) > 0 || len(r.mapping.ExcludePatterns) > 0) { + (len(r.mapping.IncludePatterns) > 0 || len(r.mapping.ExcludePatterns) > 0 || len(r.mapping.PackageFilters) > 0) { warnMsg := fmt.Sprintf( "Registry %s: Composer package filters reduced %d package(s) to 0; nothing will be migrated — "+ - "use vendor/package names (e.g. harness/migtest, acme/*), not zip basenames, in includePatterns and excludePatterns", + "use vendor/package names (e.g. harness/migtest, acme/*), not zip basenames, in includePatterns, excludePatterns, and packageFilters", r.srcRegistry, composerPkgsBeforeFilters, ) logger.Warn().Msg(warnMsg) @@ -291,7 +303,7 @@ func (r *Registry) Migrate(ctx context.Context) error { indexApplicable := func(t types.ArtifactType) bool { switch t { case types.GENERIC, types.RAW, types.MAVEN, types.PYTHON, - types.NUGET, types.NPM, types.DART, types.PUPPET: + types.NUGET, types.NPM, types.DART, types.PUPPET, types.RUBY: return true } return false @@ -315,6 +327,11 @@ func (r *Registry) Migrate(ctx context.Context) error { } } + var filesByPkg map[string][]types.File + if r.artifactType == types.CRAN { + filesByPkg = util.BuildCranPackageFilesMap(files) + } + var jobs []engine.Job for _, pkg := range pkgs { treeNode, err2 := tree.GetNodeForPath(root, pkg.Path) @@ -322,8 +339,12 @@ func (r *Registry) Migrate(ctx context.Context) error { logger.Error().Msgf("Failed to get node for path %s", pkg.Path) return fmt.Errorf("get node for path %s failed: %w", pkg.Path, err2) } + var pkgFiles []types.File + if filesByPkg != nil { + pkgFiles = filesByPkg[pkg.Name] + } job := NewPackageJob(r.srcAdapter, r.destAdapter, r.srcRegistry, r.sourcePackageHostname, r.destRegistry, r.artifactType, pkg, treeNode, - r.stats, r.mapping, r.config, r.registry, r.dryRunStats, unfilteredRoot, existingIndex) + r.stats, r.mapping, r.config, r.registry, r.dryRunStats, unfilteredRoot, existingIndex, pkgFiles) jobs = append(jobs, job) } diff --git a/modules/har/pkg/har/migrate/migratable/version.go b/modules/har/pkg/har/migrate/migratable/version.go index 9ad49b9d..95da66bf 100644 --- a/modules/har/pkg/har/migrate/migratable/version.go +++ b/modules/har/pkg/har/migrate/migratable/version.go @@ -150,9 +150,13 @@ func (r *Version) Migrate(ctx context.Context) error { var jobs []engine.Job if r.artifactType == types.GENERIC || r.artifactType == types.RAW || r.artifactType == types.MAVEN || r.artifactType == types.PYTHON || - r.artifactType == types.NUGET || r.artifactType == types.NPM || r.artifactType == types.DART || r.artifactType == types.PUPPET { + r.artifactType == types.NUGET || r.artifactType == types.NPM || r.artifactType == types.DART || r.artifactType == types.PUPPET || + r.artifactType == types.RUBY || r.artifactType == types.TERRAFORM { // For PYTHON, use unfilteredRoot so distribution files pruned by the date filter // are still enumerated — prevents partial versions from being published. + // TERRAFORM versions are atomic multi-file versions (a provider version spans + // one zip per os/arch and a network mirror is only complete when ALL platform + // files are present), so it also swaps to the unfiltered tree when available. fileNode := r.node if r.artifactType == types.PYTHON && r.unfilteredRoot != nil { if unfilteredPkgNode, e := tree.GetNodeForPath(r.unfilteredRoot, r.pkg.Path); e == nil { @@ -161,13 +165,24 @@ func (r *Version) Migrate(ctx context.Context) error { logger.Debug().Str("version", r.version.Name).Msg("recovered distribution files from unfiltered tree") } } + } else if r.artifactType == types.TERRAFORM && r.unfilteredRoot != nil { + fileNode = r.unfilteredRoot + logger.Debug().Str("version", r.version.Name).Msg("recovered distribution files from unfiltered tree") } files, err := tree.GetAllFiles(fileNode) if err != nil { logger.Error().Err(err).Msg("Failed to get files from tree") return fmt.Errorf("get files from tree failed: %w", err) } + // Resolve the opt-in file selector (packageFilters[].files) for this package once. + fileSel, fileHasFilters, fileMatched := util.SelectorForPackage(r.mapping, r.pkg.Name) for _, file := range files { + // Skip files not named by the package file selector. No-op when there is + // no matching selector or the selector's files list is empty (all files). + if fileHasFilters && fileMatched && !util.FileSelectedBySelector(fileSel, file.Name) { + logger.Debug().Msgf("Skipping file %s: not selected by package filter", file.Name) + continue + } // For NPM, skip files that don't have .tgz extension if r.artifactType == types.NPM && !strings.HasSuffix(file.Name, ".tgz") { logger.Debug().Msgf("Skipping non-tgz file %s for NPM migration", file.Name) @@ -191,6 +206,40 @@ func (r *Version) Migrate(ctx context.Context) error { continue } } + // For RUBY, skip files that don't match current gem and version. + if r.artifactType == types.RUBY { + if !strings.HasSuffix(file.Name, ".gem") { + logger.Debug().Msgf("Skipping non-gem file %s for RUBY migration", file.Name) + continue + } + meta, ok := util.ParseRubyGemFileNameWithPath(file.Uri) + if !ok || meta.Name != r.pkg.Name || meta.Version != r.version.Name { + logger.Debug().Msgf("Skipping file %s (gem=%s, ver=%s) - doesn't match current package %s version %s", + file.Name, meta.Name, meta.Version, r.pkg.Name, r.version.Name) + continue + } + } + // For TERRAFORM, skip files that don't belong to this package+version. + if r.artifactType == types.TERRAFORM { + if util.IsTerraformModule(file.Uri) { + ns, name, provider, version, ok := util.ParseTerraformModulePath(file.Uri) + if !ok || ns+"/"+name+"/"+provider != r.pkg.Name || version != r.version.Name { + logger.Debug().Msgf("Skipping terraform module file %s - doesn't match package %s version %s", + file.Uri, r.pkg.Name, r.version.Name) + continue + } + } else if util.IsTerraformProvider(file.Uri) { + ns, typeName, version, _, _, _, ok := util.ParseTerraformProviderPath(file.Uri) + if !ok || ns+"/"+typeName != r.pkg.Name || version != r.version.Name { + logger.Debug().Msgf("Skipping terraform provider file %s - doesn't match package %s version %s", + file.Uri, r.pkg.Name, r.version.Name) + continue + } + } else { + logger.Debug().Msgf("Skipping terraform file %s - unrecognised path shape", file.Uri) + continue + } + } // Check if file already exists in destination (index takes priority). // The index is keyed on source-relative paths (types.File.Uri); HasFile // owns the HAR-path-to-source-path conversion, so query it with the @@ -215,6 +264,7 @@ func (r *Version) Migrate(ctx context.Context) error { Uri: file.Uri, Size: int64(file.Size), Status: types.StatusSkip, + Reason: types.SkipReasonAlreadyExists, } r.stats.FileStats = append(r.stats.FileStats, stat) continue diff --git a/modules/har/pkg/har/migrate/migratable/version_test.go b/modules/har/pkg/har/migrate/migratable/version_test.go index d2bd18da..37382622 100644 --- a/modules/har/pkg/har/migrate/migratable/version_test.go +++ b/modules/har/pkg/har/migrate/migratable/version_test.go @@ -175,6 +175,9 @@ func TestVersionMigrateSkipsFileByUriNotBasename(t *testing.T) { if s.Uri != nestedURI { t.Errorf("skip stat for unexpected file %q, want %q", s.Uri, nestedURI) } + if s.Reason != types.SkipReasonAlreadyExists { + t.Errorf("Reason = %q, want %q", s.Reason, types.SkipReasonAlreadyExists) + } case types.StatusSuccess: uploaded++ if s.Uri != rootURI { diff --git a/modules/har/pkg/har/migrate/migration.go b/modules/har/pkg/har/migrate/migration.go index 65384b83..04a53aa7 100644 --- a/modules/har/pkg/har/migrate/migration.go +++ b/modules/har/pkg/har/migrate/migration.go @@ -3,6 +3,7 @@ package migrate import ( "context" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -78,23 +79,55 @@ func (m *MigrationService) Run(ctx context.Context) error { } eng := engine.NewEngine(m.config.Concurrency, jobs) - err := eng.Execute(ctx) - if err != nil { - logger.Error().Err(err).Msgf("Engine execution saw following errors: %v", err) + engineErr := eng.Execute(ctx) + if engineErr != nil { + logger.Error().Err(engineErr).Msgf("Engine execution saw following errors: %v", engineErr) } logger.Info().Msg("Migration process completed") + // Handle dry-run output. Dry-run behavior is intentionally unchanged by the + // exit-code contract below: a dry-run never mutates anything, so engine + // errors there do not fail the process. if m.config.DryRun { return m.writeDryRunOutput(logger) } - printFileStats(transferStats.FileStats) + if m.config.Summary { + printSummary(transferStats.FileStats) + } else { + printFileStats(transferStats.FileStats) + } if jsonData, err := json.MarshalIndent(transferStats.FileStats, "", " "); err == nil { logger.Info().RawJSON("file_stats", jsonData).Int("total_files", len(transferStats.FileStats)).Msg("Migration file statistics") } - return nil + // Machine-readable per-coordinate result file (opt-in) — written BEFORE the + // exit-code decision so automation gets the full picture even on failure. + if m.config.ResultFile != "" { + if err := writeResultFile(m.config.ResultFile, transferStats.FileStats); err != nil { + logger.Error().Err(err).Str("path", m.config.ResultFile).Msg("Failed to write result file") + engineErr = errors.Join(engineErr, fmt.Errorf("write result file: %w", err)) + } else { + logger.Info().Str("path", m.config.ResultFile).Int("records", len(transferStats.FileStats)). + Msg("Wrote per-coordinate result file") + } + } + + // Exit-code contract: a migration with ANY failure fails the process — no + // opt-out. Failures are (a) engine-level errors (enumeration aborts, job + // panics) or (b) any per-coordinate StatusFail stat. + failed := 0 + for _, fs := range transferStats.FileStats { + if fs.Status == types.StatusFail { + failed++ + } + } + if failed > 0 { + engineErr = errors.Join(engineErr, fmt.Errorf("%d of %d artifact(s) failed to migrate", failed, len(transferStats.FileStats))) + } + + return engineErr } func printFileStats(stats []types.FileStat) { @@ -107,6 +140,42 @@ func printFileStats(stats []types.FileStat) { tw.Render() } +func printSummary(stats []types.FileStat) { + counts := make(map[types.Status]int) + for _, s := range stats { + counts[s.Status]++ + } + + fmt.Println("\nMigration Summary of total files finalized for upload :") + fmt.Printf(" %-10s %d\n", "Success :", counts[types.StatusSuccess]) + fmt.Printf(" %-10s %d\n", "Skipped :", counts[types.StatusSkip]) + fmt.Printf(" %-10s %d\n", "Failed :", counts[types.StatusFail]) + fmt.Printf(" %-10s %d\n", "Total :", len(stats)) +} + +// writeResultFile writes one JSON object per FileStat (JSON-lines) to path, +// creating parent directories as needed. +func writeResultFile(path string, fileStats []types.FileStat) error { + if dir := filepath.Dir(path); dir != "." && dir != "" { + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("create result file directory: %w", err) + } + } + f, err := os.Create(path) + if err != nil { + return fmt.Errorf("create result file: %w", err) + } + defer f.Close() + + enc := json.NewEncoder(f) + for _, fs := range fileStats { + if err := enc.Encode(fs); err != nil { + return fmt.Errorf("encode result record: %w", err) + } + } + return nil +} + func (m *MigrationService) writeDryRunOutput(logger zerolog.Logger) error { timestamp := time.Now().Format("20060102_150405") outputDir := "dry-run-output" diff --git a/modules/har/pkg/har/migrate/migration_exitcode_test.go b/modules/har/pkg/har/migrate/migration_exitcode_test.go new file mode 100644 index 00000000..5e0c0023 --- /dev/null +++ b/modules/har/pkg/har/migrate/migration_exitcode_test.go @@ -0,0 +1,229 @@ +package migrate + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "sync" + "testing" + + "github.com/harness/cli/modules/har/pkg/har/migrate/adapter/jfrog" + "github.com/harness/cli/modules/har/pkg/har/migrate/adapter/mock_jfrog" + "github.com/harness/cli/modules/har/pkg/har/migrate/types" + + "github.com/google/go-containerregistry/pkg/authn" +) + +// fakeDestAdapter is a destination Adapter that records uploads and can be +// instructed to fail (or already-exist) per URI. Source-side methods are +// unused and return zero values. +type fakeDestAdapter struct { + mu sync.Mutex + uploads []string // URIs successfully uploaded + failWith map[string]error // uri -> error to return from UploadFile + failAll error // when non-nil, every upload fails +} + +func (f *fakeDestAdapter) GetKeyChain(string) (authn.Keychain, error) { return nil, nil } +func (f *fakeDestAdapter) GetConfig() types.RegistryConfig { return types.RegistryConfig{} } +func (f *fakeDestAdapter) ValidateCredentials() (bool, error) { return true, nil } +func (f *fakeDestAdapter) GetRegistry(context.Context, string) (types.RegistryInfo, error) { + return types.RegistryInfo{Type: "HAR", Path: "dst-reg"}, nil +} +func (f *fakeDestAdapter) CreateRegistryIfDoesntExist(string) (bool, error) { return false, nil } +func (f *fakeDestAdapter) GetPackages(string, types.ArtifactType, *types.TreeNode) ([]types.Package, error) { + return nil, nil +} +func (f *fakeDestAdapter) GetVersions(types.Package, *types.TreeNode, string, string, types.ArtifactType) ([]types.Version, error) { + return nil, nil +} +func (f *fakeDestAdapter) GetFiles(string) ([]types.File, error) { return nil, nil } +func (f *fakeDestAdapter) SearchFiles(string) ([]types.SearchedFile, error) { + return nil, nil +} +func (f *fakeDestAdapter) DownloadFile(string, string) (io.ReadCloser, http.Header, error) { + return nil, nil, errors.New("fakeDestAdapter: DownloadFile not supported") +} +func (f *fakeDestAdapter) UploadFile(_ string, _ io.ReadCloser, file *types.File, _ http.Header, _, _ string, _ types.ArtifactType, _ map[string]interface{}) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.failAll != nil { + return f.failAll + } + if err, ok := f.failWith[file.Uri]; ok { + return err + } + f.uploads = append(f.uploads, file.Uri) + return nil +} +func (f *fakeDestAdapter) GetOCIImagePath(string, string, string) (string, error) { return "", nil } +func (f *fakeDestAdapter) AddNPMTag(string, string, string, string) error { return nil } +func (f *fakeDestAdapter) VersionExists(context.Context, types.Package, string, string, string, types.ArtifactType) (bool, error) { + return false, nil +} +func (f *fakeDestAdapter) FileExists(context.Context, string, string, string, *types.File, types.ArtifactType) (bool, error) { + return false, nil +} +func (f *fakeDestAdapter) GetAllFilesForVersion(context.Context, string, string, string) ([]string, error) { + return nil, nil +} +func (f *fakeDestAdapter) BuildExistingIndex(context.Context, string, int) (*types.ExistingIndex, error) { + return nil, nil +} +func (f *fakeDestAdapter) CreateVersion(string, string, string, types.ArtifactType, []*types.PackageFiles, map[string]interface{}) error { + return nil +} + +func (f *fakeDestAdapter) uploadedURIs() []string { + f.mu.Lock() + defer f.mu.Unlock() + out := make([]string, len(f.uploads)) + copy(out, f.uploads) + return out +} + +// newMockBackedService builds a MigrationService with the real jfrog adapter +// backed by the mock client as source and the given fake destination. +func newMockBackedService(cfg *types.Config, dest *fakeDestAdapter) *MigrationService { + src := jfrog.NewAdapterWithClient( + types.RegistryConfig{Type: types.MOCK_JFROG, Endpoint: "http://mock"}, + mock_jfrog.NewMockClient(), + ) + return &MigrationService{config: cfg, source: src, destination: dest} +} + +func baseMapping(at types.ArtifactType, srcReg string) types.RegistryMapping { + return types.RegistryMapping{ + ArtifactType: at, + SourceRegistry: srcReg, + DestinationRegistry: "dst-reg", + } +} + +// TestRunReturnsErrorOnUploadFailures verifies bug 1: per-coordinate upload +// failures are recorded as StatusFail stats, and Run now fails the process +// instead of always returning nil. +func TestRunReturnsErrorOnUploadFailures(t *testing.T) { + cfg := &types.Config{ + Concurrency: 1, + Overwrite: true, + Mappings: []types.RegistryMapping{baseMapping(types.NUGET, "nuget-local")}, + } + dest := &fakeDestAdapter{failAll: errors.New("boom: destination unavailable")} + svc := newMockBackedService(cfg, dest) + + err := svc.Run(context.Background()) + if err == nil { + t.Fatal("expected non-nil error when uploads fail, got nil") + } + if !strings.Contains(err.Error(), "failed to migrate") { + t.Errorf("error %q does not mention failed artifact count", err.Error()) + } +} + +// TestRunSucceedsOnCleanMigration is the corresponding happy-path check: a +// migration with no failures still returns nil (bug 1's fix must not make +// successful runs fail). +func TestRunSucceedsOnCleanMigration(t *testing.T) { + cfg := &types.Config{ + Concurrency: 1, + Overwrite: true, + Mappings: []types.RegistryMapping{baseMapping(types.NUGET, "nuget-local")}, + } + dest := &fakeDestAdapter{} + svc := newMockBackedService(cfg, dest) + + if err := svc.Run(context.Background()); err != nil { + t.Fatalf("expected nil error for clean migration, got: %v", err) + } + // nuget-local: 1.0.0.nupkg, 2.0.0.nupkg, 2.0.0.snupkg parse as package files. + uploads := dest.uploadedURIs() + if len(uploads) != 3 { + t.Fatalf("expected 3 uploads, got %d: %v", len(uploads), uploads) + } +} + +// TestRunPackageFiltersVersionSelector verifies bug 2 end-to-end: a +// packageFilters entry naming a specific version narrows the migrated files +// to that version only. +func TestRunPackageFiltersVersionSelector(t *testing.T) { + cfg := &types.Config{ + Concurrency: 1, + Overwrite: true, + Mappings: []types.RegistryMapping{{ + ArtifactType: types.NUGET, + SourceRegistry: "nuget-local", + DestinationRegistry: "dst-reg", + PackageFilters: []types.PackageSelector{ + {Package: "company.grpc.pkg", Versions: []string{"1.0.0"}}, + }, + }}, + } + dest := &fakeDestAdapter{} + svc := newMockBackedService(cfg, dest) + + if err := svc.Run(context.Background()); err != nil { + t.Fatalf("expected nil error, got: %v", err) + } + uploads := dest.uploadedURIs() + if len(uploads) != 1 || !strings.Contains(uploads[0], "1.0.0") { + t.Fatalf("expected only the 1.0.0 version to upload, got: %v", uploads) + } +} + +// TestRunPackageFiltersFileSelector verifies bug 2 end-to-end at file +// granularity: a packageFilters entry naming a specific file narrows the +// migrated files to that file only, across versions. +func TestRunPackageFiltersFileSelector(t *testing.T) { + cfg := &types.Config{ + Concurrency: 1, + Overwrite: true, + Mappings: []types.RegistryMapping{{ + ArtifactType: types.NUGET, + SourceRegistry: "nuget-local", + DestinationRegistry: "dst-reg", + PackageFilters: []types.PackageSelector{ + {Package: "company.grpc.pkg", Files: []string{"company.grpc.pkg.2.0.0.snupkg"}}, + }, + }}, + } + dest := &fakeDestAdapter{} + svc := newMockBackedService(cfg, dest) + + if err := svc.Run(context.Background()); err != nil { + t.Fatalf("expected nil error, got: %v", err) + } + uploads := dest.uploadedURIs() + if len(uploads) != 1 || !strings.Contains(uploads[0], "snupkg") { + t.Fatalf("expected only the .snupkg file to upload, got: %v", uploads) + } +} + +// TestRunPackageFiltersUnmatchedPackageMigratesNothing verifies bug 2: when +// packageFilters is set and the source package is not named in it, nothing +// migrates (the allow-list blocks it). +func TestRunPackageFiltersUnmatchedPackageMigratesNothing(t *testing.T) { + cfg := &types.Config{ + Concurrency: 1, + Overwrite: true, + Mappings: []types.RegistryMapping{{ + ArtifactType: types.NUGET, + SourceRegistry: "nuget-local", + DestinationRegistry: "dst-reg", + PackageFilters: []types.PackageSelector{ + {Package: "some-other-package"}, + }, + }}, + } + dest := &fakeDestAdapter{} + svc := newMockBackedService(cfg, dest) + + if err := svc.Run(context.Background()); err != nil { + t.Fatalf("expected nil error, got: %v", err) + } + if uploads := dest.uploadedURIs(); len(uploads) != 0 { + t.Fatalf("expected no uploads, got: %v", uploads) + } +} diff --git a/modules/har/pkg/har/migrate/migration_resultfile_test.go b/modules/har/pkg/har/migrate/migration_resultfile_test.go new file mode 100644 index 00000000..25558f79 --- /dev/null +++ b/modules/har/pkg/har/migrate/migration_resultfile_test.go @@ -0,0 +1,136 @@ +package migrate + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/harness/cli/modules/har/pkg/har/migrate/types" +) + +func readResultFile(t *testing.T, path string) []types.FileStat { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read result file: %v", err) + } + var records []types.FileStat + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + if line == "" { + continue + } + var fs types.FileStat + if err := json.Unmarshal([]byte(line), &fs); err != nil { + t.Fatalf("result file line is not valid JSON: %q: %v", line, err) + } + records = append(records, fs) + } + return records +} + +// TestRunWritesResultFile verifies the happy path: the JSON-lines result file +// reconciles exactly with the in-memory stats. +func TestRunWritesResultFile(t *testing.T) { + resultPath := filepath.Join(t.TempDir(), "result.jsonl") + cfg := &types.Config{ + Concurrency: 2, + Overwrite: true, + ResultFile: resultPath, + Mappings: []types.RegistryMapping{baseMapping(types.NUGET, "nuget-local")}, + } + dest := &fakeDestAdapter{} + svc := newMockBackedService(cfg, dest) + + if err := svc.Run(context.Background()); err != nil { + t.Fatalf("expected nil error for clean migration, got: %v", err) + } + + // nuget-local: 1.0.0.nupkg, 2.0.0.nupkg, 2.0.0.snupkg parse as package files. + uploads := dest.uploadedURIs() + if len(uploads) != 3 { + t.Fatalf("expected 3 uploads, got %d: %v", len(uploads), uploads) + } + + records := readResultFile(t, resultPath) + if len(records) != 3 { + t.Fatalf("expected 3 result records, got %d", len(records)) + } + for _, r := range records { + if r.Status != types.StatusSuccess { + t.Errorf("expected all Success in result file, got %+v", r) + } + } +} + +// TestRunWritesResultFileOnFailure verifies the result file exists (and +// includes MkdirAll'd parent directories) even when the migration fails. +func TestRunWritesResultFileOnFailure(t *testing.T) { + resultPath := filepath.Join(t.TempDir(), "nested", "result.jsonl") + cfg := &types.Config{ + Concurrency: 1, + Overwrite: true, + ResultFile: resultPath, + Mappings: []types.RegistryMapping{baseMapping(types.NUGET, "nuget-local")}, + } + dest := &fakeDestAdapter{failAll: errors.New("boom")} + svc := newMockBackedService(cfg, dest) + + if err := svc.Run(context.Background()); err == nil { + t.Fatal("expected non-nil error, got nil") + } + + records := readResultFile(t, resultPath) + if len(records) != 3 { + t.Fatalf("expected 3 result records, got %d", len(records)) + } + for _, r := range records { + if r.Status != types.StatusFail { + t.Errorf("expected all Failed in result file, got %+v", r) + } + } +} + +// TestRunResultFileRecordsSkipReason verifies a 409-style already-exists +// upload error is recorded in the result file as Skipped with +// reason=already_exists, and does not fail the run on its own. +func TestRunResultFileRecordsSkipReason(t *testing.T) { + resultPath := filepath.Join(t.TempDir(), "result.jsonl") + cfg := &types.Config{ + Concurrency: 1, + Overwrite: true, + ResultFile: resultPath, + Mappings: []types.RegistryMapping{baseMapping(types.NUGET, "nuget-local")}, + } + dest := &fakeDestAdapter{failWith: map[string]error{ + "/foo/company.grpc.pkg/1.0.0/company.grpc.pkg.1.0.0.nupkg": types.ErrArtifactAlreadyExists, + }} + svc := newMockBackedService(cfg, dest) + + if err := svc.Run(context.Background()); err != nil { + t.Fatalf("expected nil error when only already-exists skips occur, got: %v", err) + } + + records := readResultFile(t, resultPath) + success, skipped := 0, 0 + for _, r := range records { + switch r.Status { + case types.StatusSuccess: + success++ + case types.StatusSkip: + skipped++ + if r.Reason != types.SkipReasonAlreadyExists { + t.Errorf("skip record missing reason %q: %+v", types.SkipReasonAlreadyExists, r) + } + } + } + if skipped != 1 { + t.Errorf("expected 1 skipped record, got %d", skipped) + } + if success != 2 { + t.Errorf("expected 2 success records, got %d", success) + } +} diff --git a/modules/har/pkg/har/migrate/migration_summary_test.go b/modules/har/pkg/har/migrate/migration_summary_test.go new file mode 100644 index 00000000..03f4a403 --- /dev/null +++ b/modules/har/pkg/har/migrate/migration_summary_test.go @@ -0,0 +1,89 @@ +package migrate + +import ( + "bytes" + "context" + "io" + "os" + "strings" + "testing" + + "github.com/harness/cli/modules/har/pkg/har/migrate/types" +) + +// TestRunSummaryModeSucceeds verifies the --summary flag path (Config.Summary) +// takes the printSummary branch instead of the full table and still completes +// a clean migration without error. +func TestRunSummaryModeSucceeds(t *testing.T) { + cfg := &types.Config{ + Concurrency: 1, + Overwrite: true, + Summary: true, + Mappings: []types.RegistryMapping{baseMapping(types.NUGET, "nuget-local")}, + } + dest := &fakeDestAdapter{} + svc := newMockBackedService(cfg, dest) + + stdout := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("os.Pipe() error: %v", err) + } + os.Stdout = w + + runErr := svc.Run(context.Background()) + + w.Close() + os.Stdout = stdout + var buf bytes.Buffer + _, _ = io.Copy(&buf, r) + + if runErr != nil { + t.Fatalf("expected nil error for clean migration, got: %v", runErr) + } + out := buf.String() + if !strings.Contains(out, "Migration Summary") { + t.Errorf("stdout %q does not contain summary header", out) + } + if !strings.Contains(out, "Success :") || !strings.Contains(out, "Total :") { + t.Errorf("stdout %q missing expected summary lines", out) + } +} + +// TestPrintSummaryCounts verifies printSummary tallies each status correctly. +func TestPrintSummaryCounts(t *testing.T) { + stats := []types.FileStat{ + {Status: types.StatusSuccess}, + {Status: types.StatusSuccess}, + {Status: types.StatusSkip}, + {Status: types.StatusFail}, + } + + stdout := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("os.Pipe() error: %v", err) + } + os.Stdout = w + + printSummary(stats) + + w.Close() + os.Stdout = stdout + var buf bytes.Buffer + _, _ = io.Copy(&buf, r) + out := buf.String() + + if !strings.Contains(out, "Success : 2") { + t.Errorf("stdout %q missing success count of 2", out) + } + if !strings.Contains(out, "Skipped : 1") { + t.Errorf("stdout %q missing skipped count of 1", out) + } + if !strings.Contains(out, "Failed : 1") { + t.Errorf("stdout %q missing failed count of 1", out) + } + if !strings.Contains(out, "Total : 4") { + t.Errorf("stdout %q missing total count of 4", out) + } +} diff --git a/modules/har/pkg/har/migrate/types/config.go b/modules/har/pkg/har/migrate/types/config.go index 94949f91..ad721bbe 100644 --- a/modules/har/pkg/har/migrate/types/config.go +++ b/modules/har/pkg/har/migrate/types/config.go @@ -3,6 +3,7 @@ package types import ( "fmt" "os" + "strings" "time" "github.com/pterm/pterm" @@ -42,14 +43,67 @@ var ( CONAN ArtifactType = "CONAN" PUPPET ArtifactType = "PUPPET" HELM_HTTP ArtifactType = "HELM_HTTP" + RUBY ArtifactType = "RUBY" + CRAN ArtifactType = "CRAN" + TERRAFORM ArtifactType = "TERRAFORM" ) +// knownArtifactTypesList is the exhaustive, ordered list of valid ArtifactType +// values and the SINGLE SOURCE OF TRUTH for "which types exist": config +// validation and the validation error message are both derived from it. Add +// new types here (and to the var block above) whenever a new ArtifactType is +// introduced. +var knownArtifactTypesList = []ArtifactType{ + DOCKER, HELM, HELM_LEGACY, GENERIC, PYTHON, MAVEN, NPM, NUGET, + RPM, GO, CONDA, COMPOSER, DART, RAW, SWIFT, DEBIAN, CONAN, PUPPET, HELM_HTTP, RUBY, CRAN, + TERRAFORM, +} + +// knownArtifactTypes is the lookup set derived from knownArtifactTypesList. +var knownArtifactTypes = func() map[ArtifactType]struct{} { + m := make(map[ArtifactType]struct{}, len(knownArtifactTypesList)) + for _, t := range knownArtifactTypesList { + m[t] = struct{}{} + } + return m +}() + +// KnownArtifactTypes returns the ordered list of all valid ArtifactType +// values. Safe to mutate by the caller (a fresh copy is returned each time). +func KnownArtifactTypes() []ArtifactType { + out := make([]ArtifactType, len(knownArtifactTypesList)) + copy(out, knownArtifactTypesList) + return out +} + +// KnownArtifactTypesString returns the valid types as a single +// comma-separated string for error messages and help text. +func KnownArtifactTypesString() string { + parts := make([]string, len(knownArtifactTypesList)) + for i, t := range knownArtifactTypesList { + parts[i] = string(t) + } + return strings.Join(parts, ", ") +} + +// IsKnownArtifactType reports whether t is a recognised ArtifactType value. +func IsKnownArtifactType(t ArtifactType) bool { + _, ok := knownArtifactTypes[t] + return ok +} + // Config represents the top-level configuration structure type Config struct { Version string `yaml:"version"` Concurrency int `yaml:"concurrency"` Overwrite bool `yaml:"overwrite"` DryRun bool `yaml:"dryRun"` + // Summary, when true, prints condensed per-status counts instead of the + // full per-file table. + Summary bool `yaml:"summary"` + // ResultFile, when set, is a path to write one JSON-lines record per + // per-coordinate result (types.FileStat) for automation to consume. + ResultFile string `yaml:"resultFile,omitempty"` Source RegistryConfig `yaml:"source"` Dest RegistryConfig `yaml:"destination"` Mappings []RegistryMapping `yaml:"mappings"` @@ -95,8 +149,9 @@ type RegistryMapping struct { IncludePatterns []string `yaml:"includePatterns"` ExcludePatterns []string `yaml:"excludePatterns"` //Optional - SourcePackageHostname string `yaml:"sourcePackageHostname"` - DateFilter *DateFilter `yaml:"dateFilter"` + SourcePackageHostname string `yaml:"sourcePackageHostname"` + DateFilter *DateFilter `yaml:"dateFilter"` + PackageFilters []PackageSelector `yaml:"packageFilters,omitempty"` } // CredentialsConfig defines the credential configuration @@ -166,6 +221,12 @@ func validateConfig(config *Config) error { if mapping.DestinationRegistry == "" { return fmt.Errorf("mapping %d: destination registry cannot be empty", i) } + if !IsKnownArtifactType(mapping.ArtifactType) { + return fmt.Errorf("mapping %d: unknown artifactType %q — valid values are: %s", i, mapping.ArtifactType, KnownArtifactTypesString()) + } + if err := ValidatePackageFilters(mapping.PackageFilters, mapping.ArtifactType); err != nil { + return fmt.Errorf("mapping %d: %w", i, err) + } if mapping.ArtifactType == MAVEN && mapping.DateFilter != nil { msg := fmt.Sprintf("mapping %d: date filter is enabled for %s — maven-metadata.xml may not be in sync with the migrated artifacts", i, MAVEN) log.Warn().Msg(msg) diff --git a/modules/har/pkg/har/migrate/types/config_validation_test.go b/modules/har/pkg/har/migrate/types/config_validation_test.go new file mode 100644 index 00000000..acd1b525 --- /dev/null +++ b/modules/har/pkg/har/migrate/types/config_validation_test.go @@ -0,0 +1,170 @@ +package types + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// baseValidConfig returns a minimal Config that passes validateConfig, so tests +// can tweak a single field to exercise one rule at a time. +func baseValidConfig() *Config { + cred := CredentialsConfig{Username: "user", Password: "pass"} + return &Config{ + Concurrency: 1, + Source: RegistryConfig{Endpoint: "https://src.example", Type: JFROG, Credentials: cred}, + Dest: RegistryConfig{Endpoint: "https://dst.example", Type: HAR, Credentials: cred}, + Mappings: []RegistryMapping{ + { + ArtifactType: MAVEN, + SourceRegistry: "src", + DestinationRegistry: "dst", + }, + }, + } +} + +func TestValidateConfig_MavenWithDateFilterWarnsButPasses(t *testing.T) { + config := baseValidConfig() + after := time.Unix(0, 0) + config.Mappings[0].DateFilter = &DateFilter{ + Match: DateFilterMatchAny, + CreatedAfter: &after, + } + + if err := validateConfig(config); err != nil { + t.Fatalf("expected MAVEN mapping with date filter to pass with a warning, got: %v", err) + } +} + +func TestValidateConfig_UnknownArtifactTypeRejected(t *testing.T) { + config := baseValidConfig() + config.Mappings[0].ArtifactType = ArtifactType("NOTAREALTYPE") + + err := validateConfig(config) + if err == nil { + t.Fatal("expected error for unknown artifactType, got nil") + } + if !strings.Contains(err.Error(), "NOTAREALTYPE") { + t.Errorf("error should name the offending type, got: %v", err) + } +} + +func TestValidateConfig_AllKnownArtifactTypesAccepted(t *testing.T) { + for _, at := range KnownArtifactTypes() { + config := baseValidConfig() + config.Mappings[0].ArtifactType = at + if err := validateConfig(config); err != nil { + t.Errorf("expected known artifactType %q to pass, got: %v", at, err) + } + } +} + +func TestValidateConfig_EmptyArtifactTypeRejected(t *testing.T) { + config := baseValidConfig() + config.Mappings[0].ArtifactType = ArtifactType("") + + err := validateConfig(config) + if err == nil { + t.Fatal("expected error for empty artifactType, got nil") + } +} + +// TestLoadConfig_UnknownArtifactTypeFailsAtLoad verifies a typo'd artifactType +// fails at config LOAD time — i.e. before NewMigrationService and therefore +// before any source API call — and the error lists the valid values. +func TestLoadConfig_UnknownArtifactTypeFailsAtLoad(t *testing.T) { + yaml := ` +version: 1.0.0 +concurrency: 1 +source: + endpoint: https://src.example + type: JFROG + credentials: {username: u, password: p} +destination: + endpoint: https://dst.example + type: HAR + credentials: {username: u, password: p} +mappings: + - artifactType: generic + sourceRegistry: src + destinationRegistry: dst +` + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(yaml), 0644); err != nil { + t.Fatal(err) + } + + _, err := LoadConfig(path) + if err == nil { + t.Fatal("expected LoadConfig to reject lowercase artifactType, got nil") + } + if !strings.Contains(err.Error(), "generic") { + t.Errorf("error should name the offending type, got: %v", err) + } + if !strings.Contains(err.Error(), "GENERIC") { + t.Errorf("error should list valid values including GENERIC, got: %v", err) + } +} + +// TestKnownArtifactTypesSingleSource verifies the exported list and the lookup +// map stay in sync. +func TestKnownArtifactTypesSingleSource(t *testing.T) { + list := KnownArtifactTypes() + if len(list) != len(knownArtifactTypes) { + t.Fatalf("KnownArtifactTypes() len %d != map len %d", len(list), len(knownArtifactTypes)) + } + for _, at := range list { + if !IsKnownArtifactType(at) { + t.Errorf("KnownArtifactTypes() contains %q but IsKnownArtifactType says false", at) + } + } +} + +// TestValidateConfig_PackageFiltersRejectedBeyondGranularity verifies §2: +// packageFilters requesting a granularity beyond what the artifact type +// supports is a config-load-time error, not a silent no-op. +func TestValidateConfig_PackageFiltersRejectedBeyondGranularity(t *testing.T) { + config := baseValidConfig() + config.Mappings[0].ArtifactType = DOCKER // package-only granularity + config.Mappings[0].PackageFilters = []PackageSelector{ + {Package: "my-image", Versions: []string{"1.0.0"}}, + } + + err := validateConfig(config) + if err == nil { + t.Fatal("expected error for version-level packageFilters on DOCKER, got nil") + } + + config = baseValidConfig() + config.Mappings[0].ArtifactType = GO // version granularity, no file granularity + config.Mappings[0].PackageFilters = []PackageSelector{ + {Package: "my-module", Files: []string{"v1.0.0.zip"}}, + } + if err := validateConfig(config); err == nil { + t.Fatal("expected error for file-level packageFilters on GO, got nil") + } +} + +func TestValidateConfig_PackageFiltersAcceptedWithinGranularity(t *testing.T) { + config := baseValidConfig() + config.Mappings[0].ArtifactType = NUGET // file granularity + config.Mappings[0].PackageFilters = []PackageSelector{ + {Package: "my-package", Versions: []string{"1.0.0"}, Files: []string{"my-package.1.0.0.nupkg"}}, + } + + if err := validateConfig(config); err != nil { + t.Fatalf("expected file-level packageFilters on NUGET to pass, got: %v", err) + } +} + +func TestValidateConfig_PackageFiltersEmptyPackageNameRejected(t *testing.T) { + config := baseValidConfig() + config.Mappings[0].PackageFilters = []PackageSelector{{Package: ""}} + + if err := validateConfig(config); err == nil { + t.Fatal("expected error for empty package name in packageFilters, got nil") + } +} diff --git a/modules/har/pkg/har/migrate/types/selector.go b/modules/har/pkg/har/migrate/types/selector.go new file mode 100644 index 00000000..e8d57124 --- /dev/null +++ b/modules/har/pkg/har/migrate/types/selector.go @@ -0,0 +1,63 @@ +package types + +import "fmt" + +// PackageSelector defines an opt-in allow-list for packages, versions, and files. +// Empty Versions means all versions of that package. +// Empty Files means all files of a selected version. +// Package, Versions, and Files are all matched case-insensitively. +type PackageSelector struct { + Package string `yaml:"package"` + Versions []string `yaml:"versions,omitempty"` + Files []string `yaml:"files,omitempty"` +} + +// SelectorGranularity represents the level of filtering supported by an artifact type. +// Values are ordered so < comparisons work (Package < Version < File). +type SelectorGranularity int + +const ( + GranularityPackage SelectorGranularity = iota + GranularityVersion + GranularityFile +) + +// SupportedSelectorGranularity returns the maximum filtering granularity +// supported by the given artifact type. +func SupportedSelectorGranularity(t ArtifactType) SelectorGranularity { + switch t { + case GO: + return GranularityVersion + case GENERIC, RAW, MAVEN, PYTHON, NUGET, NPM, DART, PUPPET, RUBY: + return GranularityFile + default: + // DOCKER, HELM, HELM_LEGACY, HELM_HTTP, RPM, DEBIAN, CONDA, COMPOSER, SWIFT, CONAN, CRAN + return GranularityPackage + } +} + +// ValidatePackageFilters validates that the provided package filters are compatible +// with the artifact type's supported granularity. +func ValidatePackageFilters(filters []PackageSelector, t ArtifactType) error { + if len(filters) == 0 { + return nil + } + + g := SupportedSelectorGranularity(t) + + for i, selector := range filters { + if selector.Package == "" { + return fmt.Errorf("packageFilters[%d]: package name must not be empty", i) + } + + if len(selector.Versions) > 0 && g < GranularityVersion { + return fmt.Errorf("packageFilters[%d]: artifact type %s does not support version-level filtering", i, t) + } + + if len(selector.Files) > 0 && g < GranularityFile { + return fmt.Errorf("packageFilters[%d]: artifact type %s does not support file-level filtering", i, t) + } + } + + return nil +} diff --git a/modules/har/pkg/har/migrate/types/types.go b/modules/har/pkg/har/migrate/types/types.go index 8b84a77a..3db59067 100644 --- a/modules/har/pkg/har/migrate/types/types.go +++ b/modules/har/pkg/har/migrate/types/types.go @@ -71,6 +71,19 @@ const ( StatusFail Status = "Failed" ) +// Skip reasons recorded in FileStat.Reason when Status == StatusSkip, so +// reconciliation tooling can tell WHY a coordinate was skipped instead of +// treating every skip alike. +const ( + // SkipReasonAlreadyExists marks skips caused by the destination already + // holding the coordinate (destination index hit, HEAD 200, or an + // idempotent upload conflict). Safe to ignore on re-run. + SkipReasonAlreadyExists = "already_exists" + // SkipReasonNoContent marks skips where the source coordinate resolved to + // nothing migratable (e.g. an OCI repository with no tags). + SkipReasonNoContent = "no_content" +) + type FileStat struct { Name string Registry string @@ -78,6 +91,9 @@ type FileStat struct { Status Status Size int64 Error string + // Reason classifies StatusSkip rows (see SkipReason* constants). Empty for + // non-skip rows and for skips recorded before a reason was set. + Reason string } type TransferStats struct { diff --git a/modules/har/pkg/har/migrate/util/cran_util.go b/modules/har/pkg/har/migrate/util/cran_util.go new file mode 100644 index 00000000..afe53d72 --- /dev/null +++ b/modules/har/pkg/har/migrate/util/cran_util.go @@ -0,0 +1,379 @@ +package util + +import ( + "fmt" + "path" + "regexp" + "strings" + + "github.com/harness/cli/modules/har/pkg/har/migrate/types" +) + +const ( + cranContribDir = "contrib" + cranArchiveDir = "Archive" + cranSrcContribRepoPath = "src/contrib" + cranSourceExt = ".tar.gz" + cranWindowsExt = ".zip" + cranMacOSExt = ".tgz" + cranOSWindows = "windows" + cranOSMacOS = "macosx" +) + +var ( + // cranRVersionRegex matches the "major.minor" R version used in contrib directory names (e.g. 4.4). + cranRVersionRegex = regexp.MustCompile(`^[0-9]+\.[0-9]+$`) + // cranMacFlavorRegex validates a macOS build flavor directory such as "big-sur-arm64". + cranMacFlavorRegex = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]*$`) + // cranPackageNameRegex validates an R package name (letters, digits and dots; must start with a + // letter). R package names never contain underscores, which is what lets us split the file + // name on the first underscore. + cranPackageNameRegex = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9.]*$`) + + // nonMacOSBinRoots excludes known non-mac roots (e.g. "linux") from being misidentified + // as a mac build flavor by the path-shape heuristic below. + nonMacOSBinRoots = []string{"linux"} +) + +// CranPathInfo classifies a CRAN repo path and its HAR upload target. +type CranPathInfo struct { + Type string // "source" or "binary" + OS string // "" / "windows" / "macosx" + Arch string // macOS flavor when present + RVersion string // major.minor for binaries + RepoPath string // HAR contrib dir, e.g. "src/contrib" or "bin/macosx/big-sur-arm64/contrib/4.4" + FileName string + Extension string +} + +// DestUploadPath returns the flat HAR upload path; HAR recreates Archive/ itself. +func (p *CranPathInfo) DestUploadPath() string { + if p == nil { + return "" + } + return path.Join(p.RepoPath, p.FileName) +} + +// IsCranIndexFile reports whether path is a PACKAGES index file (HAR regenerates these). +func IsCranIndexFile(filePath string) bool { + name := path.Base(filePath) + switch name { + case "PACKAGES", "PACKAGES.gz", "PACKAGES.rds": + return true + default: + return false + } +} + +// ParseCranUploadPath classifies a repo-relative CRAN path (live, Archive/, or Artifactory variants). +func ParseCranUploadPath(filePath string) (*CranPathInfo, error) { + clean := strings.Trim(filePath, "/") + if clean == "" { + return nil, fmt.Errorf("ParseCranUploadPath: empty upload path") + } + segments := strings.Split(clean, "/") + fileName := segments[len(segments)-1] + + switch { + case segments[0] == "src": + return parseCranSourcePath(segments, fileName) + case segments[0] == "bin" && len(segments) >= 2 && segments[1] == cranOSWindows: + return parseCranWindowsPath(segments, fileName) + case segments[0] == "bin" && len(segments) >= 2 && segments[1] == cranOSMacOS: + return parseCranMacOSPath(segments, fileName) + case isArtifactoryMacOSBinLayout(segments): + // Artifactory macOS layout: bin//contrib//... (flavor is not "macosx"). + return parseCranArtifactoryMacOSPath(segments, fileName) + default: + return nil, fmt.Errorf("ParseCranUploadPath: unsupported R repository path %q", filePath) + } +} + +func parseCranSourcePath(segments []string, fileName string) (*CranPathInfo, error) { + // Live: src/contrib/ + // Classic Archive: src/contrib/Archive// + // Artifactory Archive: src/contrib/Archive/// + switch { + case len(segments) == 3 && segments[1] == cranContribDir: + // live + case len(segments) == 5 && segments[1] == cranContribDir && segments[2] == cranArchiveDir: + if err := validateCranArchiveDirs(segments[3], "", fileName, cranSourceExt); err != nil { + return nil, err + } + case len(segments) == 6 && segments[1] == cranContribDir && segments[2] == cranArchiveDir: + if err := validateCranArchiveDirs(segments[3], segments[4], fileName, cranSourceExt); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("parseCranSourcePath: source packages must be under src/contrib/ "+ + "or src/contrib/Archive//[/]") + } + if !strings.HasSuffix(strings.ToLower(fileName), cranSourceExt) { + return nil, fmt.Errorf("parseCranSourcePath: source packages must be .tar.gz, got %q", fileName) + } + return &CranPathInfo{ + Type: "source", + RepoPath: cranSrcContribRepoPath, + FileName: fileName, + Extension: cranSourceExt, + }, nil +} + +func parseCranWindowsPath(segments []string, fileName string) (*CranPathInfo, error) { + // Live: bin/windows/contrib// + // Classic Archive: bin/windows/contrib//Archive// + // Artifactory Archive: bin/windows/contrib//Archive/// + var rVersion string + switch { + case len(segments) == 5 && segments[2] == cranContribDir: + rVersion = segments[3] + case len(segments) == 7 && segments[2] == cranContribDir && segments[4] == cranArchiveDir: + rVersion = segments[3] + if err := validateCranArchiveDirs(segments[5], "", fileName, cranWindowsExt); err != nil { + return nil, err + } + case len(segments) == 8 && segments[2] == cranContribDir && segments[4] == cranArchiveDir: + rVersion = segments[3] + if err := validateCranArchiveDirs(segments[5], segments[6], fileName, cranWindowsExt); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf( + "parseCranWindowsPath: windows binaries must be under bin/windows/contrib// "+ + "or .../Archive//[/]") + } + if !cranRVersionRegex.MatchString(rVersion) { + return nil, fmt.Errorf("parseCranWindowsPath: invalid R version %q", rVersion) + } + if !strings.HasSuffix(strings.ToLower(fileName), cranWindowsExt) { + return nil, fmt.Errorf("parseCranWindowsPath: windows binaries must be .zip, got %q", fileName) + } + return &CranPathInfo{ + Type: "binary", + OS: cranOSWindows, + RVersion: rVersion, + RepoPath: strings.Join([]string{"bin", cranOSWindows, cranContribDir, rVersion}, "/"), + FileName: fileName, + Extension: cranWindowsExt, + }, nil +} + +func parseCranMacOSPath(segments []string, fileName string) (*CranPathInfo, error) { + // Classic CRAN/HAR: bin/macosx[/]/contrib//... + var flavor, rVersion string + var repoSegments []string + + switch { + case len(segments) == 6 && segments[3] == cranContribDir: + flavor = segments[2] + rVersion = segments[4] + repoSegments = segments[:5] + if err := validateMacFlavor(flavor); err != nil { + return nil, err + } + case len(segments) == 5 && segments[2] == cranContribDir: + rVersion = segments[3] + repoSegments = segments[:4] + case len(segments) == 8 && segments[3] == cranContribDir && segments[5] == cranArchiveDir: + flavor = segments[2] + rVersion = segments[4] + repoSegments = segments[:5] + if err := validateMacFlavor(flavor); err != nil { + return nil, err + } + if err := validateCranArchiveDirs(segments[6], "", fileName, cranMacOSExt); err != nil { + return nil, err + } + case len(segments) == 9 && segments[3] == cranContribDir && segments[5] == cranArchiveDir: + flavor = segments[2] + rVersion = segments[4] + repoSegments = segments[:5] + if err := validateMacFlavor(flavor); err != nil { + return nil, err + } + if err := validateCranArchiveDirs(segments[6], segments[7], fileName, cranMacOSExt); err != nil { + return nil, err + } + case len(segments) == 7 && segments[2] == cranContribDir && segments[4] == cranArchiveDir: + rVersion = segments[3] + repoSegments = segments[:4] + if err := validateCranArchiveDirs(segments[5], "", fileName, cranMacOSExt); err != nil { + return nil, err + } + case len(segments) == 8 && segments[2] == cranContribDir && segments[4] == cranArchiveDir: + rVersion = segments[3] + repoSegments = segments[:4] + if err := validateCranArchiveDirs(segments[5], segments[6], fileName, cranMacOSExt); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("parseCranMacOSPath: unsupported macOS binary path") + } + return newMacOSPathInfo(flavor, rVersion, repoSegments, fileName) +} + +// parseCranArtifactoryMacOSPath handles Artifactory mac layout (no "macosx" segment); rewrites to HAR path. +func parseCranArtifactoryMacOSPath(segments []string, fileName string) (*CranPathInfo, error) { + flavor := segments[1] + if err := validateMacFlavor(flavor); err != nil { + return nil, fmt.Errorf("parseCranArtifactoryMacOSPath: %w", err) + } + + var rVersion string + switch { + case len(segments) == 5 && segments[2] == cranContribDir: + rVersion = segments[3] + case len(segments) == 7 && segments[2] == cranContribDir && segments[4] == cranArchiveDir: + rVersion = segments[3] + if err := validateCranArchiveDirs(segments[5], "", fileName, cranMacOSExt); err != nil { + return nil, err + } + case len(segments) == 8 && segments[2] == cranContribDir && segments[4] == cranArchiveDir: + rVersion = segments[3] + if err := validateCranArchiveDirs(segments[5], segments[6], fileName, cranMacOSExt); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("parseCranArtifactoryMacOSPath: unsupported Artifactory macOS path") + } + + harRepo := strings.Join([]string{"bin", cranOSMacOS, flavor, cranContribDir, rVersion}, "/") + return newMacOSPathInfo(flavor, rVersion, strings.Split(harRepo, "/"), fileName) +} + +// newMacOSPathInfo validates the R version and extension shared by both macOS path variants, +// then builds the resulting CranPathInfo. +func newMacOSPathInfo(flavor, rVersion string, repoSegments []string, fileName string) (*CranPathInfo, error) { + if !cranRVersionRegex.MatchString(rVersion) { + return nil, fmt.Errorf("newMacOSPathInfo: invalid R version %q", rVersion) + } + if !strings.HasSuffix(strings.ToLower(fileName), cranMacOSExt) { + return nil, fmt.Errorf("newMacOSPathInfo: macOS binaries must be .tgz, got %q", fileName) + } + return &CranPathInfo{ + Type: "binary", + OS: cranOSMacOS, + Arch: flavor, + RVersion: rVersion, + RepoPath: strings.Join(repoSegments, "/"), + FileName: fileName, + Extension: cranMacOSExt, + }, nil +} + +func validateMacFlavor(flavor string) error { + if !cranMacFlavorRegex.MatchString(flavor) || flavor == cranContribDir || flavor == cranOSWindows { + return fmt.Errorf("invalid macOS flavor %q", flavor) + } + return nil +} + +// isArtifactoryMacOSBinLayout is true for bin//contrib/... mac builds (not linux/windows). +func isArtifactoryMacOSBinLayout(segments []string) bool { + if len(segments) < 5 || segments[0] != "bin" { + return false + } + flavor := segments[1] + if flavor == cranOSWindows || flavor == cranOSMacOS || isNonMacOSBinRoot(flavor) { + return false + } + if segments[2] != cranContribDir { + return false + } + return validateMacFlavor(flavor) == nil +} + +func isNonMacOSBinRoot(name string) bool { + for _, root := range nonMacOSBinRoots { + if name == root || strings.HasPrefix(name, root+"-") { + return true + } + } + return false +} + +// validateCranArchiveDirs ensures Archive//[version/] matches the package (and optional +// version) embedded in the archive leaf. versionDir may be empty for classic CRAN Archive paths. +func validateCranArchiveDirs(pkgDir, versionDir, fileName, extension string) error { + if !cranPackageNameRegex.MatchString(pkgDir) { + return fmt.Errorf("validateCranArchiveDirs: invalid Archive package dir %q", pkgDir) + } + name, version, err := ParseCranPackageFileName(fileName, extension) + if err != nil { + return err + } + if name != pkgDir { + return fmt.Errorf("validateCranArchiveDirs: Archive dir %q does not match package %q in %q", + pkgDir, name, fileName) + } + if versionDir != "" && versionDir != version { + return fmt.Errorf("validateCranArchiveDirs: Archive version dir %q does not match version %q in %q", + versionDir, version, fileName) + } + return nil +} + +// ParseCranPackageFileName splits an R archive leaf "_." into package name and +// version. R package names never contain underscores, so the first underscore is the delimiter. +func ParseCranPackageFileName(fileName, extension string) (name, version string, err error) { + base := strings.TrimSuffix(fileName, extension) + if base == fileName { + if idx := strings.LastIndex(strings.ToLower(fileName), extension); idx >= 0 { + base = fileName[:idx] + } + } + underscore := strings.IndexByte(base, '_') + if underscore <= 0 || underscore == len(base)-1 { + return "", "", fmt.Errorf("ParseCranPackageFileName: %q is not _%s", fileName, extension) + } + name = base[:underscore] + version = base[underscore+1:] + if !cranPackageNameRegex.MatchString(name) { + return "", "", fmt.Errorf("ParseCranPackageFileName: invalid R package name %q", name) + } + if version == "" { + return "", "", fmt.Errorf("ParseCranPackageFileName: missing version in %q", fileName) + } + return name, version, nil +} + +// BuildCranPackageFilesMap groups CRAN archive files by package name in one pass. +func BuildCranPackageFilesMap(files []types.File) map[string][]types.File { + out := make(map[string][]types.File) + for _, file := range files { + if file.Folder || IsCranIndexFile(file.Uri) { + continue + } + pkgName, _, ok := ParseCranFileNameWithPath(file.Uri) + if !ok { + continue + } + out[pkgName] = append(out[pkgName], file) + } + return out +} + +// ParseCranFileNameWithPath returns package name and version from a CRAN archive path. +func ParseCranFileNameWithPath(filePath string) (string, string, bool) { + if IsCranIndexFile(filePath) { + return "", "", false + } + pathInfo, err := ParseCranUploadPath(filePath) + if err != nil { + return "", "", false + } + name, version, err := ParseCranPackageFileName(pathInfo.FileName, pathInfo.Extension) + if err != nil { + return "", "", false + } + return name, version, true +} + +// CranHarUploadPath returns the flat HAR destination path for a source-registry CRAN file URI. +func CranHarUploadPath(filePath string) (string, bool) { + pathInfo, err := ParseCranUploadPath(filePath) + if err != nil { + return "", false + } + return pathInfo.DestUploadPath(), true +} diff --git a/modules/har/pkg/har/migrate/util/cran_util_test.go b/modules/har/pkg/har/migrate/util/cran_util_test.go new file mode 100644 index 00000000..c393a58b --- /dev/null +++ b/modules/har/pkg/har/migrate/util/cran_util_test.go @@ -0,0 +1,478 @@ +package util + +import ( + "testing" + + "github.com/harness/cli/modules/har/pkg/har/migrate/types" +) + +func TestIsCranIndexFile(t *testing.T) { + tests := []struct { + name string + path string + want bool + }{ + {"packages plain", "src/contrib/PACKAGES", true}, + {"packages gz", "/src/contrib/PACKAGES.gz", true}, + {"packages rds", "bin/windows/contrib/4.4/PACKAGES.rds", true}, + {"source archive", "src/contrib/jsonlite_1.8.0.tar.gz", false}, + {"leaf only packages", "PACKAGES", true}, + {"archived packages index", "src/contrib/Archive/jsonlite/PACKAGES", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsCranIndexFile(tt.path); got != tt.want { + t.Fatalf("IsCranIndexFile(%q) = %v, want %v", tt.path, got, tt.want) + } + }) + } +} + +func TestParseCranUploadPath(t *testing.T) { + tests := []struct { + name string + path string + wantType string + wantOS string + wantArch string + wantRVer string + wantRepo string + wantExt string + wantDest string + wantErr bool + }{ + { + name: "source", + path: "src/contrib/jsonlite_1.8.0.tar.gz", + wantType: "source", + wantRepo: "src/contrib", + wantExt: ".tar.gz", + wantDest: "src/contrib/jsonlite_1.8.0.tar.gz", + }, + { + name: "source with leading slash", + path: "/src/contrib/jsonlite_1.8.0.tar.gz", + wantType: "source", + wantRepo: "src/contrib", + wantExt: ".tar.gz", + wantDest: "src/contrib/jsonlite_1.8.0.tar.gz", + }, + { + name: "source archived", + path: "src/contrib/Archive/jsonlite/jsonlite_1.7.0.tar.gz", + wantType: "source", + wantRepo: "src/contrib", + wantExt: ".tar.gz", + wantDest: "src/contrib/jsonlite_1.7.0.tar.gz", + }, + { + name: "jfrog source archived with version dir", + path: "src/contrib/Archive/jsonlite/1.8.8/jsonlite_1.8.8.tar.gz", + wantType: "source", + wantRepo: "src/contrib", + wantExt: ".tar.gz", + wantDest: "src/contrib/jsonlite_1.8.8.tar.gz", + }, + { + name: "windows binary", + path: "bin/windows/contrib/4.4/jsonlite_1.8.0.zip", + wantType: "binary", + wantOS: "windows", + wantRVer: "4.4", + wantRepo: "bin/windows/contrib/4.4", + wantExt: ".zip", + wantDest: "bin/windows/contrib/4.4/jsonlite_1.8.0.zip", + }, + { + name: "windows archived", + path: "bin/windows/contrib/4.4/Archive/jsonlite/jsonlite_1.7.0.zip", + wantType: "binary", + wantOS: "windows", + wantRVer: "4.4", + wantRepo: "bin/windows/contrib/4.4", + wantExt: ".zip", + wantDest: "bin/windows/contrib/4.4/jsonlite_1.7.0.zip", + }, + { + name: "jfrog windows archived with version dir", + path: "bin/windows/contrib/4.4/Archive/jsonlite/1.7.0/jsonlite_1.7.0.zip", + wantType: "binary", + wantOS: "windows", + wantRVer: "4.4", + wantRepo: "bin/windows/contrib/4.4", + wantExt: ".zip", + wantDest: "bin/windows/contrib/4.4/jsonlite_1.7.0.zip", + }, + { + name: "macos legacy", + path: "bin/macosx/contrib/4.4/jsonlite_1.8.0.tgz", + wantType: "binary", + wantOS: "macosx", + wantRVer: "4.4", + wantRepo: "bin/macosx/contrib/4.4", + wantExt: ".tgz", + wantDest: "bin/macosx/contrib/4.4/jsonlite_1.8.0.tgz", + }, + { + name: "macos legacy archived", + path: "bin/macosx/contrib/4.4/Archive/jsonlite/jsonlite_1.7.0.tgz", + wantType: "binary", + wantOS: "macosx", + wantRVer: "4.4", + wantRepo: "bin/macosx/contrib/4.4", + wantExt: ".tgz", + wantDest: "bin/macosx/contrib/4.4/jsonlite_1.7.0.tgz", + }, + { + name: "macos flavored", + path: "bin/macosx/big-sur-arm64/contrib/4.4/jsonlite_1.8.0.tgz", + wantType: "binary", + wantOS: "macosx", + wantArch: "big-sur-arm64", + wantRVer: "4.4", + wantRepo: "bin/macosx/big-sur-arm64/contrib/4.4", + wantExt: ".tgz", + wantDest: "bin/macosx/big-sur-arm64/contrib/4.4/jsonlite_1.8.0.tgz", + }, + { + name: "macos flavored archived", + path: "bin/macosx/big-sur-arm64/contrib/4.4/Archive/jsonlite/jsonlite_1.7.0.tgz", + wantType: "binary", + wantOS: "macosx", + wantArch: "big-sur-arm64", + wantRVer: "4.4", + wantRepo: "bin/macosx/big-sur-arm64/contrib/4.4", + wantExt: ".tgz", + wantDest: "bin/macosx/big-sur-arm64/contrib/4.4/jsonlite_1.7.0.tgz", + }, + { + name: "jfrog macos without macosx segment", + path: "bin/big-sur-arm64/contrib/4.4/jsonlite_2.0.0.tgz", + wantType: "binary", + wantOS: "macosx", + wantArch: "big-sur-arm64", + wantRVer: "4.4", + wantRepo: "bin/macosx/big-sur-arm64/contrib/4.4", + wantExt: ".tgz", + wantDest: "bin/macosx/big-sur-arm64/contrib/4.4/jsonlite_2.0.0.tgz", + }, + { + name: "jfrog macos x86_64 without macosx segment", + path: "bin/big-sur-x86_64/contrib/4.4/jsonlite_2.0.0.tgz", + wantType: "binary", + wantOS: "macosx", + wantArch: "big-sur-x86_64", + wantRVer: "4.4", + wantRepo: "bin/macosx/big-sur-x86_64/contrib/4.4", + wantExt: ".tgz", + wantDest: "bin/macosx/big-sur-x86_64/contrib/4.4/jsonlite_2.0.0.tgz", + }, + { + name: "jfrog macos unknown codename still remaps", + path: "bin/mojave/contrib/4.4/jsonlite_2.0.0.tgz", + wantType: "binary", + wantOS: "macosx", + wantArch: "mojave", + wantRVer: "4.4", + wantRepo: "bin/macosx/mojave/contrib/4.4", + wantExt: ".tgz", + wantDest: "bin/macosx/mojave/contrib/4.4/jsonlite_2.0.0.tgz", + }, + { + name: "jfrog macos archived with version dir", + path: "bin/big-sur-arm64/contrib/4.4/Archive/jsonlite/1.7.0/jsonlite_1.7.0.tgz", + wantType: "binary", + wantOS: "macosx", + wantArch: "big-sur-arm64", + wantRVer: "4.4", + wantRepo: "bin/macosx/big-sur-arm64/contrib/4.4", + wantExt: ".tgz", + wantDest: "bin/macosx/big-sur-arm64/contrib/4.4/jsonlite_1.7.0.tgz", + }, + {name: "empty", path: "", wantErr: true}, + {name: "wrong root", path: "lib/contrib/pkg_1.0.tar.gz", wantErr: true}, + {name: "source wrong ext", path: "src/contrib/pkg_1.0.zip", wantErr: true}, + {name: "source wrong depth", path: "src/pkg_1.0.tar.gz", wantErr: true}, + {name: "windows bad r version", path: "bin/windows/contrib/4/pkg_1.0.zip", wantErr: true}, + {name: "archive pkg dir mismatch", path: "src/contrib/Archive/other/jsonlite_1.7.0.tar.gz", wantErr: true}, + {name: "archive invalid pkg dir", path: "src/contrib/Archive/1bad/jsonlite_1.7.0.tar.gz", wantErr: true}, + {name: "jfrog archive version dir mismatch", path: "src/contrib/Archive/jsonlite/9.9.9/jsonlite_1.8.8.tar.gz", wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseCranUploadPath(tt.path) + if tt.wantErr { + if err == nil { + t.Fatalf("expected error, got %+v", got) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.Type != tt.wantType || got.OS != tt.wantOS || got.Arch != tt.wantArch || + got.RVersion != tt.wantRVer || got.RepoPath != tt.wantRepo || got.Extension != tt.wantExt { + t.Fatalf("got %+v, want type=%s os=%s arch=%s r=%s repo=%s ext=%s", + got, tt.wantType, tt.wantOS, tt.wantArch, tt.wantRVer, tt.wantRepo, tt.wantExt) + } + if dest := got.DestUploadPath(); dest != tt.wantDest { + t.Fatalf("DestUploadPath() = %q, want %q", dest, tt.wantDest) + } + }) + } +} + +func TestParseCranPackageFileName(t *testing.T) { + tests := []struct { + name string + fileName string + ext string + wantName string + wantVersion string + wantErr bool + }{ + {"source", "jsonlite_1.8.0.tar.gz", ".tar.gz", "jsonlite", "1.8.0", false}, + {"dotted name", "data.table_1.14.8.tar.gz", ".tar.gz", "data.table", "1.14.8", false}, + {"windows zip", "jsonlite_1.8.0.zip", ".zip", "jsonlite", "1.8.0", false}, + {"missing underscore", "jsonlite1.8.0.tar.gz", ".tar.gz", "", "", true}, + {"invalid name", "1bad_1.0.0.tar.gz", ".tar.gz", "", "", true}, + {"empty version", "jsonlite_.tar.gz", ".tar.gz", "", "", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotName, gotVer, err := ParseCranPackageFileName(tt.fileName, tt.ext) + if tt.wantErr { + if err == nil { + t.Fatalf("expected error, got %q %q", gotName, gotVer) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if gotName != tt.wantName || gotVer != tt.wantVersion { + t.Fatalf("got (%q, %q), want (%q, %q)", gotName, gotVer, tt.wantName, tt.wantVersion) + } + }) + } +} + +func TestParseCranFileNameWithPath(t *testing.T) { + tests := []struct { + name string + input string + wantName string + wantVersion string + wantOK bool + }{ + {"source layout", "/src/contrib/jsonlite_1.8.0.tar.gz", "jsonlite", "1.8.0", true}, + {"source archived", "src/contrib/Archive/jsonlite/jsonlite_1.7.0.tar.gz", "jsonlite", "1.7.0", true}, + {"jfrog source archived", "src/contrib/Archive/jsonlite/1.8.8/jsonlite_1.8.8.tar.gz", "jsonlite", "1.8.8", true}, + {"windows layout", "bin/windows/contrib/4.4/jsonlite_1.8.0.zip", "jsonlite", "1.8.0", true}, + {"windows archived", "bin/windows/contrib/4.4/Archive/jsonlite/jsonlite_1.7.0.zip", "jsonlite", "1.7.0", true}, + {"macos flavored", "bin/macosx/big-sur-arm64/contrib/4.3/data.table_1.14.8.tgz", "data.table", "1.14.8", true}, + {"macos flavored archived", "bin/macosx/big-sur-arm64/contrib/4.3/Archive/data.table/data.table_1.14.0.tgz", "data.table", "1.14.0", true}, + {"jfrog macos", "bin/big-sur-arm64/contrib/4.4/jsonlite_2.0.0.tgz", "jsonlite", "2.0.0", true}, + {"index skipped", "src/contrib/PACKAGES.gz", "", "", false}, + {"non cran path", "/some/other/jsonlite_1.8.0.tar.gz", "", "", false}, + {"bad filename", "src/contrib/not-a-package.tar.gz", "", "", false}, + {"archive dir mismatch", "src/contrib/Archive/other/jsonlite_1.7.0.tar.gz", "", "", false}, + {"jfrog archive version mismatch", "src/contrib/Archive/jsonlite/9.9.9/jsonlite_1.8.8.tar.gz", "", "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotName, gotVer, ok := ParseCranFileNameWithPath(tt.input) + if ok != tt.wantOK { + t.Fatalf("ok = %v, want %v (got %q %q)", ok, tt.wantOK, gotName, gotVer) + } + if gotName != tt.wantName || gotVer != tt.wantVersion { + t.Fatalf("got (%q, %q), want (%q, %q)", gotName, gotVer, tt.wantName, tt.wantVersion) + } + }) + } +} + +func TestCranHarUploadPath(t *testing.T) { + tests := []struct { + name string + path string + want string + ok bool + }{ + {"live source", "/src/contrib/jsonlite_1.8.0.tar.gz", "src/contrib/jsonlite_1.8.0.tar.gz", true}, + {"archived source", "src/contrib/Archive/jsonlite/jsonlite_1.7.0.tar.gz", "src/contrib/jsonlite_1.7.0.tar.gz", true}, + {"jfrog archived source", "src/contrib/Archive/jsonlite/1.8.8/jsonlite_1.8.8.tar.gz", + "src/contrib/jsonlite_1.8.8.tar.gz", true}, + {"archived windows", "bin/windows/contrib/4.4/Archive/jsonlite/jsonlite_1.7.0.zip", + "bin/windows/contrib/4.4/jsonlite_1.7.0.zip", true}, + {"jfrog macos remap", "bin/big-sur-arm64/contrib/4.4/jsonlite_2.0.0.tgz", + "bin/macosx/big-sur-arm64/contrib/4.4/jsonlite_2.0.0.tgz", true}, + {"index", "src/contrib/PACKAGES", "", false}, + {"garbage", "foo/bar", "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := CranHarUploadPath(tt.path) + if ok != tt.ok || got != tt.want { + t.Fatalf("CranHarUploadPath(%q) = (%q, %v), want (%q, %v)", tt.path, got, ok, tt.want, tt.ok) + } + }) + } +} + +func TestParseCranUploadPathErrorCases(t *testing.T) { + tests := []struct { + name string + path string + }{ + {"windows wrong extension", "bin/windows/contrib/4.4/jsonlite_1.8.0.tar.gz"}, + {"windows bad layout", "bin/windows/jsonlite_1.8.0.zip"}, + {"windows bad r version", "bin/windows/contrib/notaver/jsonlite_1.8.0.zip"}, + {"macos wrong extension", "bin/macosx/contrib/4.4/jsonlite_1.8.0.zip"}, + {"macos bad flavor", "bin/macosx/BAD_FLAVOR/contrib/4.4/jsonlite_1.8.0.tgz"}, + {"jfrog macos wrong extension", "bin/big-sur-arm64/contrib/4.4/jsonlite_1.8.0.zip"}, + {"jfrog macos bad r version", "bin/big-sur-arm64/contrib/x/jsonlite_1.8.0.tgz"}, + {"bin linux not macos", "bin/linux/contrib/4.4/jsonlite_1.8.0.tgz"}, + {"bin without contrib shape", "bin/mojave/not-contrib/4.4/jsonlite_1.8.0.tgz"}, + {"source wrong extension", "src/contrib/jsonlite_1.8.0.zip"}, + {"source archive pkg mismatch", "src/contrib/Archive/jsonlite/data.table_1.0.0.tar.gz"}, + {"source archive version mismatch", "src/contrib/Archive/jsonlite/9.9.9/jsonlite_1.0.0.tar.gz"}, + {"empty", ""}, + {"unknown root", "lib/contrib/jsonlite_1.8.0.tar.gz"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if _, err := ParseCranUploadPath(tt.path); err == nil { + t.Fatalf("ParseCranUploadPath(%q) expected error", tt.path) + } + }) + } +} + +func TestParseCranUploadPathMacArchiveLayouts(t *testing.T) { + tests := []struct { + name string + path string + wantRepo string + wantArch string + }{ + { + name: "classic macosx with flavor archived", + path: "bin/macosx/big-sur-arm64/contrib/4.4/Archive/jsonlite/jsonlite_1.0.0.tgz", + wantRepo: "bin/macosx/big-sur-arm64/contrib/4.4", + wantArch: "big-sur-arm64", + }, + { + name: "classic macosx with flavor archived versioned", + path: "bin/macosx/big-sur-arm64/contrib/4.4/Archive/jsonlite/1.0.0/jsonlite_1.0.0.tgz", + wantRepo: "bin/macosx/big-sur-arm64/contrib/4.4", + wantArch: "big-sur-arm64", + }, + { + name: "classic macosx no flavor archived", + path: "bin/macosx/contrib/4.4/Archive/jsonlite/jsonlite_1.0.0.tgz", + wantRepo: "bin/macosx/contrib/4.4", + }, + { + name: "classic macosx no flavor archived versioned", + path: "bin/macosx/contrib/4.4/Archive/jsonlite/1.0.0/jsonlite_1.0.0.tgz", + wantRepo: "bin/macosx/contrib/4.4", + }, + { + name: "jfrog macos archived", + path: "bin/big-sur-arm64/contrib/4.4/Archive/jsonlite/jsonlite_1.0.0.tgz", + wantRepo: "bin/macosx/big-sur-arm64/contrib/4.4", + wantArch: "big-sur-arm64", + }, + { + name: "jfrog macos archived versioned", + path: "bin/big-sur-arm64/contrib/4.4/Archive/jsonlite/1.0.0/jsonlite_1.0.0.tgz", + wantRepo: "bin/macosx/big-sur-arm64/contrib/4.4", + wantArch: "big-sur-arm64", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + info, err := ParseCranUploadPath(tt.path) + if err != nil { + t.Fatalf("ParseCranUploadPath: %v", err) + } + if info.RepoPath != tt.wantRepo { + t.Errorf("RepoPath = %q, want %q", info.RepoPath, tt.wantRepo) + } + if info.Arch != tt.wantArch { + t.Errorf("Arch = %q, want %q", info.Arch, tt.wantArch) + } + wantDest := tt.wantRepo + "/jsonlite_1.0.0.tgz" + if info.DestUploadPath() != wantDest { + t.Errorf("DestUploadPath = %q, want %q", info.DestUploadPath(), wantDest) + } + }) + } +} + +func TestParseCranUploadPathMacArchiveErrors(t *testing.T) { + tests := []string{ + "bin/macosx/BAD/contrib/4.4/Archive/jsonlite/jsonlite_1.0.0.tgz", + "bin/macosx/big-sur-arm64/contrib/4.4/Archive/jsonlite/data.table_1.0.0.tgz", + "bin/macosx/contrib/4.4/Archive/jsonlite/1.0.0/jsonlite_9.9.9.tgz", + "bin/big-sur-arm64/contrib/4.4/Archive/jsonlite/data.table_1.0.0.tgz", + "bin/macosx/unsupported/path/structure.tgz", + "bin/big-sur-arm64/not-contrib/4.4/jsonlite_1.0.0.tgz", + } + for _, path := range tests { + t.Run(path, func(t *testing.T) { + if _, err := ParseCranUploadPath(path); err == nil { + t.Fatalf("expected error for %q", path) + } + }) + } +} + +func TestParseCranPackageFileNameErrors(t *testing.T) { + tests := []struct { + name string + file string + ext string + }{ + {"missing underscore", "jsonlite.tar.gz", ".tar.gz"}, + {"empty version", "jsonlite_.tar.gz", ".tar.gz"}, + {"bad package name", "1bad_1.0.0.tar.gz", ".tar.gz"}, + {"leading underscore empty name", "_1.0.0.tar.gz", ".tar.gz"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if _, _, err := ParseCranPackageFileName(tt.file, tt.ext); err == nil { + t.Fatalf("ParseCranPackageFileName(%q, %q) expected error", tt.file, tt.ext) + } + }) + } +} + +func TestDestUploadPathNil(t *testing.T) { + var p *CranPathInfo + if got := p.DestUploadPath(); got != "" { + t.Fatalf("DestUploadPath() = %q, want empty", got) + } +} + +func TestBuildCranPackageFilesMap(t *testing.T) { + files := []types.File{ + {Uri: "/src/contrib/jsonlite_1.8.0.tar.gz"}, + {Uri: "/bin/windows/contrib/4.4/jsonlite_1.8.0.zip"}, + {Uri: "/src/contrib/data.table_1.14.0.tar.gz"}, + {Uri: "/src/contrib/PACKAGES"}, + {Uri: "/not/a/cran/path.txt"}, + } + got := BuildCranPackageFilesMap(files) + if len(got["jsonlite"]) != 2 { + t.Fatalf("jsonlite files = %d, want 2", len(got["jsonlite"])) + } + if len(got["data.table"]) != 1 { + t.Fatalf("data.table files = %d, want 1", len(got["data.table"])) + } + if _, ok := got[""]; ok { + t.Fatal("unexpected empty package key") + } +} diff --git a/modules/har/pkg/har/migrate/util/pattern_util.go b/modules/har/pkg/har/migrate/util/pattern_util.go index 1b9189b8..df94d578 100644 --- a/modules/har/pkg/har/migrate/util/pattern_util.go +++ b/modules/har/pkg/har/migrate/util/pattern_util.go @@ -122,7 +122,7 @@ func FilterFilesByPatternsPackageName(packages []types.Package, includePatterns, func IsFileLevelFilterableArtifact(artifactType types.ArtifactType) bool { switch artifactType { - case types.GENERIC, types.RAW, types.PYTHON, types.MAVEN, types.NUGET, types.NPM, types.DART, types.GO: + case types.GENERIC, types.RAW, types.PYTHON, types.MAVEN, types.NUGET, types.NPM, types.DART, types.GO, types.RUBY, types.TERRAFORM: return true default: return false @@ -132,7 +132,7 @@ func IsFileLevelFilterableArtifact(artifactType types.ArtifactType) bool { func IsPackageLevelFilterableArtifact(artifactType types.ArtifactType) bool { switch artifactType { - case types.DOCKER, types.HELM, types.HELM_LEGACY, types.HELM_HTTP, types.RPM, types.CONDA, types.COMPOSER, types.SWIFT, types.CONAN: + case types.DOCKER, types.HELM, types.HELM_LEGACY, types.HELM_HTTP, types.RPM, types.CONDA, types.COMPOSER, types.SWIFT, types.CONAN, types.CRAN: return true default: return false diff --git a/modules/har/pkg/har/migrate/util/ruby_util.go b/modules/har/pkg/har/migrate/util/ruby_util.go new file mode 100644 index 00000000..06b1c0ca --- /dev/null +++ b/modules/har/pkg/har/migrate/util/ruby_util.go @@ -0,0 +1,72 @@ +package util + +import ( + "errors" + "path" + "regexp" + "strings" +) + +const rubyGemExt = ".gem" + +var errInvalidGemFilename = errors.New("invalid gem filename") + +// rubyGemVersionPattern matches RubyGems dot-prerelease versions in filenames +// (e.g. 1.0.0.beta1). Mirrors artifact-registry registry/pkg/ruby ParseGemFilename. +var rubyGemVersionPattern = regexp.MustCompile(`^\d+\.\d+(\.\d+)?(\.[0-9A-Za-z\.]+)*$`) + +// RubyMetadata holds parsed fields from a RubyGems filename +// ({name}-{version}.gem or {name}-{version}-{platform}.gem). +type RubyMetadata struct { + Name string + Version string + Platform string +} + +// ParseRubyGemFileNameWithPath parses a RubyGems download filename from a +// registry-relative path and returns the gem metadata. Multiple platform +// variants for the same version are grouped under one version entry during +// migration; Platform is populated when present in the filename. +func ParseRubyGemFileNameWithPath(filePath string) (RubyMetadata, bool) { + meta, err := parseGemFilename(path.Base(filePath)) + if err != nil { + return RubyMetadata{}, false + } + return meta, true +} + +func parseGemFilename(filename string) (RubyMetadata, error) { + filename = strings.TrimSpace(filename) + if filename == "" { + return RubyMetadata{}, errInvalidGemFilename + } + stem, ok := strings.CutSuffix(filename, rubyGemExt) + if !ok { + return RubyMetadata{}, errInvalidGemFilename + } + + parts := strings.Split(stem, "-") + if len(parts) < 2 { + return RubyMetadata{}, errInvalidGemFilename + } + + for i := len(parts) - 1; i >= 1; i-- { + candidateVersion := parts[i] + if !rubyGemVersionPattern.MatchString(candidateVersion) { + continue + } + name := strings.Join(parts[:i], "-") + if name == "" { + continue + } + meta := RubyMetadata{ + Name: name, + Version: candidateVersion, + } + if i+1 < len(parts) { + meta.Platform = strings.Join(parts[i+1:], "-") + } + return meta, nil + } + return RubyMetadata{}, errInvalidGemFilename +} diff --git a/modules/har/pkg/har/migrate/util/ruby_util_test.go b/modules/har/pkg/har/migrate/util/ruby_util_test.go new file mode 100644 index 00000000..991b69b6 --- /dev/null +++ b/modules/har/pkg/har/migrate/util/ruby_util_test.go @@ -0,0 +1,63 @@ +package util + +import "testing" + +func TestParseRubyGemFileNameWithPath(t *testing.T) { + tests := []struct { + name string + input string + wantName string + wantVersion string + wantPlatform string + wantOK bool + }{ + {"default platform gem", "rails-8.0.2.gem", "rails", "8.0.2", "", true}, + {"nested path", "/gems/rails-8.0.2.gem", "rails", "8.0.2", "", true}, + {"hyphenated gem name", "faraday-net_http-3.1.1.gem", "faraday-net_http", "3.1.1", "", true}, + { + "platform gem mingw", + "platform-gem-2.0.0-x86-mingw32-20.gem", + "platform-gem", + "2.0.0", + "x86-mingw32-20", + true, + }, + { + "platform gem linux", + "nokogiri-1.15.0-x86_64-linux.gem", + "nokogiri", + "1.15.0", + "x86_64-linux", + true, + }, + {"ruby prerelease dot", "mygem-1.0.0.beta1.gem", "mygem", "1.0.0.beta1", "", true}, + {"ruby short prerelease", "mygem-1.0.pre.gem", "mygem", "1.0.pre", "", true}, + {"missing extension", "rails-8.0.2.zip", "", "", "", false}, + {"missing version", "rails.gem", "", "", "", false}, + {"no valid version", "foo-bar.gem", "", "", "", false}, + {"empty name", "-1.0.0.gem", "", "", "", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + meta, ok := ParseRubyGemFileNameWithPath(tt.input) + if ok != tt.wantOK { + t.Fatalf("ok = %v, want %v", ok, tt.wantOK) + } + if meta.Name != tt.wantName || meta.Version != tt.wantVersion || meta.Platform != tt.wantPlatform { + t.Fatalf("got name=%q version=%q platform=%q, want name=%q version=%q platform=%q", + meta.Name, meta.Version, meta.Platform, tt.wantName, tt.wantVersion, tt.wantPlatform) + } + }) + } +} + +func TestParseGemFilename_platform(t *testing.T) { + meta, err := parseGemFilename("nokogiri-1.15.0-x86_64-linux.gem") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if meta.Name != "nokogiri" || meta.Version != "1.15.0" || meta.Platform != "x86_64-linux" { + t.Fatalf("got name=%q version=%q platform=%q", meta.Name, meta.Version, meta.Platform) + } +} diff --git a/modules/har/pkg/har/migrate/util/selector_util.go b/modules/har/pkg/har/migrate/util/selector_util.go new file mode 100644 index 00000000..481917b2 --- /dev/null +++ b/modules/har/pkg/har/migrate/util/selector_util.go @@ -0,0 +1,90 @@ +package util + +import ( + "strings" + + "github.com/harness/cli/modules/har/pkg/har/migrate/types" +) + +// FilterPackagesBySelectors returns a slice containing only those packages whose +// Name appears in the filters. If filters is empty, all packages pass through +// unchanged (behavior-preserving passthrough). Package names are matched +// case-insensitively. +func FilterPackagesBySelectors(pkgs []types.Package, filters []types.PackageSelector) []types.Package { + if len(filters) == 0 { + return pkgs + } + + allowed := make(map[string]struct{}, len(filters)) + for _, filter := range filters { + allowed[strings.ToLower(filter.Package)] = struct{}{} + } + + var filtered []types.Package + for _, pkg := range pkgs { + if _, ok := allowed[strings.ToLower(pkg.Name)]; ok { + filtered = append(filtered, pkg) + } + } + + return filtered +} + +// SelectorForPackage looks up a PackageSelector for the given package name in the +// mapping's PackageFilters. Returns (selector, hasFilters, matched) where: +// - selector: the matching PackageSelector (zero value if not matched) +// - hasFilters: true if the mapping has any PackageFilters configured +// - matched: true if a selector for pkgName was found +// +// If the mapping has no filters, returns (zero, false, false) — a passthrough signal. +// If the mapping has filters but pkgName is not in the allow-list, returns +// (zero, true, false) — a block signal. +func SelectorForPackage(mapping *types.RegistryMapping, pkgName string) (types.PackageSelector, bool, bool) { + if mapping == nil || len(mapping.PackageFilters) == 0 { + return types.PackageSelector{}, false, false + } + + for _, sel := range mapping.PackageFilters { + if strings.EqualFold(sel.Package, pkgName) { + return sel, true, true + } + } + + return types.PackageSelector{}, true, false +} + +// VersionSelectedBySelector reports whether the given version name is allowed by +// the selector. If the selector's Versions list is empty, all versions pass (returns +// true). Otherwise returns true iff versionName matches one of the entries +// (case-insensitive). +func VersionSelectedBySelector(sel types.PackageSelector, versionName string) bool { + if len(sel.Versions) == 0 { + return true + } + + for _, v := range sel.Versions { + if strings.EqualFold(v, versionName) { + return true + } + } + + return false +} + +// FileSelectedBySelector reports whether the given file name is allowed by the +// selector. If the selector's Files list is empty, all files pass (returns true). +// Otherwise returns true iff fileName matches one of the entries (case-insensitive). +func FileSelectedBySelector(sel types.PackageSelector, fileName string) bool { + if len(sel.Files) == 0 { + return true + } + + lowerFileName := strings.ToLower(fileName) + for _, f := range sel.Files { + if strings.ToLower(f) == lowerFileName { + return true + } + } + + return false +} diff --git a/modules/har/pkg/har/migrate/util/selector_util_test.go b/modules/har/pkg/har/migrate/util/selector_util_test.go new file mode 100644 index 00000000..20959088 --- /dev/null +++ b/modules/har/pkg/har/migrate/util/selector_util_test.go @@ -0,0 +1,268 @@ +package util + +import ( + "testing" + + "github.com/harness/cli/modules/har/pkg/har/migrate/types" + + "github.com/stretchr/testify/assert" +) + +func TestFilterPackagesBySelectors(t *testing.T) { + tests := []struct { + name string + pkgs []types.Package + filters []types.PackageSelector + wantLen int + wantPkgs []string + }{ + { + name: "empty filters returns all packages unchanged", + pkgs: []types.Package{ + {Name: "express"}, + {Name: "lodash"}, + {Name: "react"}, + }, + filters: []types.PackageSelector{}, + wantLen: 3, + wantPkgs: []string{"express", "lodash", "react"}, + }, + { + name: "filters naming a subset returns only matching packages", + pkgs: []types.Package{ + {Name: "express"}, + {Name: "lodash"}, + {Name: "react"}, + }, + filters: []types.PackageSelector{ + {Package: "express"}, + {Package: "react"}, + }, + wantLen: 2, + wantPkgs: []string{"express", "react"}, + }, + { + name: "case-insensitive: selector Foo matches both foo and Foo", + pkgs: []types.Package{ + {Name: "foo"}, + {Name: "Foo"}, + {Name: "bar"}, + }, + filters: []types.PackageSelector{ + {Package: "Foo"}, + }, + wantLen: 2, + wantPkgs: []string{"foo", "Foo"}, + }, + { + name: "no matching packages returns empty slice", + pkgs: []types.Package{ + {Name: "express"}, + {Name: "lodash"}, + }, + filters: []types.PackageSelector{ + {Package: "nonexistent"}, + }, + wantLen: 0, + wantPkgs: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := FilterPackagesBySelectors(tt.pkgs, tt.filters) + assert.Len(t, result, tt.wantLen) + for i, pkgName := range tt.wantPkgs { + assert.Equal(t, pkgName, result[i].Name) + } + }) + } +} + +func TestSelectorForPackage(t *testing.T) { + tests := []struct { + name string + mapping *types.RegistryMapping + pkgName string + wantSel types.PackageSelector + hasFilters bool + matched bool + }{ + { + name: "nil mapping returns hasFilters=false, matched=false", + mapping: nil, + pkgName: "express", + wantSel: types.PackageSelector{}, + hasFilters: false, + matched: false, + }, + { + name: "mapping with empty PackageFilters returns hasFilters=false", + mapping: &types.RegistryMapping{ + PackageFilters: []types.PackageSelector{}, + }, + pkgName: "express", + wantSel: types.PackageSelector{}, + hasFilters: false, + matched: false, + }, + { + name: "mapping with filters but no name match returns hasFilters=true, matched=false", + mapping: &types.RegistryMapping{ + PackageFilters: []types.PackageSelector{ + {Package: "lodash"}, + {Package: "react"}, + }, + }, + pkgName: "express", + wantSel: types.PackageSelector{}, + hasFilters: true, + matched: false, + }, + { + name: "mapping with matching selector returns hasFilters=true, matched=true, and the right selector", + mapping: &types.RegistryMapping{ + PackageFilters: []types.PackageSelector{ + {Package: "lodash", Versions: []string{"4.17.0"}}, + {Package: "express", Versions: []string{"4.18.0", "4.19.0"}, Files: []string{"express.tgz"}}, + {Package: "react"}, + }, + }, + pkgName: "express", + wantSel: types.PackageSelector{ + Package: "express", + Versions: []string{"4.18.0", "4.19.0"}, + Files: []string{"express.tgz"}, + }, + hasFilters: true, + matched: true, + }, + { + name: "case-insensitive matching: Express matches express", + mapping: &types.RegistryMapping{ + PackageFilters: []types.PackageSelector{ + {Package: "express"}, + }, + }, + pkgName: "Express", + wantSel: types.PackageSelector{Package: "express"}, + hasFilters: true, + matched: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sel, hasFilters, matched := SelectorForPackage(tt.mapping, tt.pkgName) + assert.Equal(t, tt.hasFilters, hasFilters) + assert.Equal(t, tt.matched, matched) + assert.Equal(t, tt.wantSel, sel) + }) + } +} + +func TestVersionSelectedBySelector(t *testing.T) { + tests := []struct { + name string + selector types.PackageSelector + versionName string + want bool + }{ + { + name: "empty Versions returns true for any version", + selector: types.PackageSelector{ + Package: "express", + Versions: []string{}, + }, + versionName: "4.18.0", + want: true, + }, + { + name: "non-empty Versions returns true for listed version", + selector: types.PackageSelector{ + Package: "express", + Versions: []string{"4.18.0", "4.19.0", "5.0.0"}, + }, + versionName: "4.19.0", + want: true, + }, + { + name: "non-empty Versions returns false for unlisted version", + selector: types.PackageSelector{ + Package: "express", + Versions: []string{"4.18.0", "4.19.0"}, + }, + versionName: "5.0.0", + want: false, + }, + { + name: "version matching is case-insensitive", + selector: types.PackageSelector{ + Package: "mypkg", + Versions: []string{"1.0.0-RC1"}, + }, + versionName: "1.0.0-rc1", + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := VersionSelectedBySelector(tt.selector, tt.versionName) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestFileSelectedBySelector(t *testing.T) { + tests := []struct { + name string + selector types.PackageSelector + fileName string + want bool + }{ + { + name: "empty Files returns true for any file", + selector: types.PackageSelector{ + Package: "express", + Files: []string{}, + }, + fileName: "express-4.18.0.tgz", + want: true, + }, + { + name: "non-empty Files returns true for listed file", + selector: types.PackageSelector{ + Package: "express", + Files: []string{"express.tgz", "express.min.js"}, + }, + fileName: "express.tgz", + want: true, + }, + { + name: "case-insensitive match: selector File.TXT matches file.txt", + selector: types.PackageSelector{ + Package: "mypackage", + Files: []string{"File.TXT", "Other.ZIP"}, + }, + fileName: "file.txt", + want: true, + }, + { + name: "non-empty Files returns false for unlisted file", + selector: types.PackageSelector{ + Package: "express", + Files: []string{"express.tgz"}, + }, + fileName: "express.min.js", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := FileSelectedBySelector(tt.selector, tt.fileName) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/modules/har/pkg/har/migrate/util/terraform_util.go b/modules/har/pkg/har/migrate/util/terraform_util.go new file mode 100644 index 00000000..98e42ed8 --- /dev/null +++ b/modules/har/pkg/har/migrate/util/terraform_util.go @@ -0,0 +1,106 @@ +package util + +import ( + "path" + "regexp" + "strings" +) + +// JFrog Terraform module storage layouts (two variants observed in the wild): +// Layout A (goreleaser): /////-.tar.gz (5+ parts) +// Layout B (flat): ////.zip (4 parts, version = filename stem) +// +// JFrog Terraform provider storage layout: +// ////terraform-provider-___.zip + +// terraformProviderFilenameRegex matches the standard convention: +// +// terraform-provider-{type}_{version}_{os}_{arch}.zip +var terraformProviderFilenameRegex = regexp.MustCompile( + `^terraform-provider-([a-zA-Z0-9-]+)_(\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?)_([a-z0-9]+)_([a-z0-9]+)\.zip$`, +) + +// ParseTerraformModulePath parses a JFrog module file URI and returns +// (namespace, name, provider, version, ok). +// +// Layout A (goreleaser): /////-.tar.gz (5+ segments) +// Layout B (flat zip): ////.zip (4 segments, version = filename stem) +func ParseTerraformModulePath(filePath string) (ns, name, provider, version string, ok bool) { + filePath = strings.TrimPrefix(filePath, "/") + parts := strings.Split(filePath, "/") + + // Layout A: ns / name / provider / ver / .tar.gz|.tgz + if len(parts) >= 5 { + filename := parts[len(parts)-1] + lower := strings.ToLower(filename) + if strings.HasSuffix(lower, ".tar.gz") || strings.HasSuffix(lower, ".tgz") { + ns = parts[0] + name = parts[1] + provider = parts[2] + version = parts[3] + ok = ns != "" && name != "" && provider != "" && version != "" + return + } + } + + // Layout B: ns / name / provider / .zip (flat, version is the filename stem) + // The stem must start with a digit (semver) to avoid matching provider filenames + // like "terraform-provider-aws_2.0.0_linux_amd64.zip" which share the 4-part shape. + if len(parts) == 4 { + filename := parts[3] + lower := strings.ToLower(filename) + if strings.HasSuffix(lower, ".zip") { + stem := filename[:len(filename)-4] + if stem != "" && stem[0] >= '0' && stem[0] <= '9' { + ns = parts[0] + name = parts[1] + provider = parts[2] + version = stem + ok = ns != "" && name != "" && provider != "" && version != "" + return + } + } + } + + return +} + +// ParseTerraformProviderPath parses a JFrog provider file URI and returns +// (namespace, typeName, version, filename, os, arch, ok). +// Expected form: ////terraform-provider-___.zip +func ParseTerraformProviderPath(filePath string) (ns, typeName, version, filename, osName, arch string, ok bool) { + filePath = strings.TrimPrefix(filePath, "/") + parts := strings.Split(filePath, "/") + // Need at least: ns / type / ver / filename + if len(parts) < 4 { + return + } + filename = path.Base(filePath) + if !strings.HasSuffix(strings.ToLower(filename), ".zip") { + return + } + ns = parts[0] + typeName = parts[1] + version = parts[2] + m := terraformProviderFilenameRegex.FindStringSubmatch(filename) + if m == nil { + return + } + // m[1]=type, m[2]=version, m[3]=os, m[4]=arch + osName = m[3] + arch = m[4] + ok = ns != "" && typeName != "" && version != "" + return +} + +// IsTerraformModule reports whether a file path looks like a Terraform module. +func IsTerraformModule(filePath string) bool { + _, _, _, _, ok := ParseTerraformModulePath(filePath) + return ok +} + +// IsTerraformProvider reports whether a file path looks like a Terraform provider. +func IsTerraformProvider(filePath string) bool { + _, _, _, _, _, _, ok := ParseTerraformProviderPath(filePath) + return ok +} diff --git a/modules/har/pkg/har/migrate/util/terraform_util_test.go b/modules/har/pkg/har/migrate/util/terraform_util_test.go new file mode 100644 index 00000000..6307112b --- /dev/null +++ b/modules/har/pkg/har/migrate/util/terraform_util_test.go @@ -0,0 +1,221 @@ +package util + +import ( + "testing" +) + +func TestParseTerraformModulePath(t *testing.T) { + tests := []struct { + name string + path string + wantNS string + wantName string + wantProv string + wantVer string + wantOK bool + }{ + { + name: "happy path with leading slash", + path: "/hashicorp/vpc/aws/1.0.0/vpc-1.0.0.tar.gz", + wantNS: "hashicorp", wantName: "vpc", wantProv: "aws", wantVer: "1.0.0", wantOK: true, + }, + { + name: "happy path without leading slash", + path: "hashicorp/vpc/aws/1.0.0/vpc-1.0.0.tar.gz", + wantNS: "hashicorp", wantName: "vpc", wantProv: "aws", wantVer: "1.0.0", wantOK: true, + }, + { + name: "tgz extension", + path: "/hashicorp/vpc/aws/2.0.0/vpc-2.0.0.tgz", + wantNS: "hashicorp", wantName: "vpc", wantProv: "aws", wantVer: "2.0.0", wantOK: true, + }, + { + name: "wrong extension", + path: "/hashicorp/vpc/aws/1.0.0/vpc-1.0.0.zip", + wantOK: false, + }, + { + name: "too few segments (depth 4)", + path: "/hashicorp/vpc/aws/vpc-1.0.0.tar.gz", + wantOK: false, + }, + { + name: "too few segments (depth 3)", + path: "/hashicorp/vpc/vpc-1.0.0.tar.gz", + wantOK: false, + }, + { + name: "empty path", + path: "", + wantOK: false, + }, + { + name: "only slash", + path: "/", + wantOK: false, + }, + { + name: "depth 6 (extra subdirectory)", + path: "/hashicorp/vpc/aws/1.0.0/subdir/vpc-1.0.0.tar.gz", + wantNS: "hashicorp", wantName: "vpc", wantProv: "aws", wantVer: "1.0.0", wantOK: true, + }, + // Layout B: flat zip, version = filename stem + { + name: "layout B flat zip", + path: "/myorg/s3module/aws/1.0.0.zip", + wantNS: "myorg", wantName: "s3module", wantProv: "aws", wantVer: "1.0.0", wantOK: true, + }, + { + name: "layout B flat zip without leading slash", + path: "myorg/s3module/google/2.3.4.zip", + wantNS: "myorg", wantName: "s3module", wantProv: "google", wantVer: "2.3.4", wantOK: true, + }, + { + name: "layout B rejected: provider filename in 4-segment path", + path: "/hashicorp/aws/2.0.0/terraform-provider-aws_linux_amd64.zip", + wantOK: false, + }, + { + name: "layout B rejected: module.json sidecar", + path: "/myorg/s3module/aws/module.json", + wantOK: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ns, name, provider, version, ok := ParseTerraformModulePath(tt.path) + if ok != tt.wantOK { + t.Fatalf("ok=%v want %v", ok, tt.wantOK) + } + if !ok { + return + } + if ns != tt.wantNS { + t.Errorf("ns=%q want %q", ns, tt.wantNS) + } + if name != tt.wantName { + t.Errorf("name=%q want %q", name, tt.wantName) + } + if provider != tt.wantProv { + t.Errorf("provider=%q want %q", provider, tt.wantProv) + } + if version != tt.wantVer { + t.Errorf("version=%q want %q", version, tt.wantVer) + } + }) + } +} + +func TestParseTerraformProviderPath(t *testing.T) { + tests := []struct { + name string + path string + wantNS string + wantType string + wantVer string + wantFile string + wantOS string + wantArch string + wantOK bool + }{ + { + name: "happy path linux amd64", + path: "/hashicorp/aws/2.0.0/terraform-provider-aws_2.0.0_linux_amd64.zip", + wantNS: "hashicorp", wantType: "aws", wantVer: "2.0.0", + wantFile: "terraform-provider-aws_2.0.0_linux_amd64.zip", + wantOS: "linux", wantArch: "amd64", wantOK: true, + }, + { + name: "filename missing version segment", + path: "/hashicorp/aws/2.0.0/terraform-provider-aws_linux_amd64.zip", + wantOK: false, + }, + { + name: "darwin arm64", + path: "/hashicorp/aws/2.0.0/terraform-provider-aws_2.0.0_darwin_arm64.zip", + wantNS: "hashicorp", wantType: "aws", wantVer: "2.0.0", + wantFile: "terraform-provider-aws_2.0.0_darwin_arm64.zip", + wantOS: "darwin", wantArch: "arm64", wantOK: true, + }, + { + name: "with leading slash", + path: "/hashicorp/google/3.1.0/terraform-provider-google_3.1.0_windows_386.zip", + wantNS: "hashicorp", wantType: "google", wantVer: "3.1.0", + wantFile: "terraform-provider-google_3.1.0_windows_386.zip", + wantOS: "windows", wantArch: "386", wantOK: true, + }, + { + name: "non-provider zip (filename doesn't match regex)", + path: "/hashicorp/aws/2.0.0/aws-sdk-2.0.0_linux_amd64.zip", + wantOK: false, + }, + { + name: "tar.gz not a provider", + path: "/hashicorp/aws/2.0.0/terraform-provider-aws_linux_amd64.tar.gz", + wantOK: false, + }, + { + name: "too few path segments (3)", + path: "/hashicorp/aws/terraform-provider-aws_linux_amd64.zip", + wantOK: false, + }, + { + name: "empty path", + path: "", + wantOK: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ns, typeName, version, filename, osName, arch, ok := ParseTerraformProviderPath(tt.path) + if ok != tt.wantOK { + t.Fatalf("ok=%v want %v for path %q", ok, tt.wantOK, tt.path) + } + if !ok { + return + } + if ns != tt.wantNS { + t.Errorf("ns=%q want %q", ns, tt.wantNS) + } + if typeName != tt.wantType { + t.Errorf("typeName=%q want %q", typeName, tt.wantType) + } + if version != tt.wantVer { + t.Errorf("version=%q want %q", version, tt.wantVer) + } + if filename != tt.wantFile { + t.Errorf("filename=%q want %q", filename, tt.wantFile) + } + if osName != tt.wantOS { + t.Errorf("os=%q want %q", osName, tt.wantOS) + } + if arch != tt.wantArch { + t.Errorf("arch=%q want %q", arch, tt.wantArch) + } + }) + } +} + +func TestIsTerraformModule(t *testing.T) { + if !IsTerraformModule("/hashicorp/vpc/aws/1.0.0/vpc-1.0.0.tar.gz") { + t.Error("expected true for Layout A module path") + } + if !IsTerraformModule("/myorg/s3module/aws/1.0.0.zip") { + t.Error("expected true for Layout B flat zip module path") + } + if IsTerraformModule("/hashicorp/aws/2.0.0/terraform-provider-aws_linux_amd64.zip") { + t.Error("expected false for provider filename in 4-segment path") + } + if IsTerraformModule("/myorg/s3module/aws/module.json") { + t.Error("expected false for module.json sidecar") + } +} + +func TestIsTerraformProvider(t *testing.T) { + if !IsTerraformProvider("/hashicorp/aws/2.0.0/terraform-provider-aws_2.0.0_linux_amd64.zip") { + t.Error("expected true for valid provider path") + } + if IsTerraformProvider("/hashicorp/vpc/aws/1.0.0/vpc-1.0.0.tar.gz") { + t.Error("expected false for module path") + } +} diff --git a/modules/har/pkg/har/migrate_handler.go b/modules/har/pkg/har/migrate_handler.go index 7ea3c856..0e2d8f35 100644 --- a/modules/har/pkg/har/migrate_handler.go +++ b/modules/har/pkg/har/migrate_handler.go @@ -25,6 +25,8 @@ func executeRegistryMigrateHandler(ctx *cmdctx.Ctx) error { concurrencyStr := cmdctx.GetString(ctx.FlagValues, "concurrency") overwrite := cmdctx.GetBool(ctx.FlagValues, "overwrite") dryRun := cmdctx.GetBool(ctx.FlagValues, "dry-run") + summary := cmdctx.GetBool(ctx.FlagValues, "summary") + resultFile := cmdctx.GetString(ctx.FlagValues, "result-file") cfg, err := types.LoadConfig(filePath) if err != nil { @@ -42,6 +44,12 @@ func executeRegistryMigrateHandler(ctx *cmdctx.Ctx) error { if dryRun { cfg.DryRun = true } + if summary { + cfg.Summary = true + } + if resultFile != "" { + cfg.ResultFile = resultFile + } // Thread auth context into the destination (HAR) registry config. cfg.Dest.AccountID = a.AccountID diff --git a/pkg/spec/har.spec.yaml b/pkg/spec/har.spec.yaml index c7e5ff80..6b9b1cab 100644 --- a/pkg/spec/har.spec.yaml +++ b/pkg/spec/har.spec.yaml @@ -1082,6 +1082,89 @@ commands: noun: registry noun_variant: migrate short: Migrate artifacts from a source registry into Harness using a config file + long: | + Migrate artifacts from a source registry to a Harness Artifact Registry. + + This command reads a YAML configuration file that defines the source and + destination registries, credentials, and artifact mappings. + + Example configuration file (config.yaml): + + version: 1.0.0 + concurrency: 5 + overwrite: false + + source: + endpoint: https://source-registry.example.com + type: JFROG # Supported: JFROG, NEXUS, HARBOR, HAR + credentials: + username: source_user + password: source_password + insecure: false + + destination: + endpoint: https://pkg.harness.io + type: HAR + credentials: + username: harness_user + password: harness_api_key + + mappings: + - artifactType: DOCKER + sourceRegistry: docker-repo + destinationRegistry: harness-docker-repo + + - artifactType: MAVEN + sourceRegistry: maven-releases + destinationRegistry: harness-maven + # MAVEN identity is derived from file paths, so filter via files (see note below). + packageFilters: + - files: [my-artifact-1.0.0.jar, my-artifact-1.1.0.pom] + + - artifactType: PYTHON + sourceRegistry: pypi-local + destinationRegistry: harness-pypi + packageFilters: + - package: requests + versions: [2.28.0, 2.29.0] + files: [requests-2.28.0-py3-none-any.whl] + + Supported artifact types: + DOCKER, HELM, HELM_LEGACY, GENERIC, PYTHON, MAVEN, NPM, NUGET, RPM, GO, + CONDA, COMPOSER, DART, RAW, SWIFT, DEBIAN, CONAN, PUPPET, HELM_HTTP, + RUBY, CRAN, TERRAFORM + + Package Filtering (packageFilters, opt-in): + An allow-list: when set, only the named packages (and optionally specific + versions/files) are migrated; when omitted, everything migrates. + + Supported packageFilters granularity by type: + - package name only: DOCKER, HELM, HELM_LEGACY, HELM_HTTP, RPM, DEBIAN, + CONDA, COMPOSER, SWIFT, CONAN, CRAN, TERRAFORM + - package name + versions: GO + - package name + versions + files: GENERIC, RAW, MAVEN, PYTHON, NUGET, + NPM, DART, PUPPET, RUBY + + For synthetic-identity types (MAVEN, NPM, GENERIC, RAW), filter via the + files field rather than package names, since package/version names are + derived from file paths. + + Include/Exclude Patterns (includePatterns / excludePatterns, mutually exclusive): + Glob patterns (* and ** wildcards) applied at a type-dependent granularity: + - File level: GENERIC, RAW, PYTHON, MAVEN, NUGET, NPM, DART, GO, RUBY, TERRAFORM + - Package level: DOCKER, HELM, HELM_LEGACY, HELM_HTTP, RPM, CONDA, COMPOSER, SWIFT, CONAN, CRAN + Setting them for PUPPET or DEBIAN currently has no effect (silently ignored, + not rejected at config load). + + Output and automation flags: + --summary prints condensed per-status counts instead of the full per-file table. + --result-file writes one JSON-lines record per migrated coordinate (status, + reason, error) for scripts/CI to consume, even when the run fails. + + Environment variables can be used in the config file using ${VAR_NAME} syntax. + + Usage example: + harness execute registry:migrate --config config.yaml no_id: true flags_builtin: handler_type: workflow @@ -1098,6 +1181,11 @@ commands: - name: dry-run description: "Dry run — discover files without uploading, write report to ./dry-run-output/" is_bool: true + - name: summary + description: "Print a condensed status summary instead of the full per-file table" + is_bool: true + - name: result-file + description: "Path to a JSON-lines file to write one per-coordinate result record (overrides config)" - command: configure registry:npm verb: configure