From 8012dabf2709cd0259db94fac63ef385e8b0e4c3 Mon Sep 17 00:00:00 2001 From: Simon Castagna Date: Thu, 22 Oct 2026 16:51:03 +0200 Subject: [PATCH 1/5] feat: kosli get rego-policy --download writes a version's files to a directory The server now serves every file of a rego policy version as one gzipped tar (kosli-dev/server#7380). --download DIR, with --policy-version, writes those files into DIR at the paths they were published with, so a version can be inspected or replayed offline. Every member is checked before any file is written: it must be a regular file at a path inside DIR, and no file may already exist there. A refused bundle leaves the directory as it was. Co-Authored-By: Claude Opus 5.5 --- cmd/kosli/getRegoPolicy.go | 51 ++++++- cmd/kosli/getRegoPolicy_test.go | 89 ++++++++++++ cmd/kosli/root.go | 1 + .../testdata/empty-flag-audit-coverage.json | 1 + hack/empty-flag-audit/spec.json | 2 + internal/regopolicies/bundle.go | 76 +++++++++++ internal/regopolicies/bundle_test.go | 128 ++++++++++++++++++ internal/regopolicies/client.go | 9 ++ internal/regopolicies/urls_test.go | 7 + 9 files changed, 361 insertions(+), 3 deletions(-) create mode 100644 internal/regopolicies/bundle.go create mode 100644 internal/regopolicies/bundle_test.go diff --git a/cmd/kosli/getRegoPolicy.go b/cmd/kosli/getRegoPolicy.go index 259b024de..316429dc3 100644 --- a/cmd/kosli/getRegoPolicy.go +++ b/cmd/kosli/getRegoPolicy.go @@ -21,7 +21,11 @@ one an evaluation runs, so it can be older than the newest version. With ^--policy-version^, that version is shown with its status, its digest, who published it and when, and the reason when it is rejected or failed. The command -exits 0 whatever the status of the version.` +exits 0 whatever the status of the version. + +With ^--download^ and ^--policy-version^, the files of that version are written into the +given directory at the paths they were published with, whatever the status of the +version, in place of printing it. A digest computed from these files equals the version's digest.` const getRegoPolicyExample = ` # get a rego policy and its current version: @@ -35,6 +39,13 @@ kosli get rego-policy pr-approved \ --api-token yourAPIToken \ --org yourOrgName +# download the files of version 3 of a rego policy into ./pr-approved-v3: +kosli get rego-policy pr-approved \ + --policy-version 3 \ + --download ./pr-approved-v3 \ + --api-token yourAPIToken \ + --org yourOrgName + # get version 3 of a rego policy as JSON: kosli get rego-policy pr-approved \ --policy-version 3 \ @@ -44,8 +55,9 @@ kosli get rego-policy pr-approved \ ` type getRegoPolicyOptions struct { - output string - version int + output string + version int + download string } func newGetRegoPolicyCmd(out io.Writer) *cobra.Command { @@ -67,15 +79,22 @@ func newGetRegoPolicyCmd(out io.Writer) *cobra.Command { if cmd.Flags().Changed("policy-version") && o.version < 1 { return ErrorBeforePrintingUsage(cmd, "--policy-version must be a positive integer") } + if o.download != "" && o.version == 0 { + return ErrorBeforePrintingUsage(cmd, "--download needs --policy-version") + } return nil }, RunE: func(cmd *cobra.Command, args []string) error { + if o.download != "" { + return o.runDownload(out, args[0]) + } return o.run(out, args[0]) }, } cmd.Flags().IntVar(&o.version, "policy-version", 0, regoPolicyVersionFlag) cmd.Flags().StringVarP(&o.output, "output", "o", "table", outputFlag) + cmd.Flags().StringVar(&o.download, "download", "", regoPolicyDownloadFlag) return cmd } @@ -111,6 +130,32 @@ func (o *getRegoPolicyOptions) run(out io.Writer, identifier string) error { }) } +func (o *getRegoPolicyOptions) runDownload(out io.Writer, identifier string) error { + endpoint, err := regopolicies.BundleURL(global.Host, global.Org, identifier, o.version) + if err != nil { + return err + } + response, err := kosliClient.Do(&requests.RequestParams{ + Method: http.MethodGet, + URL: endpoint, + Token: global.ApiToken, + }) + if err != nil { + return err + } + written, err := regopolicies.ExtractBundle([]byte(response.Body), o.download) + if err != nil { + return err + } + noun := "files" + if len(written) == 1 { + noun = "file" + } + _, err = fmt.Fprintf(out, "Downloaded %d %s of version %d of rego policy '%s' to %s\n", + len(written), noun, o.version, identifier, o.download) + return err +} + func printRegoPolicyAsTable(raw string, out io.Writer, page int) error { var policy map[string]any if err := json.Unmarshal([]byte(raw), &policy); err != nil { diff --git a/cmd/kosli/getRegoPolicy_test.go b/cmd/kosli/getRegoPolicy_test.go index 1af1f083e..413e51cfc 100644 --- a/cmd/kosli/getRegoPolicy_test.go +++ b/cmd/kosli/getRegoPolicy_test.go @@ -1,9 +1,14 @@ package main import ( + "archive/tar" + "bytes" + "compress/gzip" "fmt" "io" "net/http" + "os" + "path/filepath" "testing" "github.com/stretchr/testify/require" @@ -236,6 +241,90 @@ func (suite *GetRegoPolicyCommandTestSuite) TestItReturnsANotFoundInTheServersWo require.Contains(suite.T(), combined, "Rego policy 'pr-approved' has no version 9 in organization 'test-org'") } +func regoPolicyBundle(t *testing.T, files map[string]string) string { + t.Helper() + var archive bytes.Buffer + zipped := gzip.NewWriter(&archive) + tw := tar.NewWriter(zipped) + for _, path := range []string{"lib/helpers.rego", "policy.rego"} { + body, ok := files[path] + if !ok { + continue + } + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: path, Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg, + })) + _, err := tw.Write([]byte(body)) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + require.NoError(t, zipped.Close()) + return archive.String() +} + +func (suite *GetRegoPolicyCommandTestSuite) TestItDownloadsTheFilesOfAVersionIntoADirectory() { + files := map[string]string{ + "lib/helpers.rego": "package lib\n", + "policy.rego": "package policy\n", + } + server, fake := newFakeRegoPolicyRead(suite.T(), regoPolicyBundle(suite.T(), files)) + dir := filepath.Join(suite.T().TempDir(), "pr-approved-v3") + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, + "pr-approved --policy-version 3 --download "+dir)) + + require.NoError(suite.T(), err) + require.Equal(suite.T(), "/api/v2/rego-policies/test-org/pr-approved/versions/3/bundle", fake.escapedPath) + require.Equal(suite.T(), + fmt.Sprintf("Downloaded 2 files of version 3 of rego policy 'pr-approved' to %s\n", dir), + combined) + for path, body := range files { + written, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(path))) + require.NoError(suite.T(), err, path) + require.Equal(suite.T(), body, string(written), path) + } +} + +func (suite *GetRegoPolicyCommandTestSuite) TestItCountsASingleDownloadedFileInTheSingular() { + bundle := regoPolicyBundle(suite.T(), map[string]string{"policy.rego": "package policy\n"}) + server, _ := newFakeRegoPolicyRead(suite.T(), bundle) + dir := suite.T().TempDir() + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, + "pr-approved --policy-version 1 --download "+dir)) + + require.NoError(suite.T(), err) + require.Equal(suite.T(), + fmt.Sprintf("Downloaded 1 file of version 1 of rego policy 'pr-approved' to %s\n", dir), + combined) +} + +func (suite *GetRegoPolicyCommandTestSuite) TestItRefusesADownloadWithoutAVersion() { + server, fake := newFakeRegoPolicyRead(suite.T(), regoPolicyWithCurrentVersion) + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, + "pr-approved --download "+suite.T().TempDir())) + + require.Error(suite.T(), err) + require.Contains(suite.T(), combined, "--download needs --policy-version") + require.Equal(suite.T(), 0, fake.calls) +} + +func (suite *GetRegoPolicyCommandTestSuite) TestItWritesNothingWhenTheVersionIsNotFound() { + server, fake := newFakeRegoPolicyRead(suite.T(), + `{"message":"Rego policy 'pr-approved' has no version 9 in organization 'test-org'"}`) + fake.status = http.StatusNotFound + dir := filepath.Join(suite.T().TempDir(), "pr-approved-v9") + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, + "pr-approved --policy-version 9 --download "+dir)) + + require.Error(suite.T(), err) + require.Contains(suite.T(), combined, "Rego policy 'pr-approved' has no version 9 in organization 'test-org'") + require.Equal(suite.T(), 1, fake.calls) + require.NoDirExists(suite.T(), dir) +} + func TestGetRegoPolicyCommandTestSuite(t *testing.T) { suite.Run(t, new(GetRegoPolicyCommandTestSuite)) } diff --git a/cmd/kosli/root.go b/cmd/kosli/root.go index 6aac93f0c..83d032d2d 100644 --- a/cmd/kosli/root.go +++ b/cmd/kosli/root.go @@ -164,6 +164,7 @@ Paths the list already matches stay excluded whatever is later added there, so k regoPolicyDescriptionFlag = "[optional] The description of the rego policy. Without it the current description is kept." regoPolicyCommentFlag = "[optional] Why this version is published. It is kept with the version." regoPolicyVersionFlag = "[optional] The version of the rego policy to get. Without it, the policy and its current version are shown." + regoPolicyDownloadFlag = "[optional] The directory to write the files of the version given by --policy-version into. It is created if it does not exist, and no existing file in it is overwritten." regoPolicySearchFlag = "[optional] Only list rego policies whose name or identifier contains this substring (case-insensitive)." regoPolicySortFlag = "[optional] The field to sort rego policies by. Valid values are: [updated_at, identifier]. (defaults to updated_at)" regoPolicySortDirectionFlag = "[optional] The direction to sort rego policies in. Valid values are: [asc, desc]. (defaults to desc)" diff --git a/cmd/kosli/testdata/empty-flag-audit-coverage.json b/cmd/kosli/testdata/empty-flag-audit-coverage.json index 0a09464af..d2247dba3 100644 --- a/cmd/kosli/testdata/empty-flag-audit-coverage.json +++ b/cmd/kosli/testdata/empty-flag-audit-coverage.json @@ -679,6 +679,7 @@ "output": "string" }, "get rego-policy": { + "download": "string", "output": "string", "policy-version": "int" }, diff --git a/hack/empty-flag-audit/spec.json b/hack/empty-flag-audit/spec.json index 64f0bc553..bd3a1cf80 100644 --- a/hack/empty-flag-audit/spec.json +++ b/hack/empty-flag-audit/spec.json @@ -1562,10 +1562,12 @@ "error": "the server the audit runs against does not offer the rego policy endpoints", "needs": "server-side evaluation", "flags_to_test": [ + "download", "output", "policy-version" ], "flag_values": { + "download": "probe-bundle", "output": "json", "policy-version": "1" }, diff --git a/internal/regopolicies/bundle.go b/internal/regopolicies/bundle.go new file mode 100644 index 000000000..ef9a02ae0 --- /dev/null +++ b/internal/regopolicies/bundle.go @@ -0,0 +1,76 @@ +package regopolicies + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" +) + +type bundleFile struct { + path string + content []byte +} + +// ExtractBundle writes the files of a version's bundle into dir and returns their +// paths in archive order. +func ExtractBundle(archive []byte, dir string) ([]string, error) { + files, err := readBundle(archive) + if err != nil { + return nil, err + } + for _, f := range files { + target := filepath.Join(dir, filepath.FromSlash(f.path)) + if _, err := os.Lstat(target); err == nil { + return nil, fmt.Errorf("%s already exists, so the bundle was not written", target) + } else if !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + } + written := make([]string, 0, len(files)) + for _, f := range files { + target := filepath.Join(dir, filepath.FromSlash(f.path)) + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return written, err + } + if err := os.WriteFile(target, f.content, 0o644); err != nil { + return written, err + } + written = append(written, f.path) + } + return written, nil +} + +func readBundle(archive []byte) ([]bundleFile, error) { + zipped, err := gzip.NewReader(bytes.NewReader(archive)) + if err != nil { + return nil, fmt.Errorf("the rego policy bundle is not a gzipped tar: %w", err) + } + tr := tar.NewReader(zipped) + var files []bundleFile + for { + header, err := tr.Next() + if errors.Is(err, io.EOF) { + return files, nil + } + if err != nil { + return nil, fmt.Errorf("the rego policy bundle is not a gzipped tar: %w", err) + } + if header.Typeflag != tar.TypeReg { + return nil, fmt.Errorf("the rego policy bundle holds %q, which is not a regular file", header.Name) + } + if !filepath.IsLocal(filepath.FromSlash(header.Name)) { + return nil, fmt.Errorf("the rego policy bundle holds %q, a path outside the directory", header.Name) + } + content, err := io.ReadAll(tr) + if err != nil { + return nil, err + } + files = append(files, bundleFile{path: header.Name, content: content}) + } +} diff --git a/internal/regopolicies/bundle_test.go b/internal/regopolicies/bundle_test.go new file mode 100644 index 000000000..9056fa309 --- /dev/null +++ b/internal/regopolicies/bundle_test.go @@ -0,0 +1,128 @@ +package regopolicies + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +type member struct { + name string + body string + typeflag byte +} + +func tarball(t *testing.T, members ...member) []byte { + t.Helper() + var archive bytes.Buffer + zipped := gzip.NewWriter(&archive) + tw := tar.NewWriter(zipped) + for _, m := range members { + typeflag := m.typeflag + if typeflag == 0 { + typeflag = tar.TypeReg + } + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: m.name, Mode: 0o644, Size: int64(len(m.body)), Typeflag: typeflag, + })) + _, err := tw.Write([]byte(m.body)) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + require.NoError(t, zipped.Close()) + return archive.Bytes() +} + +func filesIn(t *testing.T, dir string) map[string]string { + t.Helper() + files := map[string]string{} + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + content, err := os.ReadFile(path) + if err != nil { + return err + } + rel, err := filepath.Rel(dir, path) + files[filepath.ToSlash(rel)] = string(content) + return err + }) + require.NoError(t, err) + return files +} + +func TestExtractBundleWritesEachFileAtItsPath(t *testing.T) { + dir := t.TempDir() + archive := tarball(t, + member{name: "lib/my café.rego", body: "package lib\n"}, + member{name: "policy.rego", body: "package policy\n"}, + ) + + written, err := ExtractBundle(archive, dir) + + require.NoError(t, err) + require.Equal(t, []string{"lib/my café.rego", "policy.rego"}, written) + require.Equal(t, map[string]string{ + "lib/my café.rego": "package lib\n", + "policy.rego": "package policy\n", + }, filesIn(t, dir)) +} + +func TestExtractBundleRefusesAPathOutsideTheDirectoryAndWritesNothing(t *testing.T) { + for _, name := range []string{"../escape.rego", "lib/../../escape.rego", "/etc/escape.rego"} { + parent := t.TempDir() + dir := filepath.Join(parent, "out") + archive := tarball(t, + member{name: "policy.rego", body: "package policy\n"}, + member{name: name, body: "package escape\n"}, + ) + + _, err := ExtractBundle(archive, dir) + + require.ErrorContains(t, err, name, name) + require.Empty(t, filesIn(t, parent), name) + } +} + +func TestExtractBundleRefusesAMemberThatIsNotARegularFileAndWritesNothing(t *testing.T) { + dir := t.TempDir() + archive := tarball(t, + member{name: "policy.rego", body: "package policy\n"}, + member{name: "link.rego", typeflag: tar.TypeSymlink}, + ) + + _, err := ExtractBundle(archive, dir) + + require.ErrorContains(t, err, "link.rego") + require.Empty(t, filesIn(t, dir)) +} + +func TestExtractBundleRefusesToOverwriteAFileAndWritesNothing(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "policy.rego"), []byte("mine\n"), 0o644)) + archive := tarball(t, + member{name: "a.rego", body: "package a\n"}, + member{name: "policy.rego", body: "package policy\n"}, + ) + + _, err := ExtractBundle(archive, dir) + + require.ErrorContains(t, err, "policy.rego") + require.Equal(t, map[string]string{"policy.rego": "mine\n"}, filesIn(t, dir)) +} + +func TestExtractBundleCreatesTheDirectory(t *testing.T) { + dir := filepath.Join(t.TempDir(), "pr-approved-v3") + archive := tarball(t, member{name: "policy.rego", body: "package policy\n"}) + + _, err := ExtractBundle(archive, dir) + + require.NoError(t, err) + require.Equal(t, map[string]string{"policy.rego": "package policy\n"}, filesIn(t, dir)) +} diff --git a/internal/regopolicies/client.go b/internal/regopolicies/client.go index 887ba695f..1d2bef1a1 100644 --- a/internal/regopolicies/client.go +++ b/internal/regopolicies/client.go @@ -157,6 +157,15 @@ func VersionURL(host, org, identifier string, version int) (string, error) { return url.JoinPath(endpoint, "versions", strconv.Itoa(version)) } +// BundleURL is the endpoint of the gzipped tar of one version's files. +func BundleURL(host, org, identifier string, version int) (string, error) { + endpoint, err := VersionURL(host, org, identifier, version) + if err != nil { + return "", err + } + return url.JoinPath(endpoint, "bundle") +} + // The server refuses unknown fields, so the wire shape is stated here exactly. type publishPayload struct { Name string `json:"name,omitempty"` diff --git a/internal/regopolicies/urls_test.go b/internal/regopolicies/urls_test.go index 5a2aa698a..261ce1521 100644 --- a/internal/regopolicies/urls_test.go +++ b/internal/regopolicies/urls_test.go @@ -34,3 +34,10 @@ func TestListURLNamesTheOrg(t *testing.T) { require.NoError(t, err) require.Equal(t, "https://app.kosli.com/api/v2/rego-policies/my%20org", endpoint) } + +func TestBundleURLNamesTheBundleOfTheVersion(t *testing.T) { + endpoint, err := BundleURL("https://app.kosli.com", "my-org", "pr-approved", 3) + + require.NoError(t, err) + require.Equal(t, "https://app.kosli.com/api/v2/rego-policies/my-org/pr-approved/versions/3/bundle", endpoint) +} From ef8d15f9d662e7b63b10e360f3a8991020760e03 Mon Sep 17 00:00:00 2001 From: Simon Castagna Date: Thu, 22 Oct 2026 18:44:56 +0200 Subject: [PATCH 2/5] fix: refuse colliding bundle paths and write the bundle through os.Root Review of #1290 found ways past the promise that a refused bundle writes nothing. A path held twice was written twice, and a path held both as a file and as a directory failed after part of the bundle was written. Both are refused now, before any write. The server does not refuse the second case at publish, so a real bundle can hold it. Writes go through os.Root with O_EXCL, so a symlink already inside the directory cannot lead a write out of it. A bundle over 8 MiB decompressed is refused, and directory entries are accepted. A test pins the refusal of an empty --download, which the root command already gives. Co-Authored-By: Claude Opus 5.5 --- cmd/kosli/getRegoPolicy_test.go | 12 ++++ internal/regopolicies/bundle.go | 95 ++++++++++++++++++++++++---- internal/regopolicies/bundle_test.go | 68 ++++++++++++++++++++ 3 files changed, 161 insertions(+), 14 deletions(-) diff --git a/cmd/kosli/getRegoPolicy_test.go b/cmd/kosli/getRegoPolicy_test.go index 413e51cfc..beee6efa9 100644 --- a/cmd/kosli/getRegoPolicy_test.go +++ b/cmd/kosli/getRegoPolicy_test.go @@ -310,6 +310,18 @@ func (suite *GetRegoPolicyCommandTestSuite) TestItRefusesADownloadWithoutAVersio require.Equal(suite.T(), 0, fake.calls) } +// An unset variable in --download "$OUT" must fail, not fall back to printing the version. +func (suite *GetRegoPolicyCommandTestSuite) TestItRefusesAnEmptyDownloadDirectory() { + server, fake := newFakeRegoPolicyRead(suite.T(), regoPolicyValid) + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, + `pr-approved --policy-version 1 --download ""`)) + + require.Error(suite.T(), err) + require.Contains(suite.T(), combined, "flag '--download' was given an empty value") + require.Equal(suite.T(), 0, fake.calls) +} + func (suite *GetRegoPolicyCommandTestSuite) TestItWritesNothingWhenTheVersionIsNotFound() { server, fake := newFakeRegoPolicyRead(suite.T(), `{"message":"Rego policy 'pr-approved' has no version 9 in organization 'test-org'"}`) diff --git a/internal/regopolicies/bundle.go b/internal/regopolicies/bundle.go index ef9a02ae0..03bad6a71 100644 --- a/internal/regopolicies/bundle.go +++ b/internal/regopolicies/bundle.go @@ -9,68 +9,135 @@ import ( "io" "io/fs" "os" + "path" "path/filepath" ) +// The server caps a bundle at 1 MiB; the margin is for tar headers and a future raise. +const maxBundleBytes = 8 << 20 + type bundleFile struct { path string content []byte } // ExtractBundle writes the files of a version's bundle into dir and returns their -// paths in archive order. +// paths in archive order. It writes nothing unless every file can be written. func ExtractBundle(archive []byte, dir string) ([]string, error) { files, err := readBundle(archive) if err != nil { return nil, err } + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, err + } + // Writing through a Root keeps a symlink already inside dir from leading a write out of it. + root, err := os.OpenRoot(dir) + if err != nil { + return nil, err + } + defer func() { _ = root.Close() }() + for _, f := range files { - target := filepath.Join(dir, filepath.FromSlash(f.path)) - if _, err := os.Lstat(target); err == nil { - return nil, fmt.Errorf("%s already exists, so the bundle was not written", target) + if _, err := root.Lstat(f.path); err == nil { + return nil, fmt.Errorf("%s already exists, so the bundle was not written", filepath.Join(dir, f.path)) } else if !errors.Is(err, fs.ErrNotExist) { - return nil, err + return nil, fmt.Errorf("cannot write %q into %s: %w", f.path, dir, err) } } written := make([]string, 0, len(files)) for _, f := range files { - target := filepath.Join(dir, filepath.FromSlash(f.path)) - if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { - return written, err - } - if err := os.WriteFile(target, f.content, 0o644); err != nil { - return written, err + if err := writeFile(root, f); err != nil { + return written, fmt.Errorf("cannot write %q into %s: %w", f.path, dir, err) } written = append(written, f.path) } return written, nil } +func writeFile(root *os.Root, f bundleFile) error { + if err := root.MkdirAll(path.Dir(f.path), 0o755); err != nil { + return err + } + out, err := root.OpenFile(f.path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return err + } + if _, err := out.Write(f.content); err != nil { + _ = out.Close() + return err + } + return out.Close() +} + func readBundle(archive []byte) ([]bundleFile, error) { zipped, err := gzip.NewReader(bytes.NewReader(archive)) if err != nil { return nil, fmt.Errorf("the rego policy bundle is not a gzipped tar: %w", err) } - tr := tar.NewReader(zipped) + limited := &io.LimitedReader{R: zipped, N: maxBundleBytes + 1} + tr := tar.NewReader(limited) var files []bundleFile + isFile := map[string]bool{} + isDir := map[string]bool{} for { header, err := tr.Next() + if limited.N <= 0 { + return nil, errBundleTooLarge + } if errors.Is(err, io.EOF) { return files, nil } if err != nil { return nil, fmt.Errorf("the rego policy bundle is not a gzipped tar: %w", err) } - if header.Typeflag != tar.TypeReg { + if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeDir { return nil, fmt.Errorf("the rego policy bundle holds %q, which is not a regular file", header.Name) } if !filepath.IsLocal(filepath.FromSlash(header.Name)) { return nil, fmt.Errorf("the rego policy bundle holds %q, a path outside the directory", header.Name) } + name := path.Clean(header.Name) + if err := markParents(name, isFile, isDir); err != nil { + return nil, err + } + if header.Typeflag == tar.TypeDir { + if isFile[name] { + return nil, bothFileAndDirectory(name) + } + isDir[name] = true + continue + } + if isFile[name] { + return nil, fmt.Errorf("the rego policy bundle holds %q twice", name) + } + if isDir[name] { + return nil, bothFileAndDirectory(name) + } + isFile[name] = true content, err := io.ReadAll(tr) + if limited.N <= 0 { + return nil, errBundleTooLarge + } if err != nil { return nil, err } - files = append(files, bundleFile{path: header.Name, content: content}) + files = append(files, bundleFile{path: name, content: content}) } } + +var errBundleTooLarge = fmt.Errorf("the rego policy bundle is over the %d MiB limit", maxBundleBytes>>20) + +func markParents(name string, isFile, isDir map[string]bool) error { + for parent := path.Dir(name); parent != "."; parent = path.Dir(parent) { + if isFile[parent] { + return bothFileAndDirectory(parent) + } + isDir[parent] = true + } + return nil +} + +func bothFileAndDirectory(name string) error { + return fmt.Errorf("the rego policy bundle holds %q both as a file and as a directory", name) +} diff --git a/internal/regopolicies/bundle_test.go b/internal/regopolicies/bundle_test.go index 9056fa309..523b1565e 100644 --- a/internal/regopolicies/bundle_test.go +++ b/internal/regopolicies/bundle_test.go @@ -126,3 +126,71 @@ func TestExtractBundleCreatesTheDirectory(t *testing.T) { require.NoError(t, err) require.Equal(t, map[string]string{"policy.rego": "package policy\n"}, filesIn(t, dir)) } + +func TestExtractBundleRefusesAPathItHoldsTwiceAndWritesNothing(t *testing.T) { + dir := t.TempDir() + archive := tarball(t, + member{name: "policy.rego", body: "package one\n"}, + member{name: "policy.rego", body: "package two\n"}, + ) + + _, err := ExtractBundle(archive, dir) + + require.ErrorContains(t, err, `"policy.rego" twice`) + require.Empty(t, filesIn(t, dir)) +} + +func TestExtractBundleRefusesAPathThatIsBothAFileAndADirectoryAndWritesNothing(t *testing.T) { + for _, members := range [][]member{ + {{name: "lib", body: "x\n"}, {name: "lib/x.rego", body: "package x\n"}}, + {{name: "lib/x.rego", body: "package x\n"}, {name: "lib", body: "x\n"}}, + } { + dir := t.TempDir() + + _, err := ExtractBundle(tarball(t, members...), dir) + + require.ErrorContains(t, err, `"lib" both as a file and as a directory`) + require.Empty(t, filesIn(t, dir)) + } +} + +func TestExtractBundleDoesNotFollowASymlinkOutOfTheDirectory(t *testing.T) { + outside := t.TempDir() + dir := t.TempDir() + require.NoError(t, os.Symlink(outside, filepath.Join(dir, "lib"))) + archive := tarball(t, + member{name: "a.rego", body: "package a\n"}, + member{name: "lib/x.rego", body: "package x\n"}, + ) + + _, err := ExtractBundle(archive, dir) + + require.ErrorContains(t, err, "lib/x.rego") + require.Empty(t, filesIn(t, outside)) + _, statErr := os.Lstat(filepath.Join(dir, "a.rego")) + require.ErrorIs(t, statErr, os.ErrNotExist) +} + +func TestExtractBundleAcceptsDirectoryEntries(t *testing.T) { + dir := t.TempDir() + archive := tarball(t, + member{name: "lib/", typeflag: tar.TypeDir}, + member{name: "lib/x.rego", body: "package x\n"}, + ) + + written, err := ExtractBundle(archive, dir) + + require.NoError(t, err) + require.Equal(t, []string{"lib/x.rego"}, written) + require.Equal(t, map[string]string{"lib/x.rego": "package x\n"}, filesIn(t, dir)) +} + +func TestExtractBundleRefusesABundleOverTheSizeLimit(t *testing.T) { + dir := t.TempDir() + archive := tarball(t, member{name: "big.rego", body: string(make([]byte, maxBundleBytes+1))}) + + _, err := ExtractBundle(archive, dir) + + require.ErrorContains(t, err, "over the 8 MiB limit") + require.Empty(t, filesIn(t, dir)) +} From 002f85d52ad609219176f12bcc44c688939d0294 Mon Sep 17 00:00:00 2001 From: Simon Castagna Date: Mon, 26 Oct 2026 14:45:20 +0100 Subject: [PATCH 3/5] fix: bound the bundle size, share the path rule, and remove a directory left by a failed write Review of #1290. The decompression cap is a 100 MiB safety bound, no longer tied to the server's upload rule. Entry names go through utils.LocalRelativePath, so a leading "/" is trimmed and ".. " or "..." segments are refused as in the Azure unzip. The already-exists error says how to recover. A directory the command created is removed when a write fails; one that existed before is kept. Co-Authored-By: Claude Fable 5.1 --- internal/regopolicies/bundle.go | 33 +++++++++++---- internal/regopolicies/bundle_test.go | 62 +++++++++++++++++++++++++++- 2 files changed, 86 insertions(+), 9 deletions(-) diff --git a/internal/regopolicies/bundle.go b/internal/regopolicies/bundle.go index 03bad6a71..f8abdd336 100644 --- a/internal/regopolicies/bundle.go +++ b/internal/regopolicies/bundle.go @@ -11,10 +11,13 @@ import ( "os" "path" "path/filepath" + + "github.com/kosli-dev/cli/internal/utils" ) -// The server caps a bundle at 1 MiB; the margin is for tar headers and a future raise. -const maxBundleBytes = 8 << 20 +// A safety bound against a decompression bomb, as the whole bundle is held in memory. +// It is far above the server's upload cap on purpose, so a raise there needs no CLI release. +const maxBundleBytes = 100 << 20 type bundleFile struct { path string @@ -22,15 +25,26 @@ type bundleFile struct { } // ExtractBundle writes the files of a version's bundle into dir and returns their -// paths in archive order. It writes nothing unless every file can be written. +// paths in archive order. It writes nothing unless every file can be written, and +// a directory it created is removed again when a write fails. func ExtractBundle(archive []byte, dir string) ([]string, error) { files, err := readBundle(archive) if err != nil { return nil, err } + _, statErr := os.Lstat(dir) + created := errors.Is(statErr, fs.ErrNotExist) if err := os.MkdirAll(dir, 0o755); err != nil { return nil, err } + written, err := writeFiles(files, dir) + if err != nil && created { + _ = os.RemoveAll(dir) + } + return written, err +} + +func writeFiles(files []bundleFile, dir string) ([]string, error) { // Writing through a Root keeps a symlink already inside dir from leading a write out of it. root, err := os.OpenRoot(dir) if err != nil { @@ -40,7 +54,8 @@ func ExtractBundle(archive []byte, dir string) ([]string, error) { for _, f := range files { if _, err := root.Lstat(f.path); err == nil { - return nil, fmt.Errorf("%s already exists, so the bundle was not written", filepath.Join(dir, f.path)) + return nil, fmt.Errorf("%s already exists, so the bundle was not written; remove it or choose another directory", + filepath.Join(dir, f.path)) } else if !errors.Is(err, fs.ErrNotExist) { return nil, fmt.Errorf("cannot write %q into %s: %w", f.path, dir, err) } @@ -94,10 +109,14 @@ func readBundle(archive []byte) ([]bundleFile, error) { if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeDir { return nil, fmt.Errorf("the rego policy bundle holds %q, which is not a regular file", header.Name) } - if !filepath.IsLocal(filepath.FromSlash(header.Name)) { - return nil, fmt.Errorf("the rego policy bundle holds %q, a path outside the directory", header.Name) + rel, err := utils.LocalRelativePath(header.Name) + if errors.Is(err, utils.ErrNamesNoFile) && header.Typeflag == tar.TypeDir { + continue + } + if err != nil { + return nil, fmt.Errorf("the rego policy bundle holds %q, which %w", header.Name, err) } - name := path.Clean(header.Name) + name := path.Clean(rel) if err := markParents(name, isFile, isDir); err != nil { return nil, err } diff --git a/internal/regopolicies/bundle_test.go b/internal/regopolicies/bundle_test.go index 523b1565e..0dd4737b7 100644 --- a/internal/regopolicies/bundle_test.go +++ b/internal/regopolicies/bundle_test.go @@ -6,6 +6,7 @@ import ( "compress/gzip" "os" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/require" @@ -75,7 +76,7 @@ func TestExtractBundleWritesEachFileAtItsPath(t *testing.T) { } func TestExtractBundleRefusesAPathOutsideTheDirectoryAndWritesNothing(t *testing.T) { - for _, name := range []string{"../escape.rego", "lib/../../escape.rego", "/etc/escape.rego"} { + for _, name := range []string{"../escape.rego", "lib/../../escape.rego", "lib/.. /escape.rego", "lib/.../escape.rego"} { parent := t.TempDir() dir := filepath.Join(parent, "out") archive := tarball(t, @@ -114,6 +115,7 @@ func TestExtractBundleRefusesToOverwriteAFileAndWritesNothing(t *testing.T) { _, err := ExtractBundle(archive, dir) require.ErrorContains(t, err, "policy.rego") + require.ErrorContains(t, err, "remove it or choose another directory") require.Equal(t, map[string]string{"policy.rego": "mine\n"}, filesIn(t, dir)) } @@ -191,6 +193,62 @@ func TestExtractBundleRefusesABundleOverTheSizeLimit(t *testing.T) { _, err := ExtractBundle(archive, dir) - require.ErrorContains(t, err, "over the 8 MiB limit") + require.ErrorContains(t, err, "over the 100 MiB limit") + require.Empty(t, filesIn(t, dir)) +} + +func TestExtractBundleTrimsALeadingSlash(t *testing.T) { + dir := t.TempDir() + archive := tarball(t, member{name: "/etc/escape.rego", body: "package escape\n"}) + + written, err := ExtractBundle(archive, dir) + + require.NoError(t, err) + require.Equal(t, []string{"etc/escape.rego"}, written) + require.Equal(t, map[string]string{"etc/escape.rego": "package escape\n"}, filesIn(t, dir)) +} + +func TestExtractBundleSkipsTheCurrentDirectoryEntry(t *testing.T) { + dir := t.TempDir() + archive := tarball(t, + member{name: "./", typeflag: tar.TypeDir}, + member{name: "./policy.rego", body: "package policy\n"}, + ) + + written, err := ExtractBundle(archive, dir) + + require.NoError(t, err) + require.Equal(t, []string{"policy.rego"}, written) + require.Equal(t, map[string]string{"policy.rego": "package policy\n"}, filesIn(t, dir)) +} + +// A file name over the OS limit passes every check on the archive and fails only at the directory. +var unwritableName = strings.Repeat("x", 300) + ".rego" + +func TestExtractBundleRemovesTheDirectoryItCreatedWhenAWriteFails(t *testing.T) { + dir := filepath.Join(t.TempDir(), "out") + archive := tarball(t, + member{name: "policy.rego", body: "package policy\n"}, + member{name: unwritableName, body: "package x\n"}, + ) + + _, err := ExtractBundle(archive, dir) + + require.Error(t, err) + _, statErr := os.Lstat(dir) + require.ErrorIs(t, statErr, os.ErrNotExist) +} + +func TestExtractBundleKeepsADirectoryThatExistedWhenAWriteFails(t *testing.T) { + dir := t.TempDir() + archive := tarball(t, + member{name: "policy.rego", body: "package policy\n"}, + member{name: unwritableName, body: "package x\n"}, + ) + + _, err := ExtractBundle(archive, dir) + + require.Error(t, err) + require.DirExists(t, dir) require.Empty(t, filesIn(t, dir)) } From 1a3bf2661cb8d1e96e572e4ff64a199d2528d32c Mon Sep 17 00:00:00 2001 From: Simon Castagna Date: Mon, 26 Oct 2026 15:00:22 +0100 Subject: [PATCH 4/5] feat: kosli get rego-policy --download writes a version's bundle to a file Review of #1290. The bundle is written as the server sends it, a gzipped tar, with no extraction. This drops internal/regopolicies/bundle.go and its path-safety, size-cap and cleanup code. An existing file is refused and kept. Extraction into a directory can follow as its own slice if users ask for it. Co-Authored-By: Claude Fable 5.1 --- cmd/kosli/getRegoPolicy.go | 37 ++-- cmd/kosli/getRegoPolicy_test.go | 68 +++---- cmd/kosli/root.go | 2 +- internal/regopolicies/bundle.go | 162 ----------------- internal/regopolicies/bundle_test.go | 254 --------------------------- 5 files changed, 55 insertions(+), 468 deletions(-) delete mode 100644 internal/regopolicies/bundle.go delete mode 100644 internal/regopolicies/bundle_test.go diff --git a/cmd/kosli/getRegoPolicy.go b/cmd/kosli/getRegoPolicy.go index 316429dc3..8bfe9b6dd 100644 --- a/cmd/kosli/getRegoPolicy.go +++ b/cmd/kosli/getRegoPolicy.go @@ -2,9 +2,12 @@ package main import ( "encoding/json" + "errors" "fmt" "io" + "io/fs" "net/http" + "os" "github.com/kosli-dev/cli/internal/output" "github.com/kosli-dev/cli/internal/regopolicies" @@ -23,9 +26,9 @@ With ^--policy-version^, that version is shown with its status, its digest, who published it and when, and the reason when it is rejected or failed. The command exits 0 whatever the status of the version. -With ^--download^ and ^--policy-version^, the files of that version are written into the -given directory at the paths they were published with, whatever the status of the -version, in place of printing it. A digest computed from these files equals the version's digest.` +With ^--download^ and ^--policy-version^, the bundle of that version is written to the +given file as a gzipped tar, unchanged and whatever the status of the version, in place +of printing it. An existing file is not overwritten.` const getRegoPolicyExample = ` # get a rego policy and its current version: @@ -39,10 +42,10 @@ kosli get rego-policy pr-approved \ --api-token yourAPIToken \ --org yourOrgName -# download the files of version 3 of a rego policy into ./pr-approved-v3: +# download the bundle of version 3 of a rego policy to ./pr-approved-v3.tgz: kosli get rego-policy pr-approved \ --policy-version 3 \ - --download ./pr-approved-v3 \ + --download ./pr-approved-v3.tgz \ --api-token yourAPIToken \ --org yourOrgName @@ -143,17 +146,27 @@ func (o *getRegoPolicyOptions) runDownload(out io.Writer, identifier string) err if err != nil { return err } - written, err := regopolicies.ExtractBundle([]byte(response.Body), o.download) + if err := writeNewFile(o.download, []byte(response.Body)); err != nil { + return err + } + _, err = fmt.Fprintf(out, "Downloaded version %d of rego policy '%s' to %s\n", + o.version, identifier, o.download) + return err +} + +func writeNewFile(path string, content []byte) error { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if errors.Is(err, fs.ErrExist) { + return fmt.Errorf("%s already exists; remove it or choose another file", path) + } if err != nil { return err } - noun := "files" - if len(written) == 1 { - noun = "file" + if _, err := file.Write(content); err != nil { + _ = file.Close() + return err } - _, err = fmt.Fprintf(out, "Downloaded %d %s of version %d of rego policy '%s' to %s\n", - len(written), noun, o.version, identifier, o.download) - return err + return file.Close() } func printRegoPolicyAsTable(raw string, out io.Writer, page int) error { diff --git a/cmd/kosli/getRegoPolicy_test.go b/cmd/kosli/getRegoPolicy_test.go index beee6efa9..c87ae3928 100644 --- a/cmd/kosli/getRegoPolicy_test.go +++ b/cmd/kosli/getRegoPolicy_test.go @@ -241,69 +241,59 @@ func (suite *GetRegoPolicyCommandTestSuite) TestItReturnsANotFoundInTheServersWo require.Contains(suite.T(), combined, "Rego policy 'pr-approved' has no version 9 in organization 'test-org'") } -func regoPolicyBundle(t *testing.T, files map[string]string) string { +// A gzipped tar, as the server sends it; the command must not look inside it. +func regoPolicyBundle(t *testing.T) string { t.Helper() var archive bytes.Buffer zipped := gzip.NewWriter(&archive) tw := tar.NewWriter(zipped) - for _, path := range []string{"lib/helpers.rego", "policy.rego"} { - body, ok := files[path] - if !ok { - continue - } - require.NoError(t, tw.WriteHeader(&tar.Header{ - Name: path, Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg, - })) - _, err := tw.Write([]byte(body)) - require.NoError(t, err) - } + body := "package policy\n" + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "policy.rego", Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg})) + _, err := tw.Write([]byte(body)) + require.NoError(t, err) require.NoError(t, tw.Close()) require.NoError(t, zipped.Close()) return archive.String() } -func (suite *GetRegoPolicyCommandTestSuite) TestItDownloadsTheFilesOfAVersionIntoADirectory() { - files := map[string]string{ - "lib/helpers.rego": "package lib\n", - "policy.rego": "package policy\n", - } - server, fake := newFakeRegoPolicyRead(suite.T(), regoPolicyBundle(suite.T(), files)) - dir := filepath.Join(suite.T().TempDir(), "pr-approved-v3") +func (suite *GetRegoPolicyCommandTestSuite) TestItDownloadsTheBundleOfAVersionToAFile() { + bundle := regoPolicyBundle(suite.T()) + server, fake := newFakeRegoPolicyRead(suite.T(), bundle) + file := filepath.Join(suite.T().TempDir(), "pr-approved-v3.tgz") _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, - "pr-approved --policy-version 3 --download "+dir)) + "pr-approved --policy-version 3 --download "+file)) require.NoError(suite.T(), err) require.Equal(suite.T(), "/api/v2/rego-policies/test-org/pr-approved/versions/3/bundle", fake.escapedPath) require.Equal(suite.T(), - fmt.Sprintf("Downloaded 2 files of version 3 of rego policy 'pr-approved' to %s\n", dir), + fmt.Sprintf("Downloaded version 3 of rego policy 'pr-approved' to %s\n", file), combined) - for path, body := range files { - written, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(path))) - require.NoError(suite.T(), err, path) - require.Equal(suite.T(), body, string(written), path) - } + written, err := os.ReadFile(file) + require.NoError(suite.T(), err) + require.Equal(suite.T(), bundle, string(written)) } -func (suite *GetRegoPolicyCommandTestSuite) TestItCountsASingleDownloadedFileInTheSingular() { - bundle := regoPolicyBundle(suite.T(), map[string]string{"policy.rego": "package policy\n"}) - server, _ := newFakeRegoPolicyRead(suite.T(), bundle) - dir := suite.T().TempDir() +func (suite *GetRegoPolicyCommandTestSuite) TestItRefusesToOverwriteAnExistingFile() { + server, _ := newFakeRegoPolicyRead(suite.T(), regoPolicyBundle(suite.T())) + file := filepath.Join(suite.T().TempDir(), "pr-approved-v3.tgz") + require.NoError(suite.T(), os.WriteFile(file, []byte("mine\n"), 0o644)) _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, - "pr-approved --policy-version 1 --download "+dir)) + "pr-approved --policy-version 3 --download "+file)) + require.Error(suite.T(), err) + require.Contains(suite.T(), combined, file+" already exists; remove it or choose another file") + kept, err := os.ReadFile(file) require.NoError(suite.T(), err) - require.Equal(suite.T(), - fmt.Sprintf("Downloaded 1 file of version 1 of rego policy 'pr-approved' to %s\n", dir), - combined) + require.Equal(suite.T(), "mine\n", string(kept)) } func (suite *GetRegoPolicyCommandTestSuite) TestItRefusesADownloadWithoutAVersion() { server, fake := newFakeRegoPolicyRead(suite.T(), regoPolicyWithCurrentVersion) _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, - "pr-approved --download "+suite.T().TempDir())) + "pr-approved --download "+filepath.Join(suite.T().TempDir(), "pr-approved.tgz"))) require.Error(suite.T(), err) require.Contains(suite.T(), combined, "--download needs --policy-version") @@ -311,7 +301,7 @@ func (suite *GetRegoPolicyCommandTestSuite) TestItRefusesADownloadWithoutAVersio } // An unset variable in --download "$OUT" must fail, not fall back to printing the version. -func (suite *GetRegoPolicyCommandTestSuite) TestItRefusesAnEmptyDownloadDirectory() { +func (suite *GetRegoPolicyCommandTestSuite) TestItRefusesAnEmptyDownloadFile() { server, fake := newFakeRegoPolicyRead(suite.T(), regoPolicyValid) _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, @@ -326,15 +316,15 @@ func (suite *GetRegoPolicyCommandTestSuite) TestItWritesNothingWhenTheVersionIsN server, fake := newFakeRegoPolicyRead(suite.T(), `{"message":"Rego policy 'pr-approved' has no version 9 in organization 'test-org'"}`) fake.status = http.StatusNotFound - dir := filepath.Join(suite.T().TempDir(), "pr-approved-v9") + file := filepath.Join(suite.T().TempDir(), "pr-approved-v9.tgz") _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, - "pr-approved --policy-version 9 --download "+dir)) + "pr-approved --policy-version 9 --download "+file)) require.Error(suite.T(), err) require.Contains(suite.T(), combined, "Rego policy 'pr-approved' has no version 9 in organization 'test-org'") require.Equal(suite.T(), 1, fake.calls) - require.NoDirExists(suite.T(), dir) + require.NoFileExists(suite.T(), file) } func TestGetRegoPolicyCommandTestSuite(t *testing.T) { diff --git a/cmd/kosli/root.go b/cmd/kosli/root.go index 808788444..c69458649 100644 --- a/cmd/kosli/root.go +++ b/cmd/kosli/root.go @@ -164,7 +164,7 @@ Paths the list already matches stay excluded whatever is later added there, so k regoPolicyDescriptionFlag = "[optional] The description of the rego policy. Without it the current description is kept." regoPolicyCommentFlag = "[optional] Why this version is published. It is kept with the version." regoPolicyVersionFlag = "[optional] The version of the rego policy to get. Without it, the policy and its current version are shown." - regoPolicyDownloadFlag = "[optional] The directory to write the files of the version given by --policy-version into. It is created if it does not exist, and no existing file in it is overwritten." + regoPolicyDownloadFlag = "[optional] The file to write the bundle of the version given by --policy-version to, as a gzipped tar. An existing file is not overwritten." regoPolicySearchFlag = "[optional] Only list rego policies whose name or identifier contains this substring (case-insensitive)." regoPolicySortFlag = "[optional] The field to sort rego policies by. Valid values are: [updated_at, identifier]. (defaults to updated_at)" regoPolicySortDirectionFlag = "[optional] The direction to sort rego policies in. Valid values are: [asc, desc]. (defaults to desc)" diff --git a/internal/regopolicies/bundle.go b/internal/regopolicies/bundle.go deleted file mode 100644 index f8abdd336..000000000 --- a/internal/regopolicies/bundle.go +++ /dev/null @@ -1,162 +0,0 @@ -package regopolicies - -import ( - "archive/tar" - "bytes" - "compress/gzip" - "errors" - "fmt" - "io" - "io/fs" - "os" - "path" - "path/filepath" - - "github.com/kosli-dev/cli/internal/utils" -) - -// A safety bound against a decompression bomb, as the whole bundle is held in memory. -// It is far above the server's upload cap on purpose, so a raise there needs no CLI release. -const maxBundleBytes = 100 << 20 - -type bundleFile struct { - path string - content []byte -} - -// ExtractBundle writes the files of a version's bundle into dir and returns their -// paths in archive order. It writes nothing unless every file can be written, and -// a directory it created is removed again when a write fails. -func ExtractBundle(archive []byte, dir string) ([]string, error) { - files, err := readBundle(archive) - if err != nil { - return nil, err - } - _, statErr := os.Lstat(dir) - created := errors.Is(statErr, fs.ErrNotExist) - if err := os.MkdirAll(dir, 0o755); err != nil { - return nil, err - } - written, err := writeFiles(files, dir) - if err != nil && created { - _ = os.RemoveAll(dir) - } - return written, err -} - -func writeFiles(files []bundleFile, dir string) ([]string, error) { - // Writing through a Root keeps a symlink already inside dir from leading a write out of it. - root, err := os.OpenRoot(dir) - if err != nil { - return nil, err - } - defer func() { _ = root.Close() }() - - for _, f := range files { - if _, err := root.Lstat(f.path); err == nil { - return nil, fmt.Errorf("%s already exists, so the bundle was not written; remove it or choose another directory", - filepath.Join(dir, f.path)) - } else if !errors.Is(err, fs.ErrNotExist) { - return nil, fmt.Errorf("cannot write %q into %s: %w", f.path, dir, err) - } - } - written := make([]string, 0, len(files)) - for _, f := range files { - if err := writeFile(root, f); err != nil { - return written, fmt.Errorf("cannot write %q into %s: %w", f.path, dir, err) - } - written = append(written, f.path) - } - return written, nil -} - -func writeFile(root *os.Root, f bundleFile) error { - if err := root.MkdirAll(path.Dir(f.path), 0o755); err != nil { - return err - } - out, err := root.OpenFile(f.path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) - if err != nil { - return err - } - if _, err := out.Write(f.content); err != nil { - _ = out.Close() - return err - } - return out.Close() -} - -func readBundle(archive []byte) ([]bundleFile, error) { - zipped, err := gzip.NewReader(bytes.NewReader(archive)) - if err != nil { - return nil, fmt.Errorf("the rego policy bundle is not a gzipped tar: %w", err) - } - limited := &io.LimitedReader{R: zipped, N: maxBundleBytes + 1} - tr := tar.NewReader(limited) - var files []bundleFile - isFile := map[string]bool{} - isDir := map[string]bool{} - for { - header, err := tr.Next() - if limited.N <= 0 { - return nil, errBundleTooLarge - } - if errors.Is(err, io.EOF) { - return files, nil - } - if err != nil { - return nil, fmt.Errorf("the rego policy bundle is not a gzipped tar: %w", err) - } - if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeDir { - return nil, fmt.Errorf("the rego policy bundle holds %q, which is not a regular file", header.Name) - } - rel, err := utils.LocalRelativePath(header.Name) - if errors.Is(err, utils.ErrNamesNoFile) && header.Typeflag == tar.TypeDir { - continue - } - if err != nil { - return nil, fmt.Errorf("the rego policy bundle holds %q, which %w", header.Name, err) - } - name := path.Clean(rel) - if err := markParents(name, isFile, isDir); err != nil { - return nil, err - } - if header.Typeflag == tar.TypeDir { - if isFile[name] { - return nil, bothFileAndDirectory(name) - } - isDir[name] = true - continue - } - if isFile[name] { - return nil, fmt.Errorf("the rego policy bundle holds %q twice", name) - } - if isDir[name] { - return nil, bothFileAndDirectory(name) - } - isFile[name] = true - content, err := io.ReadAll(tr) - if limited.N <= 0 { - return nil, errBundleTooLarge - } - if err != nil { - return nil, err - } - files = append(files, bundleFile{path: name, content: content}) - } -} - -var errBundleTooLarge = fmt.Errorf("the rego policy bundle is over the %d MiB limit", maxBundleBytes>>20) - -func markParents(name string, isFile, isDir map[string]bool) error { - for parent := path.Dir(name); parent != "."; parent = path.Dir(parent) { - if isFile[parent] { - return bothFileAndDirectory(parent) - } - isDir[parent] = true - } - return nil -} - -func bothFileAndDirectory(name string) error { - return fmt.Errorf("the rego policy bundle holds %q both as a file and as a directory", name) -} diff --git a/internal/regopolicies/bundle_test.go b/internal/regopolicies/bundle_test.go deleted file mode 100644 index 0dd4737b7..000000000 --- a/internal/regopolicies/bundle_test.go +++ /dev/null @@ -1,254 +0,0 @@ -package regopolicies - -import ( - "archive/tar" - "bytes" - "compress/gzip" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -type member struct { - name string - body string - typeflag byte -} - -func tarball(t *testing.T, members ...member) []byte { - t.Helper() - var archive bytes.Buffer - zipped := gzip.NewWriter(&archive) - tw := tar.NewWriter(zipped) - for _, m := range members { - typeflag := m.typeflag - if typeflag == 0 { - typeflag = tar.TypeReg - } - require.NoError(t, tw.WriteHeader(&tar.Header{ - Name: m.name, Mode: 0o644, Size: int64(len(m.body)), Typeflag: typeflag, - })) - _, err := tw.Write([]byte(m.body)) - require.NoError(t, err) - } - require.NoError(t, tw.Close()) - require.NoError(t, zipped.Close()) - return archive.Bytes() -} - -func filesIn(t *testing.T, dir string) map[string]string { - t.Helper() - files := map[string]string{} - err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { - if err != nil || d.IsDir() { - return err - } - content, err := os.ReadFile(path) - if err != nil { - return err - } - rel, err := filepath.Rel(dir, path) - files[filepath.ToSlash(rel)] = string(content) - return err - }) - require.NoError(t, err) - return files -} - -func TestExtractBundleWritesEachFileAtItsPath(t *testing.T) { - dir := t.TempDir() - archive := tarball(t, - member{name: "lib/my café.rego", body: "package lib\n"}, - member{name: "policy.rego", body: "package policy\n"}, - ) - - written, err := ExtractBundle(archive, dir) - - require.NoError(t, err) - require.Equal(t, []string{"lib/my café.rego", "policy.rego"}, written) - require.Equal(t, map[string]string{ - "lib/my café.rego": "package lib\n", - "policy.rego": "package policy\n", - }, filesIn(t, dir)) -} - -func TestExtractBundleRefusesAPathOutsideTheDirectoryAndWritesNothing(t *testing.T) { - for _, name := range []string{"../escape.rego", "lib/../../escape.rego", "lib/.. /escape.rego", "lib/.../escape.rego"} { - parent := t.TempDir() - dir := filepath.Join(parent, "out") - archive := tarball(t, - member{name: "policy.rego", body: "package policy\n"}, - member{name: name, body: "package escape\n"}, - ) - - _, err := ExtractBundle(archive, dir) - - require.ErrorContains(t, err, name, name) - require.Empty(t, filesIn(t, parent), name) - } -} - -func TestExtractBundleRefusesAMemberThatIsNotARegularFileAndWritesNothing(t *testing.T) { - dir := t.TempDir() - archive := tarball(t, - member{name: "policy.rego", body: "package policy\n"}, - member{name: "link.rego", typeflag: tar.TypeSymlink}, - ) - - _, err := ExtractBundle(archive, dir) - - require.ErrorContains(t, err, "link.rego") - require.Empty(t, filesIn(t, dir)) -} - -func TestExtractBundleRefusesToOverwriteAFileAndWritesNothing(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "policy.rego"), []byte("mine\n"), 0o644)) - archive := tarball(t, - member{name: "a.rego", body: "package a\n"}, - member{name: "policy.rego", body: "package policy\n"}, - ) - - _, err := ExtractBundle(archive, dir) - - require.ErrorContains(t, err, "policy.rego") - require.ErrorContains(t, err, "remove it or choose another directory") - require.Equal(t, map[string]string{"policy.rego": "mine\n"}, filesIn(t, dir)) -} - -func TestExtractBundleCreatesTheDirectory(t *testing.T) { - dir := filepath.Join(t.TempDir(), "pr-approved-v3") - archive := tarball(t, member{name: "policy.rego", body: "package policy\n"}) - - _, err := ExtractBundle(archive, dir) - - require.NoError(t, err) - require.Equal(t, map[string]string{"policy.rego": "package policy\n"}, filesIn(t, dir)) -} - -func TestExtractBundleRefusesAPathItHoldsTwiceAndWritesNothing(t *testing.T) { - dir := t.TempDir() - archive := tarball(t, - member{name: "policy.rego", body: "package one\n"}, - member{name: "policy.rego", body: "package two\n"}, - ) - - _, err := ExtractBundle(archive, dir) - - require.ErrorContains(t, err, `"policy.rego" twice`) - require.Empty(t, filesIn(t, dir)) -} - -func TestExtractBundleRefusesAPathThatIsBothAFileAndADirectoryAndWritesNothing(t *testing.T) { - for _, members := range [][]member{ - {{name: "lib", body: "x\n"}, {name: "lib/x.rego", body: "package x\n"}}, - {{name: "lib/x.rego", body: "package x\n"}, {name: "lib", body: "x\n"}}, - } { - dir := t.TempDir() - - _, err := ExtractBundle(tarball(t, members...), dir) - - require.ErrorContains(t, err, `"lib" both as a file and as a directory`) - require.Empty(t, filesIn(t, dir)) - } -} - -func TestExtractBundleDoesNotFollowASymlinkOutOfTheDirectory(t *testing.T) { - outside := t.TempDir() - dir := t.TempDir() - require.NoError(t, os.Symlink(outside, filepath.Join(dir, "lib"))) - archive := tarball(t, - member{name: "a.rego", body: "package a\n"}, - member{name: "lib/x.rego", body: "package x\n"}, - ) - - _, err := ExtractBundle(archive, dir) - - require.ErrorContains(t, err, "lib/x.rego") - require.Empty(t, filesIn(t, outside)) - _, statErr := os.Lstat(filepath.Join(dir, "a.rego")) - require.ErrorIs(t, statErr, os.ErrNotExist) -} - -func TestExtractBundleAcceptsDirectoryEntries(t *testing.T) { - dir := t.TempDir() - archive := tarball(t, - member{name: "lib/", typeflag: tar.TypeDir}, - member{name: "lib/x.rego", body: "package x\n"}, - ) - - written, err := ExtractBundle(archive, dir) - - require.NoError(t, err) - require.Equal(t, []string{"lib/x.rego"}, written) - require.Equal(t, map[string]string{"lib/x.rego": "package x\n"}, filesIn(t, dir)) -} - -func TestExtractBundleRefusesABundleOverTheSizeLimit(t *testing.T) { - dir := t.TempDir() - archive := tarball(t, member{name: "big.rego", body: string(make([]byte, maxBundleBytes+1))}) - - _, err := ExtractBundle(archive, dir) - - require.ErrorContains(t, err, "over the 100 MiB limit") - require.Empty(t, filesIn(t, dir)) -} - -func TestExtractBundleTrimsALeadingSlash(t *testing.T) { - dir := t.TempDir() - archive := tarball(t, member{name: "/etc/escape.rego", body: "package escape\n"}) - - written, err := ExtractBundle(archive, dir) - - require.NoError(t, err) - require.Equal(t, []string{"etc/escape.rego"}, written) - require.Equal(t, map[string]string{"etc/escape.rego": "package escape\n"}, filesIn(t, dir)) -} - -func TestExtractBundleSkipsTheCurrentDirectoryEntry(t *testing.T) { - dir := t.TempDir() - archive := tarball(t, - member{name: "./", typeflag: tar.TypeDir}, - member{name: "./policy.rego", body: "package policy\n"}, - ) - - written, err := ExtractBundle(archive, dir) - - require.NoError(t, err) - require.Equal(t, []string{"policy.rego"}, written) - require.Equal(t, map[string]string{"policy.rego": "package policy\n"}, filesIn(t, dir)) -} - -// A file name over the OS limit passes every check on the archive and fails only at the directory. -var unwritableName = strings.Repeat("x", 300) + ".rego" - -func TestExtractBundleRemovesTheDirectoryItCreatedWhenAWriteFails(t *testing.T) { - dir := filepath.Join(t.TempDir(), "out") - archive := tarball(t, - member{name: "policy.rego", body: "package policy\n"}, - member{name: unwritableName, body: "package x\n"}, - ) - - _, err := ExtractBundle(archive, dir) - - require.Error(t, err) - _, statErr := os.Lstat(dir) - require.ErrorIs(t, statErr, os.ErrNotExist) -} - -func TestExtractBundleKeepsADirectoryThatExistedWhenAWriteFails(t *testing.T) { - dir := t.TempDir() - archive := tarball(t, - member{name: "policy.rego", body: "package policy\n"}, - member{name: unwritableName, body: "package x\n"}, - ) - - _, err := ExtractBundle(archive, dir) - - require.Error(t, err) - require.DirExists(t, dir) - require.Empty(t, filesIn(t, dir)) -} From 2333f995621cafb537ea675b5f051493a5370ec3 Mon Sep 17 00:00:00 2001 From: Simon Castagna Date: Mon, 26 Oct 2026 15:11:00 +0100 Subject: [PATCH 5/5] fix: remove the download file when its write fails Review of #1290. A partial file left by a write or close error made a retry fail as "already exists". The file is now removed on either error, so a retry starts clean. No test: a portable test cannot force a write failure on a real file. Co-Authored-By: Claude Fable 5.1 --- cmd/kosli/getRegoPolicy.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cmd/kosli/getRegoPolicy.go b/cmd/kosli/getRegoPolicy.go index 8bfe9b6dd..16ba8bd1f 100644 --- a/cmd/kosli/getRegoPolicy.go +++ b/cmd/kosli/getRegoPolicy.go @@ -162,11 +162,15 @@ func writeNewFile(path string, content []byte) error { if err != nil { return err } - if _, err := file.Write(content); err != nil { - _ = file.Close() - return err + _, err = file.Write(content) + if closeErr := file.Close(); err == nil { + err = closeErr + } + if err != nil { + // A partial file would make a retry fail as "already exists". + _ = os.Remove(path) } - return file.Close() + return err } func printRegoPolicyAsTable(raw string, out io.Writer, page int) error {