From 5313f3e12e02ceb4a1b76366d1d7266009c67c8b Mon Sep 17 00:00:00 2001 From: Edimar Cardoso Date: Fri, 13 Nov 2026 14:41:20 -0300 Subject: [PATCH 1/3] server: confine and verify fetched files before they replace the data Fetch wrote every file it downloaded from KBD_FETCH_URL straight into KBD_DATA and left verification to the later LoadNamespace call. The pack version from the unsigned catalog was part of the file name and was not checked there, so a hostile or compromised mirror could use dot segments in it to write *.json files outside KBD_DATA before any signature check. Fetch now validates the catalog payload (Catalog.Validate checks pack ids, versions and paths) and its namespace, refuses any mirror path that is not a plain relative path, and writes only through an os.Root opened on the data directory. Files go to a random staging directory, are verified there with FollowKeyrings and LoadNamespace against a copy of the trust store (Trust.Clone), and only a verified namespace is renamed into place. The staging directory is always removed, so a failed fetch leaves nothing behind and keeps the previous copy. A symbolic link at the namespace path is replaced, never written through. Tests cover traversal in the version (the old code wrote outside the data directory), absolute and unclean paths, a symbolic link in the data directory and a pack signed by an untrusted key. Co-Authored-By: Claude Opus 5.5 --- cmd/kbd/main.go | 2 +- docs/hosting.md | 8 ++ internal/server/export_test.go | 7 + internal/server/fetch.go | 146 ++++++++++++++++++-- internal/server/fetch_test.go | 239 +++++++++++++++++++++++++++++++++ internal/server/server_test.go | 2 +- signing/keyring.go | 13 ++ 7 files changed, 402 insertions(+), 15 deletions(-) create mode 100644 internal/server/export_test.go create mode 100644 internal/server/fetch_test.go diff --git a/cmd/kbd/main.go b/cmd/kbd/main.go index d0fc0e1..362566b 100644 --- a/cmd/kbd/main.go +++ b/cmd/kbd/main.go @@ -228,7 +228,7 @@ func run(log *slog.Logger) error { } for _, n := range names { fctx, cancel := context.WithTimeout(ctx, 2*time.Minute) - err := server.Fetch(fctx, url, n, data) + err := server.Fetch(fctx, url, n, data, trust) cancel() if err != nil { return fmt.Errorf("fetch %s: %v", n, err) diff --git a/docs/hosting.md b/docs/hosting.md index b748f48..aec0938 100644 --- a/docs/hosting.md +++ b/docs/hosting.md @@ -46,6 +46,14 @@ Configuration by environment (see `cmd/kbd/main.go` for the full list): | `KBD_HOSTING_QUERY_BODY_LOGGED` | `false` | | `KBD_HOSTING_PROVIDER` | `Quave ONE` (optional) | +The mirror is not trusted. kbd downloads into a staging directory inside +`KBD_DATA` (named `.fetch-` and a random suffix), never writes outside +`KBD_DATA` (paths from the mirror are checked, and writes go through a +root confined to it), verifies the staged copy against the pinned +keyrings, and only then moves it to `KBD_DATA/`. A fetch that +fails verification leaves nothing behind and keeps the copy that was +there; kbd then stops with the error. + Instead of fetching, the build can be mounted read-only at `KBD_DATA`. A new content release is a restart (or a rolling update) with the new build; a new online key or delegation is a secret and variable change. diff --git a/internal/server/export_test.go b/internal/server/export_test.go new file mode 100644 index 0000000..6c34bde --- /dev/null +++ b/internal/server/export_test.go @@ -0,0 +1,7 @@ +package server + +// CheckMirrorPath exposes the mirror path check to the external tests. +func CheckMirrorPath(rel string) error { + _, err := localPath(".fetch-test/ns", rel) + return err +} diff --git a/internal/server/fetch.go b/internal/server/fetch.go index 4deffcd..a8fb141 100644 --- a/internal/server/fetch.go +++ b/internal/server/fetch.go @@ -2,31 +2,150 @@ package server import ( "context" + "crypto/rand" "encoding/base64" + "encoding/hex" "encoding/json" + "errors" "fmt" "io" "net/http" "os" + "path" "path/filepath" "strings" + "github.com/openbasalt/knowledge/internal/build" "github.com/openbasalt/knowledge/protocol" "github.com/openbasalt/knowledge/signing" ) +// stagingPrefix names the temporary directories Fetch creates inside the +// data directory. kbd never loads them: namespace names cannot start with +// a dot. +const stagingPrefix = ".fetch-" + // Fetch copies one namespace of a published build (a static mirror or -// another server) into dir/. It checks nothing itself: the files are -// verified by build.LoadNamespace before anything is served, and a file -// that fails there is never used. -func Fetch(ctx context.Context, base, ns, dir string) error { +// another server) into dir/. +// +// The mirror is not trusted. Every catalog field that becomes part of a +// file name is checked against the protocol patterns before use, and every +// write goes through an os.Root opened on dir, so nothing can land outside +// it (no "..", no absolute path, no symbolic link that leaves dir). The +// files are written to a staging directory first and verified there with +// build.LoadNamespace against a copy of trust; only a namespace that +// verifies replaces dir/. When anything fails, the staging directory is +// removed and dir/ is left as it was. +func Fetch(ctx context.Context, base, ns, dir string, trust *signing.Trust) error { if !signing.ValidNamespace(ns) { return fmt.Errorf("namespace %q is not valid", ns) } + if trust == nil { + return errors.New("fetch needs a trust store to verify the namespace") + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + root, err := os.OpenRoot(dir) + if err != nil { + return err + } + defer root.Close() + stage, err := stagingName() + if err != nil { + return err + } + if err := root.Mkdir(stage, 0o700); err != nil { + return err + } + // Always remove the staging directory: after a successful swap it only + // holds the previous copy of the namespace. + defer root.RemoveAll(stage) + staged := path.Join(stage, ns) + if err := download(ctx, root, staged, base, ns); err != nil { + return err + } + // LoadNamespace takes the namespace from the directory name, so the + // staged copy lives at /. FollowKeyrings advances the trust + // copy through rotations the mirror carries, each one verified by the + // keyring before it, exactly as kbd does at start. + check := trust.Clone() + stagedPath := filepath.Join(dir, filepath.FromSlash(staged)) + if err := build.FollowKeyrings(stagedPath, check); err != nil { + return fmt.Errorf("fetched namespace does not verify: %v", err) + } + if _, err := build.LoadNamespace(stagedPath, check); err != nil { + return fmt.Errorf("fetched namespace does not verify: %v", err) + } + return swap(root, staged, ns, path.Join(stage, "previous")) +} + +// stagingName returns a fresh, unpredictable staging directory name. +func stagingName() (string, error) { + b := make([]byte, 12) + if _, err := rand.Read(b); err != nil { + return "", err + } + return stagingPrefix + hex.EncodeToString(b), nil +} + +// swap moves the verified staged namespace to ns. A current ns (a +// directory, or anything else, such as a symbolic link) is first renamed +// to previous, so a link is replaced, never followed; it is restored when +// the second rename fails. +func swap(root *os.Root, staged, ns, previous string) error { + hadOld := false + if _, err := root.Lstat(ns); err == nil { + if err := root.Rename(ns, previous); err != nil { + return err + } + hadOld = true + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + if err := root.Rename(staged, ns); err != nil { + if hadOld { + _ = root.Rename(previous, ns) + } + return err + } + return nil +} + +// localPath turns a slash-separated path from the mirror protocol into a +// path under the staging directory. It refuses anything that is not a +// plain relative path: absolute paths, "..", empty elements and +// backslashes (which are separators on some systems). os.Root enforces the +// same limit again when the file is written. +func localPath(staged, rel string) (string, error) { + if rel == "" || strings.ContainsAny(rel, "\\\x00") || path.Clean(rel) != rel || !filepath.IsLocal(filepath.FromSlash(rel)) { + return "", fmt.Errorf("mirror path %q is not a plain relative path", rel) + } + return path.Join(staged, rel), nil +} + +// writeConfined writes b to rel under staged through root. +func writeConfined(root *os.Root, staged, rel string, b []byte) error { + p, err := localPath(staged, rel) + if err != nil { + return err + } + if err := root.MkdirAll(path.Dir(p), 0o755); err != nil { + return err + } + return root.WriteFile(p, b, 0o644) +} + +// download copies every file of the namespace from the mirror into staged. +// It reads the unsigned payloads only to learn which files exist, and it +// validates each value that becomes part of a path before using it. +func download(ctx context.Context, root *os.Root, staged, base, ns string) error { base = strings.TrimRight(base, "/") + "/kb/v" + protocol.Version + "/" + ns + "/" - dest := filepath.Join(dir, ns) hc := &http.Client{} get := func(rel string) ([]byte, error) { + if _, err := localPath(staged, rel); err != nil { + return nil, err + } req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+rel, nil) if err != nil { return nil, err @@ -43,11 +162,7 @@ func Fetch(ctx context.Context, base, ns, dir string) error { if err != nil { return nil, err } - path := filepath.Join(dest, filepath.FromSlash(rel)) - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - return nil, err - } - return b, os.WriteFile(path, b, 0o644) + return b, writeConfined(root, staged, rel, b) } payload := func(b []byte, v any) error { env, err := signing.ParseEnvelope(b) @@ -84,10 +199,15 @@ func Fetch(ctx context.Context, base, ns, dir string) error { if err := payload(cb, &cat); err != nil { return err } + // Catalog.Validate checks every pack id and version against the + // protocol patterns and that each path is exactly PackPath(id, version). + if err := cat.Validate(); err != nil { + return fmt.Errorf("mirror catalog: %v", err) + } + if cat.Namespace != ns { + return fmt.Errorf("mirror catalog is for namespace %q", cat.Namespace) + } for _, cp := range cat.Packs { - if !protocol.ValidID(cp.ID) || cp.Path != protocol.PackPath(cp.ID, cp.Version) { - return fmt.Errorf("catalog pack %q has an unexpected path", cp.ID) - } pb, err := get(cp.Path) if err != nil { return err diff --git a/internal/server/fetch_test.go b/internal/server/fetch_test.go new file mode 100644 index 0000000..faaa913 --- /dev/null +++ b/internal/server/fetch_test.go @@ -0,0 +1,239 @@ +package server_test + +import ( + "context" + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/openbasalt/knowledge/internal/build" + "github.com/openbasalt/knowledge/internal/server" + "github.com/openbasalt/knowledge/internal/testkit" + "github.com/openbasalt/knowledge/protocol" + "github.com/openbasalt/knowledge/signing" +) + +// mirror serves the kit's build output, with some files replaced. +func mirror(t *testing.T, kit *testkit.Kit, override map[string][]byte) *httptest.Server { + t.Helper() + files := http.StripPrefix("/kb/v"+protocol.Version, http.FileServer(http.Dir(kit.Out))) + prefix := "/kb/v" + protocol.Version + "/" + kit.NS + "/" + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if b, ok := override[strings.TrimPrefix(r.URL.Path, prefix)]; ok { + _, _ = w.Write(b) + return + } + // Any other request outside the published tree is answered too, + // as a hostile mirror would. + if !strings.HasPrefix(r.URL.Path, prefix) { + _, _ = w.Write([]byte(`{"pwned":true}`)) + return + } + files.ServeHTTP(w, r) + })) + t.Cleanup(ts.Close) + return ts +} + +func readCatalog(t *testing.T, kit *testkit.Kit) protocol.Catalog { + t.Helper() + b, err := os.ReadFile(filepath.Join(kit.Dir, "catalog.json")) + if err != nil { + t.Fatal(err) + } + env, err := signing.ParseEnvelope(b) + if err != nil { + t.Fatal(err) + } + raw, err := base64.StdEncoding.DecodeString(env.Payload) + if err != nil { + t.Fatal(err) + } + var cat protocol.Catalog + if err := json.Unmarshal(raw, &cat); err != nil { + t.Fatal(err) + } + return cat +} + +func seal(t *testing.T, typ string, v any, s *signing.Signer) []byte { + t.Helper() + env, err := signing.SealJSON(typ, v, s) + if err != nil { + t.Fatal(err) + } + b, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } + return b +} + +// listTree returns every path under dir (relative), for "nothing left +// behind" checks. +func listTree(t *testing.T, dir string) []string { + t.Helper() + var out []string + err := filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if p != dir { + rel, _ := filepath.Rel(dir, p) + out = append(out, rel) + } + return nil + }) + if err != nil { + t.Fatal(err) + } + return out +} + +// A catalog version with dot segments (signed even by the real publisher) +// is refused before any request for it is made, and nothing is written +// outside the data directory, or inside it. +func TestFetchRefusesTraversalInVersion(t *testing.T) { + kit := testkit.New(t, "../../content/basalt", time.Now()) + top := t.TempDir() + data := filepath.Join(top, "data") + cat := readCatalog(t, kit) + // From //packs/-1, four levels up is top: the old code + // wrote top/escaped.json. + evil := "1/../../../../escaped" + cat.Packs[0].Version = evil + cat.Packs[0].Path = protocol.PackPath(cat.Packs[0].ID, evil) + ts := mirror(t, kit, map[string][]byte{ + "catalog.json": seal(t, protocol.TypeCatalog, cat, kit.Publisher), + cat.Packs[0].Path: []byte(`{"pwned":true}`), + }) + err := server.Fetch(context.Background(), ts.URL, kit.NS, data, kit.Trust(t)) + if err == nil { + t.Fatal("traversal in the pack version was accepted") + } + if got := listTree(t, top); len(got) != 1 || got[0] != "data" { + t.Fatalf("files left behind: %v", got) + } +} + +// Paths that are absolute, climb out, or are not clean are refused by +// the confinement check itself. +func TestFetchLocalPath(t *testing.T) { + for _, rel := range []string{"/etc/x.json", "../x.json", "packs/../../x.json", "packs//x.json", + "./x.json", "", `packs\..\x.json`, "x\x00.json"} { + if err := server.CheckMirrorPath(rel); err == nil { + t.Errorf("%q accepted", rel) + } + } + for _, rel := range []string{"catalog.json", "keyring/1.json", "packs/a-b-1.2.json", "entries/x-y.json"} { + if err := server.CheckMirrorPath(rel); err != nil { + t.Errorf("%q refused: %v", rel, err) + } + } +} + +// A symbolic link at the namespace path that points outside the data +// directory is replaced, not written through. +func TestFetchSymlinkInData(t *testing.T) { + kit := testkit.New(t, "../../content/basalt", time.Now()) + top := t.TempDir() + data := filepath.Join(top, "data") + outside := filepath.Join(top, "outside") + for _, d := range []string{data, outside} { + if err := os.Mkdir(d, 0o755); err != nil { + t.Fatal(err) + } + } + if err := os.Symlink(outside, filepath.Join(data, kit.NS)); err != nil { + t.Fatal(err) + } + ts := mirror(t, kit, nil) + if err := server.Fetch(context.Background(), ts.URL, kit.NS, data, kit.Trust(t)); err != nil { + t.Fatal(err) + } + if got := listTree(t, outside); len(got) != 0 { + t.Fatalf("written through the link: %v", got) + } + fi, err := os.Lstat(filepath.Join(data, kit.NS)) + if err != nil || !fi.IsDir() { + t.Fatalf("namespace is not a plain directory: %v %v", fi, err) + } + if _, err := build.LoadNamespace(filepath.Join(data, kit.NS), kit.Trust(t)); err != nil { + t.Fatal(err) + } + for _, p := range listTree(t, data) { + if strings.HasPrefix(p, ".fetch-") { + t.Fatalf("staging left behind: %s", p) + } + } +} + +// A pack that the trusted publisher did not sign fails verification; the +// staged files are removed and a copy fetched before stays untouched. +func TestFetchUnsignedPack(t *testing.T) { + kit := testkit.New(t, "../../content/basalt", time.Now()) + data := t.TempDir() + good := mirror(t, kit, nil) + if err := server.Fetch(context.Background(), good.URL, kit.NS, data, kit.Trust(t)); err != nil { + t.Fatal(err) + } + before := listTree(t, data) + + // Re-sign the first pack with a key nobody trusts, then give the + // catalog the matching digest and size under the real publisher, so + // only the pack signature is wrong. + cat := readCatalog(t, kit) + cp := cat.Packs[0] + pb, err := os.ReadFile(filepath.Join(kit.Dir, filepath.FromSlash(cp.Path))) + if err != nil { + t.Fatal(err) + } + env, err := signing.ParseEnvelope(pb) + if err != nil { + t.Fatal(err) + } + raw, err := base64.StdEncoding.DecodeString(env.Payload) + if err != nil { + t.Fatal(err) + } + var p protocol.Pack + if err := json.Unmarshal(raw, &p); err != nil { + t.Fatal(err) + } + stranger, err := signing.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + bad := seal(t, protocol.TypePack, p, stranger) + cat.Packs[0].Digest = signing.DigestBytes(bad) + cat.Packs[0].Size = int64(len(bad)) + evil := mirror(t, kit, map[string][]byte{ + "catalog.json": seal(t, protocol.TypeCatalog, cat, kit.Publisher), + cp.Path: bad, + }) + if err := server.Fetch(context.Background(), evil.URL, kit.NS, data, kit.Trust(t)); err == nil { + t.Fatal("a pack with an untrusted signature was accepted") + } + after := listTree(t, data) + if strings.Join(before, "\n") != strings.Join(after, "\n") { + t.Fatalf("data changed after a failed fetch:\nbefore %v\nafter %v", before, after) + } + if _, err := build.LoadNamespace(filepath.Join(data, kit.NS), kit.Trust(t)); err != nil { + t.Fatalf("earlier copy no longer verifies: %v", err) + } + + // Into an empty data directory, nothing at all is left. + empty := t.TempDir() + if err := server.Fetch(context.Background(), evil.URL, kit.NS, empty, kit.Trust(t)); err == nil { + t.Fatal("a pack with an untrusted signature was accepted") + } + if got := listTree(t, empty); len(got) != 0 { + t.Fatalf("files left behind: %v", got) + } +} diff --git a/internal/server/server_test.go b/internal/server/server_test.go index fc917c6..a2eab3a 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -154,7 +154,7 @@ func TestFetch(t *testing.T) { mirror := httptest.NewServer(http.StripPrefix("/kb/v0", http.FileServer(http.Dir(kit.Out)))) defer mirror.Close() dir := t.TempDir() - if err := server.Fetch(context.Background(), mirror.URL, "basalt", dir); err != nil { + if err := server.Fetch(context.Background(), mirror.URL, "basalt", dir, kit.Trust(t)); err != nil { t.Fatal(err) } n, err := build.LoadNamespace(dir+"/basalt", kit.Trust(t)) diff --git a/signing/keyring.go b/signing/keyring.go index 5145906..54c2b11 100644 --- a/signing/keyring.go +++ b/signing/keyring.go @@ -277,6 +277,19 @@ func (t *Trust) Update(env *Envelope) (*Keyring, error) { return next, nil } +// Clone returns an independent trust store with the same keyrings and +// clock, so a caller can verify untrusted data (and follow the rotations +// it carries) without changing t until the data is accepted. +func (t *Trust) Clone() *Trust { + t.mu.RLock() + defer t.mu.RUnlock() + c := &Trust{rings: make(map[string]*Keyring, len(t.rings)), Now: t.Now} + for ns, k := range t.rings { + c.rings[ns] = k + } + return c +} + // Keyring returns the current keyring of a namespace. func (t *Trust) Keyring(ns string) (*Keyring, bool) { t.mu.RLock() From 2d113f350b8ee22b6a5342c2c2b00029562c4a45 Mon Sep 17 00:00:00 2001 From: Edimar Cardoso Date: Fri, 13 Nov 2026 14:41:20 -0300 Subject: [PATCH 2/3] review, source: refuse symbolic links in content kb-review runs on fork pull requests in the pull_request_target workflow with model API keys in its environment, and it read content files with os.ReadFile, which follows symbolic links. A fork could add an entry file that links to a file of the runner; validation happened to reject most such targets, but nothing refused the link itself, and the model's answer is posted publicly. source.Load and review.Units now read through ReadNoFollow, which lstats every path component below a trusted base, accepts only regular files and checks that the opened file is the one that was checked. Load also refuses a namespace or entries directory that is a link, and kb-review refuses a content directory that is one. The fork workflow additionally checks out the pull request with core.symlinks=false, so links arrive as plain files. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/content-review-fork.yml | 5 +- CONTRIBUTING.md | 3 + cmd/kb-review/main.go | 5 ++ internal/review/review.go | 6 +- internal/review/review_test.go | 33 +++++++ internal/source/nofollow.go | 74 ++++++++++++++++ internal/source/nofollow_test.go | 102 ++++++++++++++++++++++ internal/source/source.go | 21 +++-- 8 files changed, 241 insertions(+), 8 deletions(-) create mode 100644 internal/source/nofollow.go create mode 100644 internal/source/nofollow_test.go diff --git a/.github/workflows/content-review-fork.yml b/.github/workflows/content-review-fork.yml index 6203b6f..a6f5f3b 100644 --- a/.github/workflows/content-review-fork.yml +++ b/.github/workflows/content-review-fork.yml @@ -47,7 +47,10 @@ jobs: git -C pr fetch -q --no-tags "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" "$BASE_SHA" "+refs/pull/$PR/head:pr-head" # The fetched head must be the commit the maintainer labeled. test "$(git -C pr rev-parse pr-head)" = "$HEAD_SHA" - git -C pr -c core.hooksPath=/dev/null checkout -q pr-head + # Symbolic links are checked out as plain files holding the + # link text, so nothing in the pull request can point at a runner + # file (kb-review also refuses links on its own). + git -C pr -c core.hooksPath=/dev/null -c core.symlinks=false checkout -q pr-head git -C pr diff --name-only "$BASE_SHA...$HEAD_SHA" > "$RUNNER_TEMP/changed.txt" - name: Review working-directory: pr diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 77f5ab9..6d7a52d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -78,6 +78,9 @@ data (it builds the tool from the base branch and never builds or runs anything from the fork) and removes the label. New commits need the label again. +Symbolic links are not allowed anywhere under `content/`: `kb validate`, +`kb build` and `kb-review` refuse them instead of following them. + ## For maintainers: setup Repository secrets (Settings, Secrets and variables, Actions): diff --git a/cmd/kb-review/main.go b/cmd/kb-review/main.go index aae5896..53bc9ea 100644 --- a/cmd/kb-review/main.go +++ b/cmd/kb-review/main.go @@ -80,6 +80,11 @@ func run(args []string) (int, error) { } // Structural validation first: a model never sees content that fails it. contentDir := filepath.Join(*repo, *content) + // The content tree comes from the pull request: a symbolic link there + // could point at any file of the runner, so none is followed. + if _, err := source.CheckNoSymlinks(*repo, *content); err != nil { + return 1, err + } nsDirs, _ := filepath.Glob(filepath.Join(contentDir, "*", "namespace.yaml")) var actions []string var sources []*source.Source diff --git a/internal/review/review.go b/internal/review/review.go index 09f6887..942dc91 100644 --- a/internal/review/review.go +++ b/internal/review/review.go @@ -14,11 +14,12 @@ import ( "errors" "fmt" "io" - "os" "path/filepath" "slices" "sort" "strings" + + "github.com/openbasalt/knowledge/internal/source" ) // Rule ids the verdict may use. They match the sections of the rules file. @@ -231,7 +232,8 @@ func Units(repoRoot, contentRoot string, changed []string) ([]Unit, error) { var out []Unit for _, u := range byID { for i := range u.Files { - b, err := os.ReadFile(filepath.Join(repoRoot, u.Files[i].Path)) + // Content comes from the pull request: never follow a link. + b, err := source.ReadNoFollow(repoRoot, filepath.FromSlash(u.Files[i].Path)) if err != nil { return nil, err } diff --git a/internal/review/review_test.go b/internal/review/review_test.go index e58d171..49a430f 100644 --- a/internal/review/review_test.go +++ b/internal/review/review_test.go @@ -7,6 +7,8 @@ import ( "io" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "testing" @@ -161,3 +163,34 @@ func TestBatches(t *testing.T) { t.Fatalf("%v", b) } } + +// A fork pull request can add a symbolic link where an entry file should +// be. Units must refuse it instead of sending the target to a model. +func TestUnitsRefuseSymlinks(t *testing.T) { + repo := t.TempDir() + dir := filepath.Join(repo, "content", "basalt", "entries", "x-entry") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + secret := filepath.Join(t.TempDir(), "secret") + if err := os.WriteFile(secret, []byte("---\nid: x\n---\nRUNNER SECRET"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "en.md"), []byte("---\nid: x-entry\n---\nok"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(secret, filepath.Join(dir, "pt-BR.md")); err != nil { + t.Fatal(err) + } + units, err := Units(repo, "content", []string{"content/basalt/entries/x-entry/pt-BR.md"}) + if !errors.Is(err, source.ErrSymlink) { + t.Fatalf("symbolic link not refused: %v", err) + } + for _, u := range units { + for _, f := range u.Files { + if strings.Contains(f.Content, "RUNNER SECRET") { + t.Fatal("link target sent for review") + } + } + } +} diff --git a/internal/source/nofollow.go b/internal/source/nofollow.go new file mode 100644 index 0000000..d26c8e8 --- /dev/null +++ b/internal/source/nofollow.go @@ -0,0 +1,74 @@ +package source + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// ErrSymlink is returned for content paths that are, or pass through, a +// symbolic link. Content comes from pull requests, including forks whose +// review runs next to model API keys, so a link could point at any file of +// the machine that reads it. Links are never followed. +var ErrSymlink = errors.New("symbolic links are not allowed in content") + +// CheckNoSymlinks verifies that every component of rel under base exists +// and is not a symbolic link. base itself is trusted (it is chosen by the +// caller, not by the content). +func CheckNoSymlinks(base, rel string) (os.FileInfo, error) { + rel = filepath.Clean(rel) + if rel == "." { + return os.Lstat(base) + } + if !filepath.IsLocal(rel) { + return nil, fmt.Errorf("%s: path leaves %s", rel, base) + } + cur := base + parts := strings.Split(filepath.ToSlash(rel), "/") + var fi os.FileInfo + for i, p := range parts { + cur = filepath.Join(cur, p) + var err error + if fi, err = os.Lstat(cur); err != nil { + return nil, err + } + if fi.Mode()&os.ModeSymlink != 0 { + return nil, fmt.Errorf("%s: %w", cur, ErrSymlink) + } + if i < len(parts)-1 && !fi.IsDir() { + return nil, fmt.Errorf("%s: not a directory", cur) + } + } + return fi, nil +} + +// ReadNoFollow reads the regular file rel under base, refusing symbolic +// links in any component of rel and anything that is not a regular file +// (devices, pipes). The opened file must be the one that was checked, so a +// swap between the check and the open is refused too. +func ReadNoFollow(base, rel string) ([]byte, error) { + fi, err := CheckNoSymlinks(base, rel) + if err != nil { + return nil, err + } + p := filepath.Join(base, rel) + if !fi.Mode().IsRegular() { + return nil, fmt.Errorf("%s: not a regular file", p) + } + f, err := os.Open(p) + if err != nil { + return nil, err + } + defer f.Close() + got, err := f.Stat() + if err != nil { + return nil, err + } + if !os.SameFile(fi, got) { + return nil, fmt.Errorf("%s: changed while it was read", p) + } + return io.ReadAll(f) +} diff --git a/internal/source/nofollow_test.go b/internal/source/nofollow_test.go new file mode 100644 index 0000000..b9bef40 --- /dev/null +++ b/internal/source/nofollow_test.go @@ -0,0 +1,102 @@ +package source + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +// seedCopy copies the seed namespace to a temporary directory and +// returns the namespace dir and a directory outside it. +func seedCopy(t *testing.T) (string, string) { + t.Helper() + top := t.TempDir() + ns := filepath.Join(top, "content", "basalt") + if err := os.CopyFS(ns, os.DirFS("../../content/basalt")); err != nil { + t.Fatal(err) + } + outside := filepath.Join(top, "outside") + if err := os.Mkdir(outside, 0o755); err != nil { + t.Fatal(err) + } + return ns, outside +} + +// replaceWithLink moves path to outside and leaves a symbolic link to it, +// so the target is valid content: only the link is wrong. +func replaceWithLink(t *testing.T, path, outside string) { + t.Helper() + target := filepath.Join(outside, filepath.Base(path)) + if err := os.Rename(path, target); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, path); err != nil { + t.Fatal(err) + } +} + +// Load refuses a symbolic link anywhere in the namespace, even when the +// file it points to is valid content. +func TestLoadRefusesSymlinks(t *testing.T) { + for _, rel := range []string{ + "entries/nvidia-legacy-580/pt-BR.md", + "entries/nvidia-legacy-580/en.md", + "namespace.yaml", + "entries", + ".", + } { + t.Run(rel, func(t *testing.T) { + ns, outside := seedCopy(t) + // With and without a trailing slash, as kb validate gets it. + for _, d := range []string{ns, ns + "/"} { + if _, err := Load(d); err != nil { + t.Fatalf("copy does not load: %v", err) + } + } + replaceWithLink(t, filepath.Join(ns, rel), outside) + _, err := Load(ns) + if err == nil { + t.Fatal("symbolic link followed") + } + if !errors.Is(err, ErrSymlink) && !containsSymlinkProblem(err) { + t.Fatalf("unexpected error: %v", err) + } + }) + } +} + +func containsSymlinkProblem(err error) bool { + var probs Problems + if !errors.As(err, &probs) { + return false + } + for _, p := range probs { + if errors.Is(p.Err, ErrSymlink) { + return true + } + } + return false +} + +func TestReadNoFollow(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "a.md"), []byte("ok"), 0o644); err != nil { + t.Fatal(err) + } + if b, err := ReadNoFollow(dir, "a.md"); err != nil || string(b) != "ok" { + t.Fatalf("%q %v", b, err) + } + if err := os.Symlink(filepath.Join(dir, "a.md"), filepath.Join(dir, "b.md")); err != nil { + t.Fatal(err) + } + if _, err := ReadNoFollow(dir, "b.md"); !errors.Is(err, ErrSymlink) { + t.Fatalf("link: %v", err) + } + if _, err := ReadNoFollow(dir, "../x.md"); err == nil { + t.Fatal("path outside base accepted") + } + if _, err := ReadNoFollow(dir, "."); err == nil { + t.Fatal("directory read as a file") + } +} diff --git a/internal/source/source.go b/internal/source/source.go index 02f5e20..56b4e12 100644 --- a/internal/source/source.go +++ b/internal/source/source.go @@ -125,11 +125,16 @@ func SplitFrontMatter(b []byte) ([]byte, string, error) { } // Load reads and validates the namespace in dir. All problems are -// returned together. +// returned together. Symbolic links inside dir (and dir itself being one) +// are refused, never followed: see ErrSymlink. func Load(dir string) (*Source, error) { var probs Problems + dir = filepath.Clean(dir) + if _, err := CheckNoSymlinks(filepath.Dir(dir), filepath.Base(dir)); err != nil { + return nil, err + } nsFile := filepath.Join(dir, "namespace.yaml") - b, err := os.ReadFile(nsFile) + b, err := ReadNoFollow(dir, "namespace.yaml") if err != nil { return nil, err } @@ -140,6 +145,9 @@ func Load(dir string) (*Source, error) { probs = append(probs, checkNamespace(nsFile, &ns)...) entriesDir := filepath.Join(dir, "entries") + if _, err := CheckNoSymlinks(dir, "entries"); err != nil { + return nil, err + } des, err := os.ReadDir(entriesDir) if err != nil { return nil, err @@ -208,10 +216,13 @@ func checkNamespace(file string, ns *Namespace) Problems { func loadEntry(dir string, ns *Namespace) (*protocol.Entry, Problems) { var probs Problems enFile := filepath.Join(dir, "en.md") - b, err := os.ReadFile(enFile) - if err != nil { + b, err := ReadNoFollow(dir, "en.md") + if errors.Is(err, os.ErrNotExist) { return nil, Problems{{dir, errors.New("en.md is required")}} } + if err != nil { + return nil, Problems{{enFile, err}} + } fm, body, err := SplitFrontMatter(b) if err != nil { return nil, Problems{{enFile, err}} @@ -257,7 +268,7 @@ func loadEntry(dir string, ns *Namespace) (*protocol.Entry, Problems) { probs = append(probs, Problem{file, fmt.Errorf("language %q is not listed in namespace.yaml", lang)}) continue } - b, err := os.ReadFile(file) + b, err := ReadNoFollow(dir, filepath.Base(file)) if err != nil { probs = append(probs, Problem{file, err}) continue From 48fcd75c1b1d1adb96b8081a625c474c629c2a76 Mon Sep 17 00:00:00 2001 From: Edimar Cardoso Date: Fri, 13 Nov 2026 14:41:20 -0300 Subject: [PATCH 3/3] Build with Go 1.27.4 and yaml 3.0.6, check advisories in CI go.mod declared go 1.27.0, which CI and the signed release build use through setup-go, and the Dockerfile used the floating golang:1.27 tag. govulncheck reports 11 reachable standard library vulnerabilities at 1.27.0, among them GO-2026-6859 (HTTP/1 server crash, fixed only in 1.27.4), so 1.27.3 is not enough. go.mod and the Dockerfile now name 1.27.4 exactly. go.yaml.in/yaml/v3 moves to 3.0.6 (the pending Dependabot update). CI runs govulncheck on every push and pull request and weekly, so a new advisory fails the build without waiting for a code change. govulncheck reports no vulnerabilities after this change. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 5 +++++ Dockerfile | 4 +++- go.mod | 4 ++-- go.sum | 4 ++-- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47775f9..6c17c30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,9 @@ on: push: branches: [main] pull_request: + # Weekly, so a newly published Go or module advisory shows up without a push. + schedule: + - cron: "23 6 * * 1" permissions: contents: read @@ -20,6 +23,8 @@ jobs: go-version-file: go.mod - name: Format and vet run: make lint + - name: Known vulnerabilities (Go toolchain and modules) + run: go run golang.org/x/vuln/cmd/govulncheck@v1.7.0 ./... - name: Unit tests (race detector) run: go test -race ./... - name: Validate content diff --git a/Dockerfile b/Dockerfile index e1331c7..c8415c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,9 @@ # container with a read-only root file system; kbd writes nothing except, # when KBD_FETCH_URL is set, the fetched bundle into KBD_DATA (mount a # small writable volume or tmpfs there). See docs/hosting.md. -FROM golang:1.27 AS build +# Exact patch release, kept equal to the go line in go.mod: update both +# together. A floating tag would make the build depend on its date. +FROM golang:1.27.4 AS build WORKDIR /src COPY go.mod go.sum ./ RUN go mod download diff --git a/go.mod b/go.mod index 0b429e8..a56f80e 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,10 @@ module github.com/openbasalt/knowledge -go 1.27.0 +go 1.27.4 require ( github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 - go.yaml.in/yaml/v3 v3.0.5 + go.yaml.in/yaml/v3 v3.0.6 ) require golang.org/x/text v0.42.0 // indirect diff --git a/go.sum b/go.sum index ff5e6d2..5acf59f 100644 --- a/go.sum +++ b/go.sum @@ -2,7 +2,7 @@ github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxK github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= -go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= -go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM= +go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=