diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47775f9..6c17c30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,9 @@ on: push: branches: [main] pull_request: + # Weekly, so a newly published Go or module advisory shows up without a push. + schedule: + - cron: "23 6 * * 1" permissions: contents: read @@ -20,6 +23,8 @@ jobs: go-version-file: go.mod - name: Format and vet run: make lint + - name: Known vulnerabilities (Go toolchain and modules) + run: go run golang.org/x/vuln/cmd/govulncheck@v1.7.0 ./... - name: Unit tests (race detector) run: go test -race ./... - name: Validate content diff --git a/.github/workflows/content-review-fork.yml b/.github/workflows/content-review-fork.yml index 6203b6f..a6f5f3b 100644 --- a/.github/workflows/content-review-fork.yml +++ b/.github/workflows/content-review-fork.yml @@ -47,7 +47,10 @@ jobs: git -C pr fetch -q --no-tags "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" "$BASE_SHA" "+refs/pull/$PR/head:pr-head" # The fetched head must be the commit the maintainer labeled. test "$(git -C pr rev-parse pr-head)" = "$HEAD_SHA" - git -C pr -c core.hooksPath=/dev/null checkout -q pr-head + # Symbolic links are checked out as plain files holding the + # link text, so nothing in the pull request can point at a runner + # file (kb-review also refuses links on its own). + git -C pr -c core.hooksPath=/dev/null -c core.symlinks=false checkout -q pr-head git -C pr diff --name-only "$BASE_SHA...$HEAD_SHA" > "$RUNNER_TEMP/changed.txt" - name: Review working-directory: pr diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 77f5ab9..6d7a52d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -78,6 +78,9 @@ data (it builds the tool from the base branch and never builds or runs anything from the fork) and removes the label. New commits need the label again. +Symbolic links are not allowed anywhere under `content/`: `kb validate`, +`kb build` and `kb-review` refuse them instead of following them. + ## For maintainers: setup Repository secrets (Settings, Secrets and variables, Actions): diff --git a/Dockerfile b/Dockerfile index e1331c7..c8415c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,9 @@ # container with a read-only root file system; kbd writes nothing except, # when KBD_FETCH_URL is set, the fetched bundle into KBD_DATA (mount a # small writable volume or tmpfs there). See docs/hosting.md. -FROM golang:1.27 AS build +# Exact patch release, kept equal to the go line in go.mod: update both +# together. A floating tag would make the build depend on its date. +FROM golang:1.27.4 AS build WORKDIR /src COPY go.mod go.sum ./ RUN go mod download diff --git a/cmd/kb-review/main.go b/cmd/kb-review/main.go index aae5896..53bc9ea 100644 --- a/cmd/kb-review/main.go +++ b/cmd/kb-review/main.go @@ -80,6 +80,11 @@ func run(args []string) (int, error) { } // Structural validation first: a model never sees content that fails it. contentDir := filepath.Join(*repo, *content) + // The content tree comes from the pull request: a symbolic link there + // could point at any file of the runner, so none is followed. + if _, err := source.CheckNoSymlinks(*repo, *content); err != nil { + return 1, err + } nsDirs, _ := filepath.Glob(filepath.Join(contentDir, "*", "namespace.yaml")) var actions []string var sources []*source.Source diff --git a/cmd/kbd/main.go b/cmd/kbd/main.go index d0fc0e1..362566b 100644 --- a/cmd/kbd/main.go +++ b/cmd/kbd/main.go @@ -228,7 +228,7 @@ func run(log *slog.Logger) error { } for _, n := range names { fctx, cancel := context.WithTimeout(ctx, 2*time.Minute) - err := server.Fetch(fctx, url, n, data) + err := server.Fetch(fctx, url, n, data, trust) cancel() if err != nil { return fmt.Errorf("fetch %s: %v", n, err) diff --git a/docs/hosting.md b/docs/hosting.md index b748f48..aec0938 100644 --- a/docs/hosting.md +++ b/docs/hosting.md @@ -46,6 +46,14 @@ Configuration by environment (see `cmd/kbd/main.go` for the full list): | `KBD_HOSTING_QUERY_BODY_LOGGED` | `false` | | `KBD_HOSTING_PROVIDER` | `Quave ONE` (optional) | +The mirror is not trusted. kbd downloads into a staging directory inside +`KBD_DATA` (named `.fetch-` and a random suffix), never writes outside +`KBD_DATA` (paths from the mirror are checked, and writes go through a +root confined to it), verifies the staged copy against the pinned +keyrings, and only then moves it to `KBD_DATA/`. A fetch that +fails verification leaves nothing behind and keeps the copy that was +there; kbd then stops with the error. + Instead of fetching, the build can be mounted read-only at `KBD_DATA`. A new content release is a restart (or a rolling update) with the new build; a new online key or delegation is a secret and variable change. diff --git a/go.mod b/go.mod index 0b429e8..a56f80e 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,10 @@ module github.com/openbasalt/knowledge -go 1.27.0 +go 1.27.4 require ( github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 - go.yaml.in/yaml/v3 v3.0.5 + go.yaml.in/yaml/v3 v3.0.6 ) require golang.org/x/text v0.42.0 // indirect diff --git a/go.sum b/go.sum index ff5e6d2..5acf59f 100644 --- a/go.sum +++ b/go.sum @@ -2,7 +2,7 @@ github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxK github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= -go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= -go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM= +go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= diff --git a/internal/review/review.go b/internal/review/review.go index 09f6887..942dc91 100644 --- a/internal/review/review.go +++ b/internal/review/review.go @@ -14,11 +14,12 @@ import ( "errors" "fmt" "io" - "os" "path/filepath" "slices" "sort" "strings" + + "github.com/openbasalt/knowledge/internal/source" ) // Rule ids the verdict may use. They match the sections of the rules file. @@ -231,7 +232,8 @@ func Units(repoRoot, contentRoot string, changed []string) ([]Unit, error) { var out []Unit for _, u := range byID { for i := range u.Files { - b, err := os.ReadFile(filepath.Join(repoRoot, u.Files[i].Path)) + // Content comes from the pull request: never follow a link. + b, err := source.ReadNoFollow(repoRoot, filepath.FromSlash(u.Files[i].Path)) if err != nil { return nil, err } diff --git a/internal/review/review_test.go b/internal/review/review_test.go index e58d171..49a430f 100644 --- a/internal/review/review_test.go +++ b/internal/review/review_test.go @@ -7,6 +7,8 @@ import ( "io" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "testing" @@ -161,3 +163,34 @@ func TestBatches(t *testing.T) { t.Fatalf("%v", b) } } + +// A fork pull request can add a symbolic link where an entry file should +// be. Units must refuse it instead of sending the target to a model. +func TestUnitsRefuseSymlinks(t *testing.T) { + repo := t.TempDir() + dir := filepath.Join(repo, "content", "basalt", "entries", "x-entry") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + secret := filepath.Join(t.TempDir(), "secret") + if err := os.WriteFile(secret, []byte("---\nid: x\n---\nRUNNER SECRET"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "en.md"), []byte("---\nid: x-entry\n---\nok"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(secret, filepath.Join(dir, "pt-BR.md")); err != nil { + t.Fatal(err) + } + units, err := Units(repo, "content", []string{"content/basalt/entries/x-entry/pt-BR.md"}) + if !errors.Is(err, source.ErrSymlink) { + t.Fatalf("symbolic link not refused: %v", err) + } + for _, u := range units { + for _, f := range u.Files { + if strings.Contains(f.Content, "RUNNER SECRET") { + t.Fatal("link target sent for review") + } + } + } +} diff --git a/internal/server/export_test.go b/internal/server/export_test.go new file mode 100644 index 0000000..6c34bde --- /dev/null +++ b/internal/server/export_test.go @@ -0,0 +1,7 @@ +package server + +// CheckMirrorPath exposes the mirror path check to the external tests. +func CheckMirrorPath(rel string) error { + _, err := localPath(".fetch-test/ns", rel) + return err +} diff --git a/internal/server/fetch.go b/internal/server/fetch.go index 4deffcd..a8fb141 100644 --- a/internal/server/fetch.go +++ b/internal/server/fetch.go @@ -2,31 +2,150 @@ package server import ( "context" + "crypto/rand" "encoding/base64" + "encoding/hex" "encoding/json" + "errors" "fmt" "io" "net/http" "os" + "path" "path/filepath" "strings" + "github.com/openbasalt/knowledge/internal/build" "github.com/openbasalt/knowledge/protocol" "github.com/openbasalt/knowledge/signing" ) +// stagingPrefix names the temporary directories Fetch creates inside the +// data directory. kbd never loads them: namespace names cannot start with +// a dot. +const stagingPrefix = ".fetch-" + // Fetch copies one namespace of a published build (a static mirror or -// another server) into dir/. It checks nothing itself: the files are -// verified by build.LoadNamespace before anything is served, and a file -// that fails there is never used. -func Fetch(ctx context.Context, base, ns, dir string) error { +// another server) into dir/. +// +// The mirror is not trusted. Every catalog field that becomes part of a +// file name is checked against the protocol patterns before use, and every +// write goes through an os.Root opened on dir, so nothing can land outside +// it (no "..", no absolute path, no symbolic link that leaves dir). The +// files are written to a staging directory first and verified there with +// build.LoadNamespace against a copy of trust; only a namespace that +// verifies replaces dir/. When anything fails, the staging directory is +// removed and dir/ is left as it was. +func Fetch(ctx context.Context, base, ns, dir string, trust *signing.Trust) error { if !signing.ValidNamespace(ns) { return fmt.Errorf("namespace %q is not valid", ns) } + if trust == nil { + return errors.New("fetch needs a trust store to verify the namespace") + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + root, err := os.OpenRoot(dir) + if err != nil { + return err + } + defer root.Close() + stage, err := stagingName() + if err != nil { + return err + } + if err := root.Mkdir(stage, 0o700); err != nil { + return err + } + // Always remove the staging directory: after a successful swap it only + // holds the previous copy of the namespace. + defer root.RemoveAll(stage) + staged := path.Join(stage, ns) + if err := download(ctx, root, staged, base, ns); err != nil { + return err + } + // LoadNamespace takes the namespace from the directory name, so the + // staged copy lives at /. FollowKeyrings advances the trust + // copy through rotations the mirror carries, each one verified by the + // keyring before it, exactly as kbd does at start. + check := trust.Clone() + stagedPath := filepath.Join(dir, filepath.FromSlash(staged)) + if err := build.FollowKeyrings(stagedPath, check); err != nil { + return fmt.Errorf("fetched namespace does not verify: %v", err) + } + if _, err := build.LoadNamespace(stagedPath, check); err != nil { + return fmt.Errorf("fetched namespace does not verify: %v", err) + } + return swap(root, staged, ns, path.Join(stage, "previous")) +} + +// stagingName returns a fresh, unpredictable staging directory name. +func stagingName() (string, error) { + b := make([]byte, 12) + if _, err := rand.Read(b); err != nil { + return "", err + } + return stagingPrefix + hex.EncodeToString(b), nil +} + +// swap moves the verified staged namespace to ns. A current ns (a +// directory, or anything else, such as a symbolic link) is first renamed +// to previous, so a link is replaced, never followed; it is restored when +// the second rename fails. +func swap(root *os.Root, staged, ns, previous string) error { + hadOld := false + if _, err := root.Lstat(ns); err == nil { + if err := root.Rename(ns, previous); err != nil { + return err + } + hadOld = true + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + if err := root.Rename(staged, ns); err != nil { + if hadOld { + _ = root.Rename(previous, ns) + } + return err + } + return nil +} + +// localPath turns a slash-separated path from the mirror protocol into a +// path under the staging directory. It refuses anything that is not a +// plain relative path: absolute paths, "..", empty elements and +// backslashes (which are separators on some systems). os.Root enforces the +// same limit again when the file is written. +func localPath(staged, rel string) (string, error) { + if rel == "" || strings.ContainsAny(rel, "\\\x00") || path.Clean(rel) != rel || !filepath.IsLocal(filepath.FromSlash(rel)) { + return "", fmt.Errorf("mirror path %q is not a plain relative path", rel) + } + return path.Join(staged, rel), nil +} + +// writeConfined writes b to rel under staged through root. +func writeConfined(root *os.Root, staged, rel string, b []byte) error { + p, err := localPath(staged, rel) + if err != nil { + return err + } + if err := root.MkdirAll(path.Dir(p), 0o755); err != nil { + return err + } + return root.WriteFile(p, b, 0o644) +} + +// download copies every file of the namespace from the mirror into staged. +// It reads the unsigned payloads only to learn which files exist, and it +// validates each value that becomes part of a path before using it. +func download(ctx context.Context, root *os.Root, staged, base, ns string) error { base = strings.TrimRight(base, "/") + "/kb/v" + protocol.Version + "/" + ns + "/" - dest := filepath.Join(dir, ns) hc := &http.Client{} get := func(rel string) ([]byte, error) { + if _, err := localPath(staged, rel); err != nil { + return nil, err + } req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+rel, nil) if err != nil { return nil, err @@ -43,11 +162,7 @@ func Fetch(ctx context.Context, base, ns, dir string) error { if err != nil { return nil, err } - path := filepath.Join(dest, filepath.FromSlash(rel)) - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - return nil, err - } - return b, os.WriteFile(path, b, 0o644) + return b, writeConfined(root, staged, rel, b) } payload := func(b []byte, v any) error { env, err := signing.ParseEnvelope(b) @@ -84,10 +199,15 @@ func Fetch(ctx context.Context, base, ns, dir string) error { if err := payload(cb, &cat); err != nil { return err } + // Catalog.Validate checks every pack id and version against the + // protocol patterns and that each path is exactly PackPath(id, version). + if err := cat.Validate(); err != nil { + return fmt.Errorf("mirror catalog: %v", err) + } + if cat.Namespace != ns { + return fmt.Errorf("mirror catalog is for namespace %q", cat.Namespace) + } for _, cp := range cat.Packs { - if !protocol.ValidID(cp.ID) || cp.Path != protocol.PackPath(cp.ID, cp.Version) { - return fmt.Errorf("catalog pack %q has an unexpected path", cp.ID) - } pb, err := get(cp.Path) if err != nil { return err diff --git a/internal/server/fetch_test.go b/internal/server/fetch_test.go new file mode 100644 index 0000000..faaa913 --- /dev/null +++ b/internal/server/fetch_test.go @@ -0,0 +1,239 @@ +package server_test + +import ( + "context" + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/openbasalt/knowledge/internal/build" + "github.com/openbasalt/knowledge/internal/server" + "github.com/openbasalt/knowledge/internal/testkit" + "github.com/openbasalt/knowledge/protocol" + "github.com/openbasalt/knowledge/signing" +) + +// mirror serves the kit's build output, with some files replaced. +func mirror(t *testing.T, kit *testkit.Kit, override map[string][]byte) *httptest.Server { + t.Helper() + files := http.StripPrefix("/kb/v"+protocol.Version, http.FileServer(http.Dir(kit.Out))) + prefix := "/kb/v" + protocol.Version + "/" + kit.NS + "/" + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if b, ok := override[strings.TrimPrefix(r.URL.Path, prefix)]; ok { + _, _ = w.Write(b) + return + } + // Any other request outside the published tree is answered too, + // as a hostile mirror would. + if !strings.HasPrefix(r.URL.Path, prefix) { + _, _ = w.Write([]byte(`{"pwned":true}`)) + return + } + files.ServeHTTP(w, r) + })) + t.Cleanup(ts.Close) + return ts +} + +func readCatalog(t *testing.T, kit *testkit.Kit) protocol.Catalog { + t.Helper() + b, err := os.ReadFile(filepath.Join(kit.Dir, "catalog.json")) + if err != nil { + t.Fatal(err) + } + env, err := signing.ParseEnvelope(b) + if err != nil { + t.Fatal(err) + } + raw, err := base64.StdEncoding.DecodeString(env.Payload) + if err != nil { + t.Fatal(err) + } + var cat protocol.Catalog + if err := json.Unmarshal(raw, &cat); err != nil { + t.Fatal(err) + } + return cat +} + +func seal(t *testing.T, typ string, v any, s *signing.Signer) []byte { + t.Helper() + env, err := signing.SealJSON(typ, v, s) + if err != nil { + t.Fatal(err) + } + b, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } + return b +} + +// listTree returns every path under dir (relative), for "nothing left +// behind" checks. +func listTree(t *testing.T, dir string) []string { + t.Helper() + var out []string + err := filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if p != dir { + rel, _ := filepath.Rel(dir, p) + out = append(out, rel) + } + return nil + }) + if err != nil { + t.Fatal(err) + } + return out +} + +// A catalog version with dot segments (signed even by the real publisher) +// is refused before any request for it is made, and nothing is written +// outside the data directory, or inside it. +func TestFetchRefusesTraversalInVersion(t *testing.T) { + kit := testkit.New(t, "../../content/basalt", time.Now()) + top := t.TempDir() + data := filepath.Join(top, "data") + cat := readCatalog(t, kit) + // From //packs/-1, four levels up is top: the old code + // wrote top/escaped.json. + evil := "1/../../../../escaped" + cat.Packs[0].Version = evil + cat.Packs[0].Path = protocol.PackPath(cat.Packs[0].ID, evil) + ts := mirror(t, kit, map[string][]byte{ + "catalog.json": seal(t, protocol.TypeCatalog, cat, kit.Publisher), + cat.Packs[0].Path: []byte(`{"pwned":true}`), + }) + err := server.Fetch(context.Background(), ts.URL, kit.NS, data, kit.Trust(t)) + if err == nil { + t.Fatal("traversal in the pack version was accepted") + } + if got := listTree(t, top); len(got) != 1 || got[0] != "data" { + t.Fatalf("files left behind: %v", got) + } +} + +// Paths that are absolute, climb out, or are not clean are refused by +// the confinement check itself. +func TestFetchLocalPath(t *testing.T) { + for _, rel := range []string{"/etc/x.json", "../x.json", "packs/../../x.json", "packs//x.json", + "./x.json", "", `packs\..\x.json`, "x\x00.json"} { + if err := server.CheckMirrorPath(rel); err == nil { + t.Errorf("%q accepted", rel) + } + } + for _, rel := range []string{"catalog.json", "keyring/1.json", "packs/a-b-1.2.json", "entries/x-y.json"} { + if err := server.CheckMirrorPath(rel); err != nil { + t.Errorf("%q refused: %v", rel, err) + } + } +} + +// A symbolic link at the namespace path that points outside the data +// directory is replaced, not written through. +func TestFetchSymlinkInData(t *testing.T) { + kit := testkit.New(t, "../../content/basalt", time.Now()) + top := t.TempDir() + data := filepath.Join(top, "data") + outside := filepath.Join(top, "outside") + for _, d := range []string{data, outside} { + if err := os.Mkdir(d, 0o755); err != nil { + t.Fatal(err) + } + } + if err := os.Symlink(outside, filepath.Join(data, kit.NS)); err != nil { + t.Fatal(err) + } + ts := mirror(t, kit, nil) + if err := server.Fetch(context.Background(), ts.URL, kit.NS, data, kit.Trust(t)); err != nil { + t.Fatal(err) + } + if got := listTree(t, outside); len(got) != 0 { + t.Fatalf("written through the link: %v", got) + } + fi, err := os.Lstat(filepath.Join(data, kit.NS)) + if err != nil || !fi.IsDir() { + t.Fatalf("namespace is not a plain directory: %v %v", fi, err) + } + if _, err := build.LoadNamespace(filepath.Join(data, kit.NS), kit.Trust(t)); err != nil { + t.Fatal(err) + } + for _, p := range listTree(t, data) { + if strings.HasPrefix(p, ".fetch-") { + t.Fatalf("staging left behind: %s", p) + } + } +} + +// A pack that the trusted publisher did not sign fails verification; the +// staged files are removed and a copy fetched before stays untouched. +func TestFetchUnsignedPack(t *testing.T) { + kit := testkit.New(t, "../../content/basalt", time.Now()) + data := t.TempDir() + good := mirror(t, kit, nil) + if err := server.Fetch(context.Background(), good.URL, kit.NS, data, kit.Trust(t)); err != nil { + t.Fatal(err) + } + before := listTree(t, data) + + // Re-sign the first pack with a key nobody trusts, then give the + // catalog the matching digest and size under the real publisher, so + // only the pack signature is wrong. + cat := readCatalog(t, kit) + cp := cat.Packs[0] + pb, err := os.ReadFile(filepath.Join(kit.Dir, filepath.FromSlash(cp.Path))) + if err != nil { + t.Fatal(err) + } + env, err := signing.ParseEnvelope(pb) + if err != nil { + t.Fatal(err) + } + raw, err := base64.StdEncoding.DecodeString(env.Payload) + if err != nil { + t.Fatal(err) + } + var p protocol.Pack + if err := json.Unmarshal(raw, &p); err != nil { + t.Fatal(err) + } + stranger, err := signing.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + bad := seal(t, protocol.TypePack, p, stranger) + cat.Packs[0].Digest = signing.DigestBytes(bad) + cat.Packs[0].Size = int64(len(bad)) + evil := mirror(t, kit, map[string][]byte{ + "catalog.json": seal(t, protocol.TypeCatalog, cat, kit.Publisher), + cp.Path: bad, + }) + if err := server.Fetch(context.Background(), evil.URL, kit.NS, data, kit.Trust(t)); err == nil { + t.Fatal("a pack with an untrusted signature was accepted") + } + after := listTree(t, data) + if strings.Join(before, "\n") != strings.Join(after, "\n") { + t.Fatalf("data changed after a failed fetch:\nbefore %v\nafter %v", before, after) + } + if _, err := build.LoadNamespace(filepath.Join(data, kit.NS), kit.Trust(t)); err != nil { + t.Fatalf("earlier copy no longer verifies: %v", err) + } + + // Into an empty data directory, nothing at all is left. + empty := t.TempDir() + if err := server.Fetch(context.Background(), evil.URL, kit.NS, empty, kit.Trust(t)); err == nil { + t.Fatal("a pack with an untrusted signature was accepted") + } + if got := listTree(t, empty); len(got) != 0 { + t.Fatalf("files left behind: %v", got) + } +} diff --git a/internal/server/server_test.go b/internal/server/server_test.go index fc917c6..a2eab3a 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -154,7 +154,7 @@ func TestFetch(t *testing.T) { mirror := httptest.NewServer(http.StripPrefix("/kb/v0", http.FileServer(http.Dir(kit.Out)))) defer mirror.Close() dir := t.TempDir() - if err := server.Fetch(context.Background(), mirror.URL, "basalt", dir); err != nil { + if err := server.Fetch(context.Background(), mirror.URL, "basalt", dir, kit.Trust(t)); err != nil { t.Fatal(err) } n, err := build.LoadNamespace(dir+"/basalt", kit.Trust(t)) diff --git a/internal/source/nofollow.go b/internal/source/nofollow.go new file mode 100644 index 0000000..d26c8e8 --- /dev/null +++ b/internal/source/nofollow.go @@ -0,0 +1,74 @@ +package source + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// ErrSymlink is returned for content paths that are, or pass through, a +// symbolic link. Content comes from pull requests, including forks whose +// review runs next to model API keys, so a link could point at any file of +// the machine that reads it. Links are never followed. +var ErrSymlink = errors.New("symbolic links are not allowed in content") + +// CheckNoSymlinks verifies that every component of rel under base exists +// and is not a symbolic link. base itself is trusted (it is chosen by the +// caller, not by the content). +func CheckNoSymlinks(base, rel string) (os.FileInfo, error) { + rel = filepath.Clean(rel) + if rel == "." { + return os.Lstat(base) + } + if !filepath.IsLocal(rel) { + return nil, fmt.Errorf("%s: path leaves %s", rel, base) + } + cur := base + parts := strings.Split(filepath.ToSlash(rel), "/") + var fi os.FileInfo + for i, p := range parts { + cur = filepath.Join(cur, p) + var err error + if fi, err = os.Lstat(cur); err != nil { + return nil, err + } + if fi.Mode()&os.ModeSymlink != 0 { + return nil, fmt.Errorf("%s: %w", cur, ErrSymlink) + } + if i < len(parts)-1 && !fi.IsDir() { + return nil, fmt.Errorf("%s: not a directory", cur) + } + } + return fi, nil +} + +// ReadNoFollow reads the regular file rel under base, refusing symbolic +// links in any component of rel and anything that is not a regular file +// (devices, pipes). The opened file must be the one that was checked, so a +// swap between the check and the open is refused too. +func ReadNoFollow(base, rel string) ([]byte, error) { + fi, err := CheckNoSymlinks(base, rel) + if err != nil { + return nil, err + } + p := filepath.Join(base, rel) + if !fi.Mode().IsRegular() { + return nil, fmt.Errorf("%s: not a regular file", p) + } + f, err := os.Open(p) + if err != nil { + return nil, err + } + defer f.Close() + got, err := f.Stat() + if err != nil { + return nil, err + } + if !os.SameFile(fi, got) { + return nil, fmt.Errorf("%s: changed while it was read", p) + } + return io.ReadAll(f) +} diff --git a/internal/source/nofollow_test.go b/internal/source/nofollow_test.go new file mode 100644 index 0000000..b9bef40 --- /dev/null +++ b/internal/source/nofollow_test.go @@ -0,0 +1,102 @@ +package source + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +// seedCopy copies the seed namespace to a temporary directory and +// returns the namespace dir and a directory outside it. +func seedCopy(t *testing.T) (string, string) { + t.Helper() + top := t.TempDir() + ns := filepath.Join(top, "content", "basalt") + if err := os.CopyFS(ns, os.DirFS("../../content/basalt")); err != nil { + t.Fatal(err) + } + outside := filepath.Join(top, "outside") + if err := os.Mkdir(outside, 0o755); err != nil { + t.Fatal(err) + } + return ns, outside +} + +// replaceWithLink moves path to outside and leaves a symbolic link to it, +// so the target is valid content: only the link is wrong. +func replaceWithLink(t *testing.T, path, outside string) { + t.Helper() + target := filepath.Join(outside, filepath.Base(path)) + if err := os.Rename(path, target); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, path); err != nil { + t.Fatal(err) + } +} + +// Load refuses a symbolic link anywhere in the namespace, even when the +// file it points to is valid content. +func TestLoadRefusesSymlinks(t *testing.T) { + for _, rel := range []string{ + "entries/nvidia-legacy-580/pt-BR.md", + "entries/nvidia-legacy-580/en.md", + "namespace.yaml", + "entries", + ".", + } { + t.Run(rel, func(t *testing.T) { + ns, outside := seedCopy(t) + // With and without a trailing slash, as kb validate gets it. + for _, d := range []string{ns, ns + "/"} { + if _, err := Load(d); err != nil { + t.Fatalf("copy does not load: %v", err) + } + } + replaceWithLink(t, filepath.Join(ns, rel), outside) + _, err := Load(ns) + if err == nil { + t.Fatal("symbolic link followed") + } + if !errors.Is(err, ErrSymlink) && !containsSymlinkProblem(err) { + t.Fatalf("unexpected error: %v", err) + } + }) + } +} + +func containsSymlinkProblem(err error) bool { + var probs Problems + if !errors.As(err, &probs) { + return false + } + for _, p := range probs { + if errors.Is(p.Err, ErrSymlink) { + return true + } + } + return false +} + +func TestReadNoFollow(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "a.md"), []byte("ok"), 0o644); err != nil { + t.Fatal(err) + } + if b, err := ReadNoFollow(dir, "a.md"); err != nil || string(b) != "ok" { + t.Fatalf("%q %v", b, err) + } + if err := os.Symlink(filepath.Join(dir, "a.md"), filepath.Join(dir, "b.md")); err != nil { + t.Fatal(err) + } + if _, err := ReadNoFollow(dir, "b.md"); !errors.Is(err, ErrSymlink) { + t.Fatalf("link: %v", err) + } + if _, err := ReadNoFollow(dir, "../x.md"); err == nil { + t.Fatal("path outside base accepted") + } + if _, err := ReadNoFollow(dir, "."); err == nil { + t.Fatal("directory read as a file") + } +} diff --git a/internal/source/source.go b/internal/source/source.go index 02f5e20..56b4e12 100644 --- a/internal/source/source.go +++ b/internal/source/source.go @@ -125,11 +125,16 @@ func SplitFrontMatter(b []byte) ([]byte, string, error) { } // Load reads and validates the namespace in dir. All problems are -// returned together. +// returned together. Symbolic links inside dir (and dir itself being one) +// are refused, never followed: see ErrSymlink. func Load(dir string) (*Source, error) { var probs Problems + dir = filepath.Clean(dir) + if _, err := CheckNoSymlinks(filepath.Dir(dir), filepath.Base(dir)); err != nil { + return nil, err + } nsFile := filepath.Join(dir, "namespace.yaml") - b, err := os.ReadFile(nsFile) + b, err := ReadNoFollow(dir, "namespace.yaml") if err != nil { return nil, err } @@ -140,6 +145,9 @@ func Load(dir string) (*Source, error) { probs = append(probs, checkNamespace(nsFile, &ns)...) entriesDir := filepath.Join(dir, "entries") + if _, err := CheckNoSymlinks(dir, "entries"); err != nil { + return nil, err + } des, err := os.ReadDir(entriesDir) if err != nil { return nil, err @@ -208,10 +216,13 @@ func checkNamespace(file string, ns *Namespace) Problems { func loadEntry(dir string, ns *Namespace) (*protocol.Entry, Problems) { var probs Problems enFile := filepath.Join(dir, "en.md") - b, err := os.ReadFile(enFile) - if err != nil { + b, err := ReadNoFollow(dir, "en.md") + if errors.Is(err, os.ErrNotExist) { return nil, Problems{{dir, errors.New("en.md is required")}} } + if err != nil { + return nil, Problems{{enFile, err}} + } fm, body, err := SplitFrontMatter(b) if err != nil { return nil, Problems{{enFile, err}} @@ -257,7 +268,7 @@ func loadEntry(dir string, ns *Namespace) (*protocol.Entry, Problems) { probs = append(probs, Problem{file, fmt.Errorf("language %q is not listed in namespace.yaml", lang)}) continue } - b, err := os.ReadFile(file) + b, err := ReadNoFollow(dir, filepath.Base(file)) if err != nil { probs = append(probs, Problem{file, err}) continue diff --git a/signing/keyring.go b/signing/keyring.go index 5145906..54c2b11 100644 --- a/signing/keyring.go +++ b/signing/keyring.go @@ -277,6 +277,19 @@ func (t *Trust) Update(env *Envelope) (*Keyring, error) { return next, nil } +// Clone returns an independent trust store with the same keyrings and +// clock, so a caller can verify untrusted data (and follow the rotations +// it carries) without changing t until the data is accepted. +func (t *Trust) Clone() *Trust { + t.mu.RLock() + defer t.mu.RUnlock() + c := &Trust{rings: make(map[string]*Keyring, len(t.rings)), Now: t.Now} + for ns, k := range t.rings { + c.rings[ns] = k + } + return c +} + // Keyring returns the current keyring of a namespace. func (t *Trust) Keyring(ns string) (*Keyring, bool) { t.mu.RLock()