Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ on:
push:
branches: [main]
pull_request:
# Weekly, so a newly published Go or module advisory shows up without a push.
schedule:
- cron: "23 6 * * 1"

permissions:
contents: read
Expand All @@ -20,6 +23,8 @@ jobs:
go-version-file: go.mod
- name: Format and vet
run: make lint
- name: Known vulnerabilities (Go toolchain and modules)
run: go run golang.org/x/vuln/cmd/govulncheck@v1.7.0 ./...
- name: Unit tests (race detector)
run: go test -race ./...
- name: Validate content
Expand Down
5 changes: 4 additions & 1 deletion .github/workflows/content-review-fork.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,10 @@ jobs:
git -C pr fetch -q --no-tags "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" "$BASE_SHA" "+refs/pull/$PR/head:pr-head"
# The fetched head must be the commit the maintainer labeled.
test "$(git -C pr rev-parse pr-head)" = "$HEAD_SHA"
git -C pr -c core.hooksPath=/dev/null checkout -q pr-head
# Symbolic links are checked out as plain files holding the
# link text, so nothing in the pull request can point at a runner
# file (kb-review also refuses links on its own).
git -C pr -c core.hooksPath=/dev/null -c core.symlinks=false checkout -q pr-head
git -C pr diff --name-only "$BASE_SHA...$HEAD_SHA" > "$RUNNER_TEMP/changed.txt"
- name: Review
working-directory: pr
Expand Down
3 changes: 3 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,9 @@ data (it builds the tool from the base branch and never builds or runs
anything from the fork) and removes the label. New commits need the label
again.

Symbolic links are not allowed anywhere under `content/`: `kb validate`,
`kb build` and `kb-review` refuse them instead of following them.

## For maintainers: setup

Repository secrets (Settings, Secrets and variables, Actions):
Expand Down
4 changes: 3 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
# container with a read-only root file system; kbd writes nothing except,
# when KBD_FETCH_URL is set, the fetched bundle into KBD_DATA (mount a
# small writable volume or tmpfs there). See docs/hosting.md.
FROM golang:1.27 AS build
# Exact patch release, kept equal to the go line in go.mod: update both
# together. A floating tag would make the build depend on its date.
FROM golang:1.27.4 AS build
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
Expand Down
5 changes: 5 additions & 0 deletions cmd/kb-review/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,11 @@ func run(args []string) (int, error) {
}
// Structural validation first: a model never sees content that fails it.
contentDir := filepath.Join(*repo, *content)
// The content tree comes from the pull request: a symbolic link there
// could point at any file of the runner, so none is followed.
if _, err := source.CheckNoSymlinks(*repo, *content); err != nil {
return 1, err
}
nsDirs, _ := filepath.Glob(filepath.Join(contentDir, "*", "namespace.yaml"))
var actions []string
var sources []*source.Source
Expand Down
2 changes: 1 addition & 1 deletion cmd/kbd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ func run(log *slog.Logger) error {
}
for _, n := range names {
fctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
err := server.Fetch(fctx, url, n, data)
err := server.Fetch(fctx, url, n, data, trust)
cancel()
if err != nil {
return fmt.Errorf("fetch %s: %v", n, err)
Expand Down
8 changes: 8 additions & 0 deletions docs/hosting.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,14 @@ Configuration by environment (see `cmd/kbd/main.go` for the full list):
| `KBD_HOSTING_QUERY_BODY_LOGGED` | `false` |
| `KBD_HOSTING_PROVIDER` | `Quave ONE` (optional) |

The mirror is not trusted. kbd downloads into a staging directory inside
`KBD_DATA` (named `.fetch-` and a random suffix), never writes outside
`KBD_DATA` (paths from the mirror are checked, and writes go through a
root confined to it), verifies the staged copy against the pinned
keyrings, and only then moves it to `KBD_DATA/<namespace>`. A fetch that
fails verification leaves nothing behind and keeps the copy that was
there; kbd then stops with the error.

Instead of fetching, the build can be mounted read-only at `KBD_DATA`.
A new content release is a restart (or a rolling update) with the new
build; a new online key or delegation is a secret and variable change.
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
module github.com/openbasalt/knowledge

go 1.27.0
go 1.27.4

require (
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3
go.yaml.in/yaml/v3 v3.0.5
go.yaml.in/yaml/v3 v3.0.6
)

require golang.org/x/text v0.42.0 // indirect
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxK
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM=
go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
6 changes: 4 additions & 2 deletions internal/review/review.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,12 @@ import (
"errors"
"fmt"
"io"
"os"
"path/filepath"
"slices"
"sort"
"strings"

"github.com/openbasalt/knowledge/internal/source"
)

// Rule ids the verdict may use. They match the sections of the rules file.
Expand Down Expand Up @@ -231,7 +232,8 @@ func Units(repoRoot, contentRoot string, changed []string) ([]Unit, error) {
var out []Unit
for _, u := range byID {
for i := range u.Files {
b, err := os.ReadFile(filepath.Join(repoRoot, u.Files[i].Path))
// Content comes from the pull request: never follow a link.
b, err := source.ReadNoFollow(repoRoot, filepath.FromSlash(u.Files[i].Path))
if err != nil {
return nil, err
}
Expand Down
33 changes: 33 additions & 0 deletions internal/review/review_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"

Expand Down Expand Up @@ -161,3 +163,34 @@ func TestBatches(t *testing.T) {
t.Fatalf("%v", b)
}
}

// A fork pull request can add a symbolic link where an entry file should
// be. Units must refuse it instead of sending the target to a model.
func TestUnitsRefuseSymlinks(t *testing.T) {
repo := t.TempDir()
dir := filepath.Join(repo, "content", "basalt", "entries", "x-entry")
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
secret := filepath.Join(t.TempDir(), "secret")
if err := os.WriteFile(secret, []byte("---\nid: x\n---\nRUNNER SECRET"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "en.md"), []byte("---\nid: x-entry\n---\nok"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.Symlink(secret, filepath.Join(dir, "pt-BR.md")); err != nil {
t.Fatal(err)
}
units, err := Units(repo, "content", []string{"content/basalt/entries/x-entry/pt-BR.md"})
if !errors.Is(err, source.ErrSymlink) {
t.Fatalf("symbolic link not refused: %v", err)
}
for _, u := range units {
for _, f := range u.Files {
if strings.Contains(f.Content, "RUNNER SECRET") {
t.Fatal("link target sent for review")
}
}
}
}
7 changes: 7 additions & 0 deletions internal/server/export_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package server

// CheckMirrorPath exposes the mirror path check to the external tests.
func CheckMirrorPath(rel string) error {
_, err := localPath(".fetch-test/ns", rel)
return err
}
146 changes: 133 additions & 13 deletions internal/server/fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,150 @@ package server

import (
"context"
"crypto/rand"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"path"
"path/filepath"
"strings"

"github.com/openbasalt/knowledge/internal/build"
"github.com/openbasalt/knowledge/protocol"
"github.com/openbasalt/knowledge/signing"
)

// stagingPrefix names the temporary directories Fetch creates inside the
// data directory. kbd never loads them: namespace names cannot start with
// a dot.
const stagingPrefix = ".fetch-"

// Fetch copies one namespace of a published build (a static mirror or
// another server) into dir/<ns>. It checks nothing itself: the files are
// verified by build.LoadNamespace before anything is served, and a file
// that fails there is never used.
func Fetch(ctx context.Context, base, ns, dir string) error {
// another server) into dir/<ns>.
//
// The mirror is not trusted. Every catalog field that becomes part of a
// file name is checked against the protocol patterns before use, and every
// write goes through an os.Root opened on dir, so nothing can land outside
// it (no "..", no absolute path, no symbolic link that leaves dir). The
// files are written to a staging directory first and verified there with
// build.LoadNamespace against a copy of trust; only a namespace that
// verifies replaces dir/<ns>. When anything fails, the staging directory is
// removed and dir/<ns> is left as it was.
func Fetch(ctx context.Context, base, ns, dir string, trust *signing.Trust) error {
if !signing.ValidNamespace(ns) {
return fmt.Errorf("namespace %q is not valid", ns)
}
if trust == nil {
return errors.New("fetch needs a trust store to verify the namespace")
}
if err := os.MkdirAll(dir, 0o755); err != nil {
return err
}
root, err := os.OpenRoot(dir)
if err != nil {
return err
}
defer root.Close()
stage, err := stagingName()
if err != nil {
return err
}
if err := root.Mkdir(stage, 0o700); err != nil {
return err
}
// Always remove the staging directory: after a successful swap it only
// holds the previous copy of the namespace.
defer root.RemoveAll(stage)
staged := path.Join(stage, ns)
if err := download(ctx, root, staged, base, ns); err != nil {
return err
}
// LoadNamespace takes the namespace from the directory name, so the
// staged copy lives at <stage>/<ns>. FollowKeyrings advances the trust
// copy through rotations the mirror carries, each one verified by the
// keyring before it, exactly as kbd does at start.
check := trust.Clone()
stagedPath := filepath.Join(dir, filepath.FromSlash(staged))
if err := build.FollowKeyrings(stagedPath, check); err != nil {
return fmt.Errorf("fetched namespace does not verify: %v", err)
}
if _, err := build.LoadNamespace(stagedPath, check); err != nil {
return fmt.Errorf("fetched namespace does not verify: %v", err)
}
return swap(root, staged, ns, path.Join(stage, "previous"))
}

// stagingName returns a fresh, unpredictable staging directory name.
func stagingName() (string, error) {
b := make([]byte, 12)
if _, err := rand.Read(b); err != nil {
return "", err
}
return stagingPrefix + hex.EncodeToString(b), nil
}

// swap moves the verified staged namespace to ns. A current ns (a
// directory, or anything else, such as a symbolic link) is first renamed
// to previous, so a link is replaced, never followed; it is restored when
// the second rename fails.
func swap(root *os.Root, staged, ns, previous string) error {
hadOld := false
if _, err := root.Lstat(ns); err == nil {
if err := root.Rename(ns, previous); err != nil {
return err
}
hadOld = true
} else if !errors.Is(err, os.ErrNotExist) {
return err
}
if err := root.Rename(staged, ns); err != nil {
if hadOld {
_ = root.Rename(previous, ns)
}
return err
}
return nil
}

// localPath turns a slash-separated path from the mirror protocol into a
// path under the staging directory. It refuses anything that is not a
// plain relative path: absolute paths, "..", empty elements and
// backslashes (which are separators on some systems). os.Root enforces the
// same limit again when the file is written.
func localPath(staged, rel string) (string, error) {
if rel == "" || strings.ContainsAny(rel, "\\\x00") || path.Clean(rel) != rel || !filepath.IsLocal(filepath.FromSlash(rel)) {
return "", fmt.Errorf("mirror path %q is not a plain relative path", rel)
}
return path.Join(staged, rel), nil
}

// writeConfined writes b to rel under staged through root.
func writeConfined(root *os.Root, staged, rel string, b []byte) error {
p, err := localPath(staged, rel)
if err != nil {
return err
}
if err := root.MkdirAll(path.Dir(p), 0o755); err != nil {
return err
}
return root.WriteFile(p, b, 0o644)
}

// download copies every file of the namespace from the mirror into staged.
// It reads the unsigned payloads only to learn which files exist, and it
// validates each value that becomes part of a path before using it.
func download(ctx context.Context, root *os.Root, staged, base, ns string) error {
base = strings.TrimRight(base, "/") + "/kb/v" + protocol.Version + "/" + ns + "/"
dest := filepath.Join(dir, ns)
hc := &http.Client{}
get := func(rel string) ([]byte, error) {
if _, err := localPath(staged, rel); err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+rel, nil)
if err != nil {
return nil, err
Expand All @@ -43,11 +162,7 @@ func Fetch(ctx context.Context, base, ns, dir string) error {
if err != nil {
return nil, err
}
path := filepath.Join(dest, filepath.FromSlash(rel))
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return nil, err
}
return b, os.WriteFile(path, b, 0o644)
return b, writeConfined(root, staged, rel, b)
}
payload := func(b []byte, v any) error {
env, err := signing.ParseEnvelope(b)
Expand Down Expand Up @@ -84,10 +199,15 @@ func Fetch(ctx context.Context, base, ns, dir string) error {
if err := payload(cb, &cat); err != nil {
return err
}
// Catalog.Validate checks every pack id and version against the
// protocol patterns and that each path is exactly PackPath(id, version).
if err := cat.Validate(); err != nil {
return fmt.Errorf("mirror catalog: %v", err)
}
if cat.Namespace != ns {
return fmt.Errorf("mirror catalog is for namespace %q", cat.Namespace)
}
for _, cp := range cat.Packs {
if !protocol.ValidID(cp.ID) || cp.Path != protocol.PackPath(cp.ID, cp.Version) {
return fmt.Errorf("catalog pack %q has an unexpected path", cp.ID)
}
pb, err := get(cp.Path)
if err != nil {
return err
Expand Down
Loading
Loading