From d21561fe320297d31c38a3a184b633e209c0fdec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20=C3=85slund?= Date: Thu, 22 Oct 2026 12:50:59 +0000 Subject: [PATCH 1/3] feat(bootstrap): registry mirror (Harbor pull-through cache) for new robots TemplateData gets RegistryMirror, RegistryUsername and RegistryPassword. The new partial registries.sh.tmpl, included in server.sh.tmpl and agent.sh.tmpl before the k3s installer, writes /etc/rancher/k3s/registries.yaml (0600, marker '# managed by tnctl bootstrap') with the six mirrors and the auth. It does nothing without a mirror, keeps a foreign file with a warning, and removes a managed one for 'none'. The credential is validated for the single-quoted shell context and never printed. tnctl bootstrap: --registry-mirror HOST|none and --registry-credential-file FILE. The default comes from an optional registry field of the device credentials response (phase 1b, ignored when absent). Golden files regenerated; docs/commands/bootstrap.md. Design: plans/harbor-proxy-staging.md Co-Authored-By: Claude Sonnet 5.5 --- cmd/bootstrap.go | 28 ++ cmd/bootstrap_registry.go | 116 +++++ cmd/bootstrap_registry_test.go | 185 ++++++++ docs/commands/bootstrap.md | 38 ++ pkg/api/fleet_devices.go | 12 + pkg/api/fleet_devices_test.go | 23 + pkg/templates/agent.sh.tmpl | 2 +- pkg/templates/bootstrap_templates.go | 26 ++ pkg/templates/golden_data_test.go | 4 + pkg/templates/partials/registries.sh.tmpl | 50 +++ pkg/templates/registries_test.go | 403 ++++++++++++++++++ pkg/templates/render.go | 34 ++ pkg/templates/server.sh.tmpl | 2 +- pkg/templates/testdata/agent.golden.sh | 42 ++ pkg/templates/testdata/agent.roles.golden.sh | 42 ++ pkg/templates/testdata/server.golden.sh | 42 ++ pkg/templates/testdata/server.roles.golden.sh | 42 ++ 17 files changed, 1089 insertions(+), 2 deletions(-) create mode 100644 cmd/bootstrap_registry.go create mode 100644 cmd/bootstrap_registry_test.go create mode 100644 pkg/templates/partials/registries.sh.tmpl create mode 100644 pkg/templates/registries_test.go diff --git a/cmd/bootstrap.go b/cmd/bootstrap.go index 298ce94..2e9d9ba 100644 --- a/cmd/bootstrap.go +++ b/cmd/bootstrap.go @@ -23,6 +23,12 @@ var bootstrapCmd = &cobra.Command{ if err := templates.ValidateK3SVersion(k3sVersion); err != nil { return err } + registryMirror, _ := cmd.Flags().GetString("registry-mirror") + registryCredFile, _ := cmd.Flags().GetString("registry-credential-file") + // Fail on a bad mirror or an unusable credential file before the platform is asked for anything. + if err := checkRegistryFlags(registryMirror, registryCredFile); err != nil { + return err + } nodeOpts, err := nodeOptionsFromFlags(cmd) if err != nil { return err @@ -66,6 +72,11 @@ var bootstrapCmd = &cobra.Command{ return fmt.Errorf("--force only applies together with --k3s-upgrade") } + registry, err := registryFromFlags(registryMirror, registryCredFile, bl.registry) + if err != nil { + return err + } + data := templates.TemplateData{ Domain: domain, Namespace: namespace, @@ -81,6 +92,9 @@ var bootstrapCmd = &cobra.Command{ K3SVersionExplicit: cmd.Flags().Changed("k3s-version"), K3SUpgrade: k3sUpgrade, K3SForce: force, + RegistryMirror: registry.Mirror, + RegistryUsername: registry.Username, + RegistryPassword: registry.Password, } if err := templates.ValidateTemplateData(data); err != nil { @@ -117,6 +131,13 @@ var bootstrapCmd = &cobra.Command{ fmt.Printf("- %s\n", serverFile) fmt.Printf("- %s\n", dockerServerFile) fmt.Print(nodes.describe(agentFile, serverFile)) + switch registry.Mirror { + case "": + case templates.RegistryMirrorNone: + fmt.Printf("%s and %s remove a registries.yaml that an earlier script wrote (registry mirror none)\n", agentFile, serverFile) + default: + fmt.Printf("%s and %s configure the registry mirror %s (/etc/rancher/k3s/registries.yaml); they contain its credential, keep them private\n", agentFile, serverFile, registry.Mirror) + } skipUpload, _ := cmd.Flags().GetBool("skip-upload") if !skipUpload { @@ -130,6 +151,8 @@ var bootstrapCmd = &cobra.Command{ // bootstrapLogin is the decoded login of a device that the bootstrap scripts install. type bootstrapLogin struct { rsaPrivate, rsaPublic, username, password, publicSSHKey string + // registry is the registry mirror the platform offers the device, nil when it sends none. + registry *api.FleetRegistry } // loadBootstrapLogin asks the platform for the login of a device and decodes it. The platform gives @@ -170,6 +193,7 @@ func loadBootstrapLogin(ctx context.Context, fl *api.Fleet, namespace, deviceNam if bl.publicSSHKey, err = decode("publicsshkey", c.PublicSSHKey); err != nil { return nil, err } + bl.registry = c.Registry return &bl, nil } @@ -191,5 +215,9 @@ func init() { "with --k3s-upgrade: allow an upgrade over more than one minor version (TN_K3S_FORCE=true is the default in the scripts)") addNodeFlags(bootstrapCmd) bootstrapCmd.Flags().String("docker-secret", "", "The contents of the .dockerconfigjson") + bootstrapCmd.Flags().String("registry-mirror", "", + "pull images through this Harbor host (pull-through cache projects dockerhub, ghcr, gcr, quay, k8s, nvcr), with the upstream registry as the fallback: the scripts write /etc/rancher/k3s/registries.yaml before the k3s installer. \"none\" removes a file the scripts wrote. Without it the file is left alone (or follows the registry that the platform sends with the device credentials, when it sends one)") + bootstrapCmd.Flags().String("registry-credential-file", "", + "file with the Harbor credential for --registry-mirror, one line username:password (a robot account of the teamspace), readable by you only. The password goes into the generated scripts and is never printed") rootCmd.AddCommand(bootstrapCmd) } diff --git a/cmd/bootstrap_registry.go b/cmd/bootstrap_registry.go new file mode 100644 index 0000000..803ae20 --- /dev/null +++ b/cmd/bootstrap_registry.go @@ -0,0 +1,116 @@ +package cmd + +import ( + "fmt" + "os" + "runtime" + "strings" + + "github.com/teknoir/cli/pkg/api" + "github.com/teknoir/cli/pkg/templates" +) + +// registrySettings is what `tnctl bootstrap` puts into TemplateData for the registry mirror. +type registrySettings struct { + Mirror, Username, Password string +} + +// registryFromFlags decides the registry mirror of the generated scripts (plans/harbor-proxy-staging.md): +// +// - --registry-mirror none removes a registries.yaml that an earlier script wrote; no credential; +// - --registry-mirror HOST uses HOST with the credential of --registry-credential-file +// (one line username:password), or the platform's credential when the device's credentials +// response names the same host; +// - no --registry-mirror: the platform's registry of the device credentials response when it sends +// one (not yet), otherwise nothing: registries.yaml is left alone. +// +// Errors never contain the credential. +func registryFromFlags(mirrorFlag, credFile string, platform *api.FleetRegistry) (registrySettings, error) { + mirror := strings.TrimSpace(mirrorFlag) + if err := templates.ValidateRegistryMirror(mirror); err != nil { + return registrySettings{}, err + } + fromPlatform := false + if mirror == "" && platform != nil && platform.Host != "" { + mirror = platform.Host + fromPlatform = true + if err := templates.ValidateRegistryMirror(mirror); err != nil { + return registrySettings{}, fmt.Errorf("the registry mirror that the platform sent is unusable: %w", err) + } + } + switch mirror { + case "": + if credFile != "" { + return registrySettings{}, fmt.Errorf("--registry-credential-file needs --registry-mirror HOST") + } + return registrySettings{}, nil + case templates.RegistryMirrorNone: + return registrySettings{Mirror: mirror}, nil + } + s := registrySettings{Mirror: mirror} + switch { + case credFile != "": + user, pass, err := readRegistryCredentialFile(credFile) + if err != nil { + return registrySettings{}, err + } + s.Username, s.Password = user, pass + case platform != nil && platform.Host == mirror && platform.Username != "" && platform.Password != "": + s.Username, s.Password = platform.Username, platform.Password + default: + what := "--registry-mirror " + mirror + if fromPlatform { + what = "the registry mirror " + mirror + " that the platform sent" + } + return registrySettings{}, fmt.Errorf("%s needs a credential: pass --registry-credential-file FILE (one line username:password; the Harbor proxy projects are private)", what) + } + return s, nil +} + +// checkRegistryFlags checks the two flags on their own, before the platform is asked for anything: the +// mirror syntax and, when given, the credential file. Whether a mirror and a credential are both there +// is decided by registryFromFlags once the platform's answer is known. +func checkRegistryFlags(mirrorFlag, credFile string) error { + if err := templates.ValidateRegistryMirror(strings.TrimSpace(mirrorFlag)); err != nil { + return err + } + if credFile != "" { + if _, _, err := readRegistryCredentialFile(credFile); err != nil { + return err + } + } + return nil +} + +// readRegistryCredentialFile reads `username:password` from the first line of path. Where file modes +// mean something, a file that group or others can read is refused: it holds a password. The password +// is never part of an error. +func readRegistryCredentialFile(path string) (user, pass string, err error) { + fi, err := os.Stat(path) + if err != nil { + return "", "", fmt.Errorf("--registry-credential-file: %w", err) + } + if !fi.Mode().IsRegular() { + return "", "", fmt.Errorf("--registry-credential-file %s is not a regular file", path) + } + if runtime.GOOS != "windows" && fi.Mode().Perm()&0o077 != 0 { + return "", "", fmt.Errorf("--registry-credential-file %s has mode %04o: it holds a password, make it readable by you only (chmod 600)", path, fi.Mode().Perm()) + } + b, err := os.ReadFile(path) + if err != nil { + return "", "", fmt.Errorf("--registry-credential-file: %w", err) + } + if len(strings.TrimSpace(string(b))) == 0 { + return "", "", fmt.Errorf("--registry-credential-file %s is empty", path) + } + lines := strings.Split(strings.TrimRight(string(b), "\r\n"), "\n") + if len(lines) != 1 { + return "", "", fmt.Errorf("--registry-credential-file %s must hold exactly one line, username:password", path) + } + line := strings.TrimSuffix(lines[0], "\r") + user, pass, ok := strings.Cut(line, ":") + if !ok || user == "" || pass == "" { + return "", "", fmt.Errorf("--registry-credential-file %s must hold one line username:password", path) + } + return user, pass, nil +} diff --git a/cmd/bootstrap_registry_test.go b/cmd/bootstrap_registry_test.go new file mode 100644 index 0000000..47a5b14 --- /dev/null +++ b/cmd/bootstrap_registry_test.go @@ -0,0 +1,185 @@ +package cmd + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/teknoir/cli/pkg/api" + "github.com/teknoir/cli/pkg/templates" +) + +const testRegPassword = "S3cretRobotPassw0rdFORtest" + +func credFile(t *testing.T, content string, mode os.FileMode) string { + t.Helper() + p := filepath.Join(t.TempDir(), "cred") + if err := os.WriteFile(p, []byte(content), mode); err != nil { + t.Fatal(err) + } + if err := os.Chmod(p, mode); err != nil { + t.Fatal(err) + } + return p +} + +func TestBootstrapRegistryFlagsExist(t *testing.T) { + for _, name := range []string{"registry-mirror", "registry-credential-file"} { + f := bootstrapCmd.Flags().Lookup(name) + if f == nil { + t.Fatalf("no --%s flag", name) + } + if f.DefValue != "" { + t.Errorf("--%s defaults to %q: without it registries.yaml must be left alone", name, f.DefValue) + } + } +} + +func TestRegistryFromFlags_NothingGivenChangesNothing(t *testing.T) { + s, err := registryFromFlags("", "", nil) + if err != nil || s != (registrySettings{}) { + t.Fatalf("%+v %v", s, err) + } + // a platform without a registry field, or with an empty one, changes nothing either + for _, p := range []*api.FleetRegistry{nil, {}} { + if s, err := registryFromFlags("", "", p); err != nil || s != (registrySettings{}) { + t.Fatalf("%+v %v", s, err) + } + } +} + +func TestRegistryFromFlags_HostWithCredentialFile(t *testing.T) { + f := credFile(t, "robot$nodes:"+testRegPassword+"\n", 0o600) + s, err := registryFromFlags("harbor.teknoir.cloud", f, nil) + if err != nil { + t.Fatal(err) + } + if s != (registrySettings{Mirror: "harbor.teknoir.cloud", Username: "robot$nodes", Password: testRegPassword}) { + t.Errorf("%+v", s) + } + // CRLF files and a colon inside the password + f = credFile(t, "u:pa:ss\r\n", 0o600) + s, err = registryFromFlags("harbor.teknoir.cloud", f, nil) + if err != nil || s.Username != "u" || s.Password != "pa:ss" { + t.Errorf("%+v %v", s, err) + } +} + +func TestRegistryFromFlags_None(t *testing.T) { + s, err := registryFromFlags("none", "", nil) + if err != nil || s != (registrySettings{Mirror: "none"}) { + t.Fatalf("%+v %v", s, err) + } + // none carries no credential, even when the platform sends one + s, err = registryFromFlags("none", "", &api.FleetRegistry{Host: "h.example", Username: "u", Password: "p"}) + if err != nil || s != (registrySettings{Mirror: "none"}) { + t.Fatalf("%+v %v", s, err) + } +} + +func TestRegistryFromFlags_PlatformDefault(t *testing.T) { + p := &api.FleetRegistry{Host: "harbor.example.test", Username: "robot$ts-demo", Password: testRegPassword} + s, err := registryFromFlags("", "", p) + if err != nil || s != (registrySettings{Mirror: "harbor.example.test", Username: "robot$ts-demo", Password: testRegPassword}) { + t.Fatalf("%+v %v", s, err) + } + // the same host given explicitly takes the platform's credential + s, err = registryFromFlags("harbor.example.test", "", p) + if err != nil || s.Password != testRegPassword { + t.Fatalf("%+v %v", s, err) + } + // a file wins over the platform's credential + f := credFile(t, "mine:filepw\n", 0o600) + s, err = registryFromFlags("", f, p) + if err != nil || s.Username != "mine" || s.Mirror != "harbor.example.test" { + t.Fatalf("%+v %v", s, err) + } + // another host explicitly: the platform's credential is for its own host only + if _, err = registryFromFlags("other.example.test", "", p); err == nil { + t.Fatal("the platform's credential must not be sent to another host") + } + // the platform's mirror without a credential still needs one + if _, err = registryFromFlags("", "", &api.FleetRegistry{Host: "harbor.example.test"}); err == nil || !strings.Contains(err.Error(), "that the platform sent") { + t.Fatalf("%v", err) + } + // an unusable host from the platform is refused, not rendered + if _, err = registryFromFlags("", "", &api.FleetRegistry{Host: "bad host", Username: "u", Password: "p"}); err == nil { + t.Fatal("a bad host from the platform was accepted") + } +} + +func TestRegistryFromFlags_Rejects(t *testing.T) { + good := credFile(t, "u:"+testRegPassword+"\n", 0o600) + cases := map[string]struct { + mirror, file string + }{ + "host without a credential": {"harbor.example.test", ""}, + "credential without a host": {"", good}, + "invalid host": {"bad host", good}, + "host with scheme": {"https://harbor.example.test", good}, + "host with injection": {"a$(id).example", good}, + "missing file": {"harbor.example.test", filepath.Join(t.TempDir(), "nope")}, + "empty file": {"harbor.example.test", credFile(t, "", 0o600)}, + "blank file": {"harbor.example.test", credFile(t, "\n\n", 0o600)}, + "no colon": {"harbor.example.test", credFile(t, testRegPassword+"\n", 0o600)}, + "no user": {"harbor.example.test", credFile(t, ":"+testRegPassword+"\n", 0o600)}, + "no password": {"harbor.example.test", credFile(t, "user:\n", 0o600)}, + "two lines": {"harbor.example.test", credFile(t, "u:"+testRegPassword+"\nv:w\n", 0o600)}, + "directory": {"harbor.example.test", t.TempDir()}, + } + if runtime.GOOS != "windows" { + cases["group readable"] = struct{ mirror, file string }{"harbor.example.test", credFile(t, "u:"+testRegPassword+"\n", 0o640)} + cases["world readable"] = struct{ mirror, file string }{"harbor.example.test", credFile(t, "u:"+testRegPassword+"\n", 0o644)} + } + for name, c := range cases { + s, err := registryFromFlags(c.mirror, c.file, nil) + if err == nil { + t.Errorf("%s: accepted (%+v)", name, s) + continue + } + if strings.Contains(err.Error(), testRegPassword) { + t.Errorf("%s: the error contains the password: %v", name, err) + } + } +} + +func TestCheckRegistryFlags(t *testing.T) { + if err := checkRegistryFlags("", ""); err != nil { + t.Fatal(err) + } + // a credential file without a host is fine here: the platform may still name the host + if err := checkRegistryFlags("", credFile(t, "u:p\n", 0o600)); err != nil { + t.Fatal(err) + } + if err := checkRegistryFlags("bad host", ""); err == nil { + t.Fatal("bad host accepted") + } + if err := checkRegistryFlags("harbor.example.test", filepath.Join(t.TempDir(), "nope")); err == nil { + t.Fatal("missing file accepted") + } +} + +// What registryFromFlags returns renders into a script that ValidateTemplateData accepts, and the +// robot$ user name needs no escaping by the user. +func TestRegistrySettingsRender(t *testing.T) { + f := credFile(t, "robot$nodes:"+testRegPassword+"\n", 0o600) + s, err := registryFromFlags("harbor.teknoir.cloud", f, nil) + if err != nil { + t.Fatal(err) + } + d := templates.TemplateData{Domain: "example.test", Namespace: "ns", DeviceID: "dev-1", K3SToken: "tok", + RegistryMirror: s.Mirror, RegistryUsername: s.Username, RegistryPassword: s.Password} + for name, render := range map[string]func() ([]byte, error){ + "server": func() ([]byte, error) { return templates.RenderServer(d) }, + } { + out, err := render() + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if !strings.Contains(string(out), "_TN_REG_HOST='harbor.teknoir.cloud'") || !strings.Contains(string(out), "_TN_REG_USER='robot$nodes'") { + t.Errorf("%s: the registry block is missing", name) + } + } +} diff --git a/docs/commands/bootstrap.md b/docs/commands/bootstrap.md index 4b741a1..16d7e06 100644 --- a/docs/commands/bootstrap.md +++ b/docs/commands/bootstrap.md @@ -19,6 +19,8 @@ The device may be given as an argument, via the `--device` global flag, or selec | `--k3s-upgrade` | `false` | Let the scripts replace an installed k3s with `--k3s-version` (sets `TN_K3S_UPGRADE=true` as the script's default). Without it a re-run **keeps** the installed k3s. See [Re-running a script](#re-running-a-script-the-installed-k3s-is-kept). | | `--force` | `false` | Only with `--k3s-upgrade`: allow an upgrade over more than one minor version (`TN_K3S_FORCE=true`). A downgrade is never allowed. | | `--docker-secret` | `""` | Contents of the `.dockerconfigjson` to embed. | +| `--registry-mirror HOST\|none` | `""` | Pull images through the Harbor host `HOST` (its proxy-cache projects), with the upstream registry as the fallback. The server and agent scripts write `/etc/rancher/k3s/registries.yaml` before the k3s installer. `none` removes a file the scripts wrote. Empty leaves `registries.yaml` alone. See [Registry mirror](#registry-mirror-harbor-pull-through-cache). | +| `--registry-credential-file FILE` | `""` | The Harbor credential for `--registry-mirror`: one line `username:password` (a robot account of the teamspace), readable by you only. It goes into the generated scripts and is never printed. | | `--robot` | `""` | Name of the robot (the multi-computer cluster) the computers belong to. Becomes the node label `teknoir.org/robot` and, together with a role, the node name `-`. See [Robots with several computers](#robots-with-several-computers). | | `--role` | `""` | Node role of the **server** script (a DNS label such as `vision` or `io`). Becomes the node label `teknoir.org/node-role`. | | `--node-name` | `""` | Node name of the server script. Default: `-` with `--robot` and `--role`, otherwise `teknoir-master` as before. | @@ -151,6 +153,39 @@ Without any of these flags the scripts are exactly what they were before (`tekno server, the host name for the agent, no labels). The docker server script (`k3s < 1.24`) does not take node roles. +## Registry mirror (Harbor pull-through cache) + +With `--registry-mirror HOST` the robot pulls its images through the platform's Harbor instead of straight +from the internet: Docker Hub, GitHub (`ghcr.io`), `gcr.io`, `quay.io`, `registry.k8s.io` and `nvcr.io` map +to the Harbor projects `dockerhub`, `ghcr`, `gcr`, `quay`, `k8s` and `nvcr`. Private `ghcr.io/teknoir/*` +images then reach the robot without a GitHub token on the device, and a popular image is downloaded once +per site instead of once per robot. + +```bash +printf 'robot$ts-my-teamspace:%s\n' "$HARBOR_ROBOT_SECRET" > harbor-pull && chmod 600 harbor-pull +tnctl bootstrap edge-01 --registry-mirror harbor.teknoir.cloud --registry-credential-file harbor-pull +``` + +- **Fallback.** k3s always tries the upstream registry last. When Harbor is down or slow to fail, public + images still come from the internet; a private image that is not on the device yet does not (the robots + have no upstream credential). +- **What is written.** The server and agent scripts write `/etc/rancher/k3s/registries.yaml` (mode `0600`, + first line `# managed by tnctl bootstrap`) before the k3s installer runs, so the first start already uses + it. On a re-run the installer restarts k3s. The file is written only when it differs. +- **A file you wrote is never touched.** A `registries.yaml` without the marker line is kept, with a + warning, and the mirror is not configured. `--registry-mirror none` removes a file that the scripts wrote + (and keeps yours). Without `--registry-mirror` the scripts leave the file alone. +- **The credential** is a Harbor robot account that can only pull from the proxy projects. It is embedded in + the scripts (like the device keys, so keep the scripts private) and the scripts never print it. The file + must not be readable by group or others. +- The Docker-mode script (`bootstrap_docker_server_.sh`) is not changed. +- For a robot that is already running, the Ansible playbook `registry-mirror.yml` (repository `ansible`) + does the same, with a probe before anything changes and a rollback. + +The platform may later send the registry and the credential together with the device login; `tnctl` then +uses them as the default of `--registry-mirror` and its credential (a `registry` field in the device +credentials response, ignored when absent). + ## The k3s version The server and agent scripts install a **pinned** k3s release: `INSTALL_K3S_VERSION` is set from @@ -312,6 +347,9 @@ tnctl bootstrap --docker-secret "$(cat dockerconfig.json)" # Generate but do not upload tnctl bootstrap edge-01 --skip-upload +# Pull images through Harbor (credential in a file, never printed) +tnctl bootstrap edge-01 --registry-mirror harbor.teknoir.cloud --registry-credential-file harbor-pull + # A single-computer robot with a GPU: the server script gets the role vision tnctl bootstrap vision-station-01 --robot vision-station-01 --role vision --label teknoir.org/gpu=nvidia diff --git a/pkg/api/fleet_devices.go b/pkg/api/fleet_devices.go index 4ed3bfe..3e259d5 100644 --- a/pkg/api/fleet_devices.go +++ b/pkg/api/fleet_devices.go @@ -216,6 +216,18 @@ type FleetDeviceCredentials struct { RSAPrivate string `json:"rsaPrivate,omitempty"` RSAPublic string `json:"rsaPublic,omitempty"` PublicSSHKey string `json:"publicSshKey,omitempty"` + // Registry is the image registry mirror (Harbor pull-through cache) of the device's teamspace, when the + // platform sends one. It is the default of `tnctl bootstrap --registry-mirror` and its credential. Nothing + // sends it yet (planned, plans/harbor-proxy-staging.md phase 1b): it is ignored when absent. + Registry *FleetRegistry `json:"registry,omitempty"` +} + +// FleetRegistry is the registry mirror a platform offers a device: the Harbor host and a pull-only robot +// credential, both plain text (not base64). A secret: never log or print it. +type FleetRegistry struct { + Host string `json:"host"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` } // FleetDeviceCredentialsBulk is the answer of the bulk credentials route. diff --git a/pkg/api/fleet_devices_test.go b/pkg/api/fleet_devices_test.go index b0b2763..e1fd04e 100644 --- a/pkg/api/fleet_devices_test.go +++ b/pkg/api/fleet_devices_test.go @@ -378,3 +378,26 @@ func TestNoCatalogCallsInTheSource(t *testing.T) { t.Errorf("code that calls the catalog: %v", hits) } } + +// The registry field of the device credentials is optional (phase 1b of plans/harbor-proxy-staging.md): +// absent it is nil, present it decodes as plain text. +func TestFleetDeviceCredentials_RegistryIsOptional(t *testing.T) { + var c FleetDeviceCredentials + if err := json.Unmarshal([]byte(`{"name":"d","namespace":"ns","username":"dQ=="}`), &c); err != nil { + t.Fatal(err) + } + if c.Registry != nil { + t.Fatalf("registry %+v without the field", c.Registry) + } + if err := json.Unmarshal([]byte(`{"name":"d","namespace":"ns","registry":{"host":"harbor.example.test","username":"robot$ts-ns","password":"pw"}}`), &c); err != nil { + t.Fatal(err) + } + if c.Registry == nil || c.Registry.Host != "harbor.example.test" || c.Registry.Username != "robot$ts-ns" || c.Registry.Password != "pw" { + t.Fatalf("%+v", c.Registry) + } + // unchanged encoding when there is none + b, _ := json.Marshal(FleetDeviceCredentials{Name: "d"}) + if strings.Contains(string(b), "registry") { + t.Fatalf("empty registry encoded: %s", b) + } +} diff --git a/pkg/templates/agent.sh.tmpl b/pkg/templates/agent.sh.tmpl index 0eb8c08..070da3e 100644 --- a/pkg/templates/agent.sh.tmpl +++ b/pkg/templates/agent.sh.tmpl @@ -45,7 +45,7 @@ $SUDO cp ${CONFIG_PATH}/rsa_public.pem /usr/local/share/ca-certificates/${_DEVIC $SUDO update-ca-certificates info "Install Rancher K3s Agent" -download k3s_installer.sh https://get.k3s.io +{{template "registries.sh.tmpl" .}}download k3s_installer.sh https://get.k3s.io $SUDO chmod +x k3s_installer.sh if [ "${OS_BUILD}" ] || [ "${INSECURE}" ]; then diff --git a/pkg/templates/bootstrap_templates.go b/pkg/templates/bootstrap_templates.go index d94f358..95db99c 100644 --- a/pkg/templates/bootstrap_templates.go +++ b/pkg/templates/bootstrap_templates.go @@ -60,6 +60,32 @@ type TemplateData struct { Server string // TLSSANs are extra --tls-san values of the server script, after the two it always adds. TLSSANs []string + // RegistryMirror is the Harbor host the device pulls images through (`tnctl bootstrap --registry-mirror`): + // the scripts write /etc/rancher/k3s/registries.yaml with the six proxy-cache mirrors (docker.io, + // ghcr.io, gcr.io, quay.io, registry.k8s.io, nvcr.io) and the upstream registry as the fallback. + // "none" removes a file that the scripts wrote. Empty leaves registries.yaml alone and renders no code. + RegistryMirror string + // RegistryUsername and RegistryPassword are the Harbor robot credential for RegistryMirror. They are + // embedded in single quotes in the script and never printed by it; ValidateTemplateData allows a `$` + // here (a Harbor robot is called robot$name) but no quote, backslash, newline or control character. + RegistryUsername string + RegistryPassword string +} + +// registryHostRE is a Harbor host name, optionally with a port: what the k3s mirror endpoint +// https:// can hold. +var registryHostRE = regexp.MustCompile(`^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?(:[0-9]{1,5})?$`) + +// RegistryMirrorNone is the value of RegistryMirror that removes a registries.yaml written by the scripts. +const RegistryMirrorNone = "none" + +// ValidateRegistryMirror accepts "" (no registry mirror), "none", or a host name with an optional port. +// The value ends up in a shell script and in a YAML file, so nothing else is let through. +func ValidateRegistryMirror(v string) error { + if v == "" || v == RegistryMirrorNone || registryHostRE.MatchString(v) { + return nil + } + return fmt.Errorf("invalid registry mirror %q: want a host name such as harbor.teknoir.cloud (optionally with :port), or %q", v, RegistryMirrorNone) } func (d TemplateData) k3sVersion() string { diff --git a/pkg/templates/golden_data_test.go b/pkg/templates/golden_data_test.go index 926726b..920afa3 100644 --- a/pkg/templates/golden_data_test.go +++ b/pkg/templates/golden_data_test.go @@ -14,5 +14,9 @@ func goldenData() TemplateData { PublicSSHKey: "ssh-ed25519 TEST-ONLY-ssh-public golden@example.test", K3SToken: "TEST-ONLY-k3s-token", DockerSecret: "TEST-ONLY-docker-secret", + // the registry mirror, so the golden files cover registries.sh.tmpl + RegistryMirror: "harbor.example.test", + RegistryUsername: "robot$golden", + RegistryPassword: "TEST-ONLY-registry-password", } } diff --git a/pkg/templates/partials/registries.sh.tmpl b/pkg/templates/partials/registries.sh.tmpl new file mode 100644 index 0000000..4173564 --- /dev/null +++ b/pkg/templates/partials/registries.sh.tmpl @@ -0,0 +1,50 @@ +{{define "registries.sh.tmpl"}}{{if .RegistryMirror}}# The registry mirror (tnctl bootstrap --registry-mirror): pull images through the Harbor proxy-cache +# projects, with the upstream registry as the fallback (k3s always tries its default endpoint last, so +# a Harbor outage never blocks public images). It runs before the k3s installer, so a first start already +# uses it; a re-run restarts k3s through the installer. The file is written only when it differs, at +# mode 0600 (it holds the Harbor credential), and a registries.yaml that this script did not write is +# never touched. The credential is never printed. +_TN_REG_FILE="${K3S_CONFIG_PATH:-/etc/rancher/k3s}/registries.yaml" +_TN_REG_MARKER='# managed by tnctl bootstrap' +_TN_REG_HOST='{{.RegistryMirror}}' +{{if ne .RegistryMirror "none"}}_TN_REG_USER='{{.RegistryUsername}}' +_TN_REG_PASS='{{.RegistryPassword}}' +# A double-quoted YAML scalar: backslash and double quote escaped. +_tn_reg_q() { printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'; } +{{end}}if [ -e "${_TN_REG_FILE}" ] && ! $SUDO grep -qxF "${_TN_REG_MARKER}" "${_TN_REG_FILE}"; then + warn "${_TN_REG_FILE} exists and was not written by tnctl bootstrap: keeping it, the registry mirror is not configured" +{{- if eq .RegistryMirror "none"}} +elif [ -e "${_TN_REG_FILE}" ]; then + $SUDO rm -f "${_TN_REG_FILE}" + info "Registry mirror removed (${_TN_REG_FILE}); pulls go to the upstream registries again" +fi +{{- else}} +else + $SUDO mkdir -p "$(dirname "${_TN_REG_FILE}")" + _tn_reg_tmp=$(umask 077; mktemp "${_TN_REG_FILE}.XXXXXX") || fatal "Cannot create a temporary file next to ${_TN_REG_FILE}" + { + printf '%s\n' "${_TN_REG_MARKER}" + printf '# k3s private registry mirrors: pull-through cache on %s. containerd asks Harbor first and\n' "${_TN_REG_HOST}" + printf '# falls back to the upstream registry. Written by tnctl bootstrap --registry-mirror; do not edit.\n' + printf 'mirrors:\n' + for _tn_reg_m in docker.io=dockerhub ghcr.io=ghcr gcr.io=gcr quay.io=quay registry.k8s.io=k8s nvcr.io=nvcr; do + printf ' %s:\n endpoint:\n - "https://%s"\n rewrite:\n "^(.*)$": "%s/$1"\n' \ + "${_tn_reg_m%%=*}" "${_TN_REG_HOST}" "${_tn_reg_m#*=}" + done + printf 'configs:\n "%s":\n auth:\n username: "%s"\n password: "%s"\n' \ + "${_TN_REG_HOST}" "$(_tn_reg_q "${_TN_REG_USER}")" "$(_tn_reg_q "${_TN_REG_PASS}")" + } > "${_tn_reg_tmp}" || { rm -f "${_tn_reg_tmp}"; fatal "Cannot write ${_TN_REG_FILE}"; } + if [ -f "${_TN_REG_FILE}" ] && cmp -s "${_tn_reg_tmp}" "${_TN_REG_FILE}"; then + rm -f "${_tn_reg_tmp}" + $SUDO chmod 0600 "${_TN_REG_FILE}" + info "Registry mirror ${_TN_REG_HOST} already configured" + else + $SUDO chmod 0600 "${_tn_reg_tmp}" + $SUDO mv -f "${_tn_reg_tmp}" "${_TN_REG_FILE}" || { rm -f "${_tn_reg_tmp}"; fatal "Cannot write ${_TN_REG_FILE}"; } + info "Registry mirror ${_TN_REG_HOST} configured (${_TN_REG_FILE})" + fi + unset _tn_reg_tmp _tn_reg_m +fi +unset _TN_REG_USER _TN_REG_PASS +{{- end}} +{{end}}{{end}} diff --git a/pkg/templates/registries_test.go b/pkg/templates/registries_test.go new file mode 100644 index 0000000..584d85d --- /dev/null +++ b/pkg/templates/registries_test.go @@ -0,0 +1,403 @@ +package templates + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" +) + +const ( + regPassword = "S3cretRobotPassw0rdFORtest" + regUsername = "robot$nodes" + regMarker = "# managed by tnctl bootstrap" +) + +// registriesBlock renders the registries partial for d, the way server.sh.tmpl and agent.sh.tmpl include it. +func registriesBlock(t *testing.T, d TemplateData) string { + t.Helper() + var b strings.Builder + if err := ServerTemplate.ExecuteTemplate(&b, "registries.sh.tmpl", d); err != nil { + t.Fatal(err) + } + return b.String() +} + +func regData(mirror string) TemplateData { + d := goldenData() + d.RegistryMirror = mirror + d.RegistryUsername = regUsername + d.RegistryPassword = regPassword + if mirror == "none" || mirror == "" { + d.RegistryUsername, d.RegistryPassword = "", "" + } + return d +} + +type regRun struct { + dir string // K3S_CONFIG_PATH + output string // combined stdout and stderr + err error +} + +func (r regRun) file() string { return filepath.Join(r.dir, "registries.yaml") } + +// runRegistries runs the partial in dir under `set -e` and `umask 027` (the header of the bootstrap +// script sets both), with info/warn/fatal as in the header. SUDO is empty, as for root. +func runRegistries(t *testing.T, dir string, d TemplateData) regRun { + t.Helper() + script := "set -e\numask 027\n" + + "info() { echo \"[INFO] $*\"; }\nwarn() { echo \"[WARN] $*\" >&2; }\nfatal() { echo \"[ERROR] $*\" >&2; exit 1; }\n" + + "SUDO=\nK3S_CONFIG_PATH=" + dir + "\n" + registriesBlock(t, d) + "echo done\n" + var out bytes.Buffer + cmd := exec.Command("sh", "-c", script) + cmd.Stdout, cmd.Stderr = &out, &out + err := cmd.Run() + return regRun{dir: dir, output: out.String(), err: err} +} + +func (r regRun) mustOK(t *testing.T) regRun { + t.Helper() + if r.err != nil || !strings.HasSuffix(r.output, "done\n") { + t.Fatalf("script failed: %v\n%s", r.err, r.output) + } + return r +} + +func readFile(t *testing.T, p string) string { + t.Helper() + b, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func inode(t *testing.T, p string) uint64 { + t.Helper() + fi, err := os.Stat(p) + if err != nil { + t.Fatal(err) + } + return fi.Sys().(*syscall.Stat_t).Ino +} + +func noPassword(t *testing.T, what, s string) { + t.Helper() + if strings.Contains(s, regPassword) { + t.Errorf("%s contains the password", what) + } +} + +const wantFresh = regMarker + ` +# k3s private registry mirrors: pull-through cache on harbor.example.test. containerd asks Harbor first and +# falls back to the upstream registry. Written by tnctl bootstrap --registry-mirror; do not edit. +mirrors: + docker.io: + endpoint: + - "https://harbor.example.test" + rewrite: + "^(.*)$": "dockerhub/$1" + ghcr.io: + endpoint: + - "https://harbor.example.test" + rewrite: + "^(.*)$": "ghcr/$1" + gcr.io: + endpoint: + - "https://harbor.example.test" + rewrite: + "^(.*)$": "gcr/$1" + quay.io: + endpoint: + - "https://harbor.example.test" + rewrite: + "^(.*)$": "quay/$1" + registry.k8s.io: + endpoint: + - "https://harbor.example.test" + rewrite: + "^(.*)$": "k8s/$1" + nvcr.io: + endpoint: + - "https://harbor.example.test" + rewrite: + "^(.*)$": "nvcr/$1" +configs: + "harbor.example.test": + auth: + username: "robot$nodes" + password: "` + regPassword + `" +` + +func TestRegistries_Fresh(t *testing.T) { + dir := t.TempDir() + r := runRegistries(t, dir, regData("harbor.example.test")).mustOK(t) + if got := readFile(t, r.file()); got != wantFresh { + t.Fatalf("registries.yaml:\n%s\nwant:\n%s", got, wantFresh) + } + fi, _ := os.Stat(r.file()) + if fi.Mode().Perm() != 0o600 { + t.Errorf("mode %v, want 0600 (under umask 027)", fi.Mode().Perm()) + } + if !strings.Contains(r.output, "Registry mirror harbor.example.test configured") { + t.Errorf("output: %s", r.output) + } + noPassword(t, "output", r.output) + if left, _ := filepath.Glob(filepath.Join(dir, "registries.yaml.*")); len(left) != 0 { + t.Errorf("temp files left behind: %v", left) + } +} + +func TestRegistries_RerunIsANoOp(t *testing.T) { + dir := t.TempDir() + r := runRegistries(t, dir, regData("harbor.example.test")).mustOK(t) + ino, before := inode(t, r.file()), readFile(t, r.file()) + if err := os.Chmod(r.file(), 0o644); err != nil { // loosened by someone: repaired, content untouched + t.Fatal(err) + } + r2 := runRegistries(t, dir, regData("harbor.example.test")).mustOK(t) + if !strings.Contains(r2.output, "already configured") { + t.Errorf("output: %s", r2.output) + } + if inode(t, r.file()) != ino || readFile(t, r.file()) != before { + t.Error("a re-run rewrote the file") + } + if fi, _ := os.Stat(r.file()); fi.Mode().Perm() != 0o600 { + t.Errorf("mode %v after a re-run", fi.Mode().Perm()) + } + noPassword(t, "output", r2.output) + if left, _ := filepath.Glob(filepath.Join(dir, "registries.yaml.*")); len(left) != 0 { + t.Errorf("temp files left behind: %v", left) + } +} + +func TestRegistries_ChangedCredentialAndHostAreWritten(t *testing.T) { + dir := t.TempDir() + r := runRegistries(t, dir, regData("harbor.example.test")).mustOK(t) + d := regData("harbor.example.test") + d.RegistryPassword = "RotatedPassw0rd" + runRegistries(t, dir, d).mustOK(t) + if got := readFile(t, r.file()); !strings.Contains(got, `password: "RotatedPassw0rd"`) || strings.Contains(got, regPassword) { + t.Errorf("rotated password not written:\n%s", got) + } + d = regData("other.example.test:8443") + runRegistries(t, dir, d).mustOK(t) + if got := readFile(t, r.file()); !strings.Contains(got, `"https://other.example.test:8443"`) || strings.Contains(got, "harbor.example.test") { + t.Errorf("new host not written:\n%s", got) + } +} + +func TestRegistries_ForeignFileIsKept(t *testing.T) { + dir := t.TempDir() + foreign := "mirrors:\n docker.io:\n endpoint:\n - \"https://mine.example\"\n" + p := filepath.Join(dir, "registries.yaml") + if err := os.WriteFile(p, []byte(foreign), 0o640); err != nil { + t.Fatal(err) + } + for _, mirror := range []string{"harbor.example.test", "none"} { + r := runRegistries(t, dir, regData(mirror)).mustOK(t) + if readFile(t, p) != foreign { + t.Errorf("%s: the foreign file was changed", mirror) + } + if fi, _ := os.Stat(p); fi.Mode().Perm() != 0o640 { + t.Errorf("%s: mode of the foreign file changed to %v", mirror, fi.Mode().Perm()) + } + if !strings.Contains(r.output, "[WARN]") || !strings.Contains(r.output, "was not written by tnctl bootstrap") { + t.Errorf("%s: no warning:\n%s", mirror, r.output) + } + noPassword(t, "output", r.output) + } + // a comment that only contains the marker text is not the marker + if err := os.WriteFile(p, []byte("# not "+regMarker+"\nmirrors: {}\n"), 0o600); err != nil { + t.Fatal(err) + } + r := runRegistries(t, dir, regData("harbor.example.test")).mustOK(t) + if !strings.Contains(r.output, "[WARN]") || strings.Contains(readFile(t, p), "harbor.example.test") { + t.Errorf("a file with the marker text inside a comment was taken as ours:\n%s", r.output) + } +} + +func TestRegistries_NoneRemovesAManagedFile(t *testing.T) { + dir := t.TempDir() + r := runRegistries(t, dir, regData("harbor.example.test")).mustOK(t) + r2 := runRegistries(t, dir, regData("none")).mustOK(t) + if _, err := os.Stat(r.file()); !os.IsNotExist(err) { + t.Errorf("the managed file is still there: %v", err) + } + if !strings.Contains(r2.output, "Registry mirror removed") { + t.Errorf("output: %s", r2.output) + } + // without a file, none is a quiet no-op + r3 := runRegistries(t, dir, regData("none")).mustOK(t) + if strings.TrimSpace(r3.output) != "done" { + t.Errorf("none without a file printed: %q", r3.output) + } + noPassword(t, "output", r2.output+r3.output) + // none never carries a credential into the script + if blk := registriesBlock(t, regData("none")); strings.Contains(blk, "_TN_REG_PASS") || strings.Contains(blk, regPassword) { + t.Error("the none variant mentions a credential") + } +} + +func TestRegistries_EmptyMirrorRendersNothingAndLeavesTheFileAlone(t *testing.T) { + d := regData("") + if blk := registriesBlock(t, d); blk != "" { + t.Fatalf("an empty RegistryMirror must render nothing, got:\n%s", blk) + } + for _, tpl := range []string{"server", "agent"} { + var b bytes.Buffer + var err error + if tpl == "server" { + err = ServerTemplate.Execute(&b, d) + } else { + err = AgentTemplate.Execute(&b, d) + } + if err != nil { + t.Fatal(err) + } + if strings.Contains(b.String(), "registries.yaml") || strings.Contains(b.String(), "_TN_REG_") { + t.Errorf("%s script without a mirror mentions the registry code", tpl) + } + } + dir := t.TempDir() + p := filepath.Join(dir, "registries.yaml") + for _, content := range []string{"mine: true\n", regMarker + "\nmirrors: {}\n"} { + if err := os.WriteFile(p, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + runRegistries(t, dir, d).mustOK(t) + if readFile(t, p) != content { + t.Errorf("an empty mirror changed %q", content) + } + } +} + +func TestRegistries_IncludedBeforeTheInstaller(t *testing.T) { + d := goldenData() + for name, render := range map[string]func() (string, error){ + "server": func() (string, error) { + var b bytes.Buffer + err := ServerTemplate.Execute(&b, d) + return b.String(), err + }, + "agent": func() (string, error) { + var b bytes.Buffer + err := AgentTemplate.Execute(&b, d) + return b.String(), err + }, + } { + out, err := render() + if err != nil { + t.Fatal(err) + } + reg, inst := strings.Index(out, "_TN_REG_FILE="), strings.Index(out, "$SUDO ./k3s_installer.sh") + if reg < 0 || inst < 0 || reg > inst { + t.Errorf("%s: the registry block (%d) must come before the k3s installer (%d)", name, reg, inst) + } + if strings.Count(out, "_TN_REG_FILE=") != 1 { + t.Errorf("%s: the registry block appears %d times", name, strings.Count(out, "_TN_REG_FILE=")) + } + } + var b bytes.Buffer + if err := DockerServerTemplate.Execute(&b, d); err != nil { + t.Fatal(err) + } + if strings.Contains(b.String(), "registries.yaml") { + t.Error("the Docker-mode script must not write registries.yaml") + } +} + +// A password with characters that need care (a `#` and a `:` in YAML, `!` and `%` in printf) round-trips +// byte for byte, and a `$` in the user name is not expanded by the shell. +func TestRegistries_SpecialCharactersSurvive(t *testing.T) { + dir := t.TempDir() + d := regData("harbor.example.test") + d.RegistryUsername = "robot$a$HOME" + d.RegistryPassword = "p#a:s!s%s%d-_.~" + if err := ValidateTemplateData(d); err != nil { + t.Fatal(err) + } + r := runRegistries(t, dir, d).mustOK(t) + got := readFile(t, r.file()) + if !strings.Contains(got, `username: "robot$a$HOME"`) || !strings.Contains(got, `password: "p#a:s!s%s%d-_.~"`) { + t.Errorf("credential mangled:\n%s", got) + } +} + +// The password is in the script (it has to be), in single quotes, so a value that tries to close the +// quote or run something is refused by the validation before a script exists. +func TestRegistries_ValidateRegistryData(t *testing.T) { + ok := func(mod func(*TemplateData)) error { + d := regData("harbor.example.test") + mod(&d) + return ValidateTemplateData(d) + } + if err := ok(func(*TemplateData) {}); err != nil { + t.Fatalf("the robot$nodes credential must be accepted: %v", err) + } + for _, h := range []string{"", "none", "harbor.teknoir.cloud", "harbor.example.test:8443", "localhost", "10.0.0.5"} { + if err := ValidateRegistryMirror(h); err != nil { + t.Errorf("host %q rejected: %v", h, err) + } + } + for _, h := range []string{"a b", "a/b", `a"b`, "a'b", "a$b", "a;b", "https://harbor.example.test", "-x", "a:b:c", "x:99999999", "harbor.example.test\n", "$(id)"} { + if err := ValidateRegistryMirror(h); err == nil { + t.Errorf("host %q accepted", h) + } + } + bad := map[string]func(*TemplateData){ + "quote closes": func(d *TemplateData) { d.RegistryPassword = "x'; touch /tmp/pwn; '" }, + "single quote": func(d *TemplateData) { d.RegistryPassword = "a'b" }, + "double quote": func(d *TemplateData) { d.RegistryPassword = `a"b` }, + "backslash": func(d *TemplateData) { d.RegistryPassword = `a\b` }, + "backtick": func(d *TemplateData) { d.RegistryPassword = "a`id`b" }, + "newline": func(d *TemplateData) { d.RegistryPassword = "a\nb" }, + "carriage return": func(d *TemplateData) { d.RegistryPassword = "a\rb" }, + "space": func(d *TemplateData) { d.RegistryPassword = "a b" }, + "nul": func(d *TemplateData) { d.RegistryPassword = "a\x00b" }, + "del": func(d *TemplateData) { d.RegistryPassword = "a\x7fb" }, + "too long": func(d *TemplateData) { d.RegistryPassword = strings.Repeat("a", 513) }, + "user quote": func(d *TemplateData) { d.RegistryUsername = "robot'x" }, + "user newline": func(d *TemplateData) { d.RegistryUsername = "robot\nx" }, + "user empty": func(d *TemplateData) { d.RegistryUsername = "" }, + "password empty": func(d *TemplateData) { d.RegistryPassword = "" }, + "credential w/o host": func(d *TemplateData) { d.RegistryMirror = "" }, + "credential on none": func(d *TemplateData) { d.RegistryMirror = "none" }, + "host with quote": func(d *TemplateData) { d.RegistryMirror = "a'b" }, + "host with dollar": func(d *TemplateData) { d.RegistryMirror = "a$b.example" }, + } + for name, mod := range bad { + err := ok(mod) + if err == nil { + t.Errorf("%s: accepted", name) + continue + } + noPassword(t, name+" error", err.Error()) + for _, leak := range []string{"touch", "a'b", "a\nb"} { + if strings.Contains(err.Error(), leak) { + t.Errorf("%s: error %q leaks %q", name, err, leak) + } + } + if _, rerr := RenderServer(func() TemplateData { d := regData("harbor.example.test"); mod(&d); return d }()); rerr == nil { + t.Errorf("%s: RenderServer accepted it", name) + } + } +} + +// A credential hidden in a quote-closing value must never become shell: run the rendered block with a +// value that the validation would refuse, to prove why it must (the test documents the single-quote context). +func TestRegistries_TheValidationIsWhatKeepsTheScriptSafe(t *testing.T) { + d := regData("harbor.example.test") + d.RegistryPassword = "x'; echo INJECTED; '" + if ValidateTemplateData(d) == nil { + t.Fatal("validation must refuse a quote in the password") + } + blk := registriesBlock(t, d) // unvalidated render: the quote does close the string + if !strings.Contains(blk, "_TN_REG_PASS='x'; echo INJECTED; ''") { + t.Fatalf("test premise changed:\n%s", blk) + } +} diff --git a/pkg/templates/render.go b/pkg/templates/render.go index 00326bf..99a0628 100644 --- a/pkg/templates/render.go +++ b/pkg/templates/render.go @@ -60,6 +60,7 @@ func ValidateTemplateData(d TemplateData) error { {"Node.Robot", d.Node.Robot, false}, {"Node.Name", d.Node.Name, false}, {"Server", d.Server, false}, + {"RegistryMirror", d.RegistryMirror, false}, } for _, l := range d.Node.Labels { fields = append(fields, field{"Node.Labels", l, false}) @@ -78,12 +79,45 @@ func ValidateTemplateData(d TemplateData) error { return fmt.Errorf("template data field %s must not contain a newline", f.name) } } + if err := validateRegistry(d); err != nil { + return err + } if d.Username != "" && (!loginNameRe.MatchString(d.Username) || d.Username == "root") { return fmt.Errorf("template data field Username must be a lowercase login name (a-z, 0-9, _ and -, starting with a letter or _, at most 32 characters, not root)") } return nil } +// validateRegistry checks the registry mirror and its credential. The credential is embedded in +// single quotes (see partials/registries.sh.tmpl), so a `$` is fine; a single quote, a backslash, a +// double quote (it ends up inside a double-quoted YAML scalar after escaping, but the shell variable +// is the place to refuse), whitespace and control characters are not. Errors never contain the value. +func validateRegistry(d TemplateData) error { + if ValidateRegistryMirror(d.RegistryMirror) != nil { + return fmt.Errorf("template data field RegistryMirror must be a host name (optionally with :port) or %q", RegistryMirrorNone) + } + if d.RegistryMirror == "" || d.RegistryMirror == RegistryMirrorNone { + if d.RegistryUsername != "" || d.RegistryPassword != "" { + return fmt.Errorf("template data fields RegistryUsername and RegistryPassword need a RegistryMirror host") + } + return nil + } + if d.RegistryUsername == "" || d.RegistryPassword == "" { + return fmt.Errorf("template data fields RegistryUsername and RegistryPassword must both be set for RegistryMirror %s", d.RegistryMirror) + } + for _, f := range []field{{"RegistryUsername", d.RegistryUsername, false}, {"RegistryPassword", d.RegistryPassword, false}} { + if len(f.value) > 512 { + return fmt.Errorf("template data field %s is longer than 512 characters", f.name) + } + for _, r := range f.value { + if r == '\'' || r == '"' || r == '\\' || r == '`' || r <= ' ' || r == 0x7f { + return fmt.Errorf("template data field %s contains a character that is unsafe in a shell script (quote, backtick, backslash, whitespace or control character)", f.name) + } + } + } + return nil +} + // RenderServer validates d and renders the server (k3s server node) bootstrap // script. It produces the same bytes as ServerTemplate.Execute for valid input. func RenderServer(d TemplateData) ([]byte, error) { diff --git a/pkg/templates/server.sh.tmpl b/pkg/templates/server.sh.tmpl index 41d524d..093aafa 100644 --- a/pkg/templates/server.sh.tmpl +++ b/pkg/templates/server.sh.tmpl @@ -124,7 +124,7 @@ fs.inotify.max_user_instances=1024 EOF sysctl --system -download k3s_installer.sh https://get.k3s.io +{{template "registries.sh.tmpl" .}}download k3s_installer.sh https://get.k3s.io $SUDO chmod +x k3s_installer.sh if [ ${OS_BUILD} ] || [ ${INSECURE} ]; then diff --git a/pkg/templates/testdata/agent.golden.sh b/pkg/templates/testdata/agent.golden.sh index 7c74599..5a89abf 100644 --- a/pkg/templates/testdata/agent.golden.sh +++ b/pkg/templates/testdata/agent.golden.sh @@ -754,6 +754,48 @@ $SUDO cp ${CONFIG_PATH}/rsa_public.pem /usr/local/share/ca-certificates/${_DEVIC $SUDO update-ca-certificates info "Install Rancher K3s Agent" +# The registry mirror (tnctl bootstrap --registry-mirror): pull images through the Harbor proxy-cache +# projects, with the upstream registry as the fallback (k3s always tries its default endpoint last, so +# a Harbor outage never blocks public images). It runs before the k3s installer, so a first start already +# uses it; a re-run restarts k3s through the installer. The file is written only when it differs, at +# mode 0600 (it holds the Harbor credential), and a registries.yaml that this script did not write is +# never touched. The credential is never printed. +_TN_REG_FILE="${K3S_CONFIG_PATH:-/etc/rancher/k3s}/registries.yaml" +_TN_REG_MARKER='# managed by tnctl bootstrap' +_TN_REG_HOST='harbor.example.test' +_TN_REG_USER='robot$golden' +_TN_REG_PASS='TEST-ONLY-registry-password' +# A double-quoted YAML scalar: backslash and double quote escaped. +_tn_reg_q() { printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'; } +if [ -e "${_TN_REG_FILE}" ] && ! $SUDO grep -qxF "${_TN_REG_MARKER}" "${_TN_REG_FILE}"; then + warn "${_TN_REG_FILE} exists and was not written by tnctl bootstrap: keeping it, the registry mirror is not configured" +else + $SUDO mkdir -p "$(dirname "${_TN_REG_FILE}")" + _tn_reg_tmp=$(umask 077; mktemp "${_TN_REG_FILE}.XXXXXX") || fatal "Cannot create a temporary file next to ${_TN_REG_FILE}" + { + printf '%s\n' "${_TN_REG_MARKER}" + printf '# k3s private registry mirrors: pull-through cache on %s. containerd asks Harbor first and\n' "${_TN_REG_HOST}" + printf '# falls back to the upstream registry. Written by tnctl bootstrap --registry-mirror; do not edit.\n' + printf 'mirrors:\n' + for _tn_reg_m in docker.io=dockerhub ghcr.io=ghcr gcr.io=gcr quay.io=quay registry.k8s.io=k8s nvcr.io=nvcr; do + printf ' %s:\n endpoint:\n - "https://%s"\n rewrite:\n "^(.*)$": "%s/$1"\n' \ + "${_tn_reg_m%%=*}" "${_TN_REG_HOST}" "${_tn_reg_m#*=}" + done + printf 'configs:\n "%s":\n auth:\n username: "%s"\n password: "%s"\n' \ + "${_TN_REG_HOST}" "$(_tn_reg_q "${_TN_REG_USER}")" "$(_tn_reg_q "${_TN_REG_PASS}")" + } > "${_tn_reg_tmp}" || { rm -f "${_tn_reg_tmp}"; fatal "Cannot write ${_TN_REG_FILE}"; } + if [ -f "${_TN_REG_FILE}" ] && cmp -s "${_tn_reg_tmp}" "${_TN_REG_FILE}"; then + rm -f "${_tn_reg_tmp}" + $SUDO chmod 0600 "${_TN_REG_FILE}" + info "Registry mirror ${_TN_REG_HOST} already configured" + else + $SUDO chmod 0600 "${_tn_reg_tmp}" + $SUDO mv -f "${_tn_reg_tmp}" "${_TN_REG_FILE}" || { rm -f "${_tn_reg_tmp}"; fatal "Cannot write ${_TN_REG_FILE}"; } + info "Registry mirror ${_TN_REG_HOST} configured (${_TN_REG_FILE})" + fi + unset _tn_reg_tmp _tn_reg_m +fi +unset _TN_REG_USER _TN_REG_PASS download k3s_installer.sh https://get.k3s.io $SUDO chmod +x k3s_installer.sh diff --git a/pkg/templates/testdata/agent.roles.golden.sh b/pkg/templates/testdata/agent.roles.golden.sh index 796c214..6fabad4 100644 --- a/pkg/templates/testdata/agent.roles.golden.sh +++ b/pkg/templates/testdata/agent.roles.golden.sh @@ -906,6 +906,48 @@ $SUDO cp ${CONFIG_PATH}/rsa_public.pem /usr/local/share/ca-certificates/${_DEVIC $SUDO update-ca-certificates info "Install Rancher K3s Agent" +# The registry mirror (tnctl bootstrap --registry-mirror): pull images through the Harbor proxy-cache +# projects, with the upstream registry as the fallback (k3s always tries its default endpoint last, so +# a Harbor outage never blocks public images). It runs before the k3s installer, so a first start already +# uses it; a re-run restarts k3s through the installer. The file is written only when it differs, at +# mode 0600 (it holds the Harbor credential), and a registries.yaml that this script did not write is +# never touched. The credential is never printed. +_TN_REG_FILE="${K3S_CONFIG_PATH:-/etc/rancher/k3s}/registries.yaml" +_TN_REG_MARKER='# managed by tnctl bootstrap' +_TN_REG_HOST='harbor.example.test' +_TN_REG_USER='robot$golden' +_TN_REG_PASS='TEST-ONLY-registry-password' +# A double-quoted YAML scalar: backslash and double quote escaped. +_tn_reg_q() { printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'; } +if [ -e "${_TN_REG_FILE}" ] && ! $SUDO grep -qxF "${_TN_REG_MARKER}" "${_TN_REG_FILE}"; then + warn "${_TN_REG_FILE} exists and was not written by tnctl bootstrap: keeping it, the registry mirror is not configured" +else + $SUDO mkdir -p "$(dirname "${_TN_REG_FILE}")" + _tn_reg_tmp=$(umask 077; mktemp "${_TN_REG_FILE}.XXXXXX") || fatal "Cannot create a temporary file next to ${_TN_REG_FILE}" + { + printf '%s\n' "${_TN_REG_MARKER}" + printf '# k3s private registry mirrors: pull-through cache on %s. containerd asks Harbor first and\n' "${_TN_REG_HOST}" + printf '# falls back to the upstream registry. Written by tnctl bootstrap --registry-mirror; do not edit.\n' + printf 'mirrors:\n' + for _tn_reg_m in docker.io=dockerhub ghcr.io=ghcr gcr.io=gcr quay.io=quay registry.k8s.io=k8s nvcr.io=nvcr; do + printf ' %s:\n endpoint:\n - "https://%s"\n rewrite:\n "^(.*)$": "%s/$1"\n' \ + "${_tn_reg_m%%=*}" "${_TN_REG_HOST}" "${_tn_reg_m#*=}" + done + printf 'configs:\n "%s":\n auth:\n username: "%s"\n password: "%s"\n' \ + "${_TN_REG_HOST}" "$(_tn_reg_q "${_TN_REG_USER}")" "$(_tn_reg_q "${_TN_REG_PASS}")" + } > "${_tn_reg_tmp}" || { rm -f "${_tn_reg_tmp}"; fatal "Cannot write ${_TN_REG_FILE}"; } + if [ -f "${_TN_REG_FILE}" ] && cmp -s "${_tn_reg_tmp}" "${_TN_REG_FILE}"; then + rm -f "${_tn_reg_tmp}" + $SUDO chmod 0600 "${_TN_REG_FILE}" + info "Registry mirror ${_TN_REG_HOST} already configured" + else + $SUDO chmod 0600 "${_tn_reg_tmp}" + $SUDO mv -f "${_tn_reg_tmp}" "${_TN_REG_FILE}" || { rm -f "${_tn_reg_tmp}"; fatal "Cannot write ${_TN_REG_FILE}"; } + info "Registry mirror ${_TN_REG_HOST} configured (${_TN_REG_FILE})" + fi + unset _tn_reg_tmp _tn_reg_m +fi +unset _TN_REG_USER _TN_REG_PASS download k3s_installer.sh https://get.k3s.io $SUDO chmod +x k3s_installer.sh diff --git a/pkg/templates/testdata/server.golden.sh b/pkg/templates/testdata/server.golden.sh index c1a2ed7..485bc82 100644 --- a/pkg/templates/testdata/server.golden.sh +++ b/pkg/templates/testdata/server.golden.sh @@ -2006,6 +2006,48 @@ fs.inotify.max_user_instances=1024 EOF sysctl --system +# The registry mirror (tnctl bootstrap --registry-mirror): pull images through the Harbor proxy-cache +# projects, with the upstream registry as the fallback (k3s always tries its default endpoint last, so +# a Harbor outage never blocks public images). It runs before the k3s installer, so a first start already +# uses it; a re-run restarts k3s through the installer. The file is written only when it differs, at +# mode 0600 (it holds the Harbor credential), and a registries.yaml that this script did not write is +# never touched. The credential is never printed. +_TN_REG_FILE="${K3S_CONFIG_PATH:-/etc/rancher/k3s}/registries.yaml" +_TN_REG_MARKER='# managed by tnctl bootstrap' +_TN_REG_HOST='harbor.example.test' +_TN_REG_USER='robot$golden' +_TN_REG_PASS='TEST-ONLY-registry-password' +# A double-quoted YAML scalar: backslash and double quote escaped. +_tn_reg_q() { printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'; } +if [ -e "${_TN_REG_FILE}" ] && ! $SUDO grep -qxF "${_TN_REG_MARKER}" "${_TN_REG_FILE}"; then + warn "${_TN_REG_FILE} exists and was not written by tnctl bootstrap: keeping it, the registry mirror is not configured" +else + $SUDO mkdir -p "$(dirname "${_TN_REG_FILE}")" + _tn_reg_tmp=$(umask 077; mktemp "${_TN_REG_FILE}.XXXXXX") || fatal "Cannot create a temporary file next to ${_TN_REG_FILE}" + { + printf '%s\n' "${_TN_REG_MARKER}" + printf '# k3s private registry mirrors: pull-through cache on %s. containerd asks Harbor first and\n' "${_TN_REG_HOST}" + printf '# falls back to the upstream registry. Written by tnctl bootstrap --registry-mirror; do not edit.\n' + printf 'mirrors:\n' + for _tn_reg_m in docker.io=dockerhub ghcr.io=ghcr gcr.io=gcr quay.io=quay registry.k8s.io=k8s nvcr.io=nvcr; do + printf ' %s:\n endpoint:\n - "https://%s"\n rewrite:\n "^(.*)$": "%s/$1"\n' \ + "${_tn_reg_m%%=*}" "${_TN_REG_HOST}" "${_tn_reg_m#*=}" + done + printf 'configs:\n "%s":\n auth:\n username: "%s"\n password: "%s"\n' \ + "${_TN_REG_HOST}" "$(_tn_reg_q "${_TN_REG_USER}")" "$(_tn_reg_q "${_TN_REG_PASS}")" + } > "${_tn_reg_tmp}" || { rm -f "${_tn_reg_tmp}"; fatal "Cannot write ${_TN_REG_FILE}"; } + if [ -f "${_TN_REG_FILE}" ] && cmp -s "${_tn_reg_tmp}" "${_TN_REG_FILE}"; then + rm -f "${_tn_reg_tmp}" + $SUDO chmod 0600 "${_TN_REG_FILE}" + info "Registry mirror ${_TN_REG_HOST} already configured" + else + $SUDO chmod 0600 "${_tn_reg_tmp}" + $SUDO mv -f "${_tn_reg_tmp}" "${_TN_REG_FILE}" || { rm -f "${_tn_reg_tmp}"; fatal "Cannot write ${_TN_REG_FILE}"; } + info "Registry mirror ${_TN_REG_HOST} configured (${_TN_REG_FILE})" + fi + unset _tn_reg_tmp _tn_reg_m +fi +unset _TN_REG_USER _TN_REG_PASS download k3s_installer.sh https://get.k3s.io $SUDO chmod +x k3s_installer.sh diff --git a/pkg/templates/testdata/server.roles.golden.sh b/pkg/templates/testdata/server.roles.golden.sh index 076a8ec..2e34200 100644 --- a/pkg/templates/testdata/server.roles.golden.sh +++ b/pkg/templates/testdata/server.roles.golden.sh @@ -2159,6 +2159,48 @@ fs.inotify.max_user_instances=1024 EOF sysctl --system +# The registry mirror (tnctl bootstrap --registry-mirror): pull images through the Harbor proxy-cache +# projects, with the upstream registry as the fallback (k3s always tries its default endpoint last, so +# a Harbor outage never blocks public images). It runs before the k3s installer, so a first start already +# uses it; a re-run restarts k3s through the installer. The file is written only when it differs, at +# mode 0600 (it holds the Harbor credential), and a registries.yaml that this script did not write is +# never touched. The credential is never printed. +_TN_REG_FILE="${K3S_CONFIG_PATH:-/etc/rancher/k3s}/registries.yaml" +_TN_REG_MARKER='# managed by tnctl bootstrap' +_TN_REG_HOST='harbor.example.test' +_TN_REG_USER='robot$golden' +_TN_REG_PASS='TEST-ONLY-registry-password' +# A double-quoted YAML scalar: backslash and double quote escaped. +_tn_reg_q() { printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'; } +if [ -e "${_TN_REG_FILE}" ] && ! $SUDO grep -qxF "${_TN_REG_MARKER}" "${_TN_REG_FILE}"; then + warn "${_TN_REG_FILE} exists and was not written by tnctl bootstrap: keeping it, the registry mirror is not configured" +else + $SUDO mkdir -p "$(dirname "${_TN_REG_FILE}")" + _tn_reg_tmp=$(umask 077; mktemp "${_TN_REG_FILE}.XXXXXX") || fatal "Cannot create a temporary file next to ${_TN_REG_FILE}" + { + printf '%s\n' "${_TN_REG_MARKER}" + printf '# k3s private registry mirrors: pull-through cache on %s. containerd asks Harbor first and\n' "${_TN_REG_HOST}" + printf '# falls back to the upstream registry. Written by tnctl bootstrap --registry-mirror; do not edit.\n' + printf 'mirrors:\n' + for _tn_reg_m in docker.io=dockerhub ghcr.io=ghcr gcr.io=gcr quay.io=quay registry.k8s.io=k8s nvcr.io=nvcr; do + printf ' %s:\n endpoint:\n - "https://%s"\n rewrite:\n "^(.*)$": "%s/$1"\n' \ + "${_tn_reg_m%%=*}" "${_TN_REG_HOST}" "${_tn_reg_m#*=}" + done + printf 'configs:\n "%s":\n auth:\n username: "%s"\n password: "%s"\n' \ + "${_TN_REG_HOST}" "$(_tn_reg_q "${_TN_REG_USER}")" "$(_tn_reg_q "${_TN_REG_PASS}")" + } > "${_tn_reg_tmp}" || { rm -f "${_tn_reg_tmp}"; fatal "Cannot write ${_TN_REG_FILE}"; } + if [ -f "${_TN_REG_FILE}" ] && cmp -s "${_tn_reg_tmp}" "${_TN_REG_FILE}"; then + rm -f "${_tn_reg_tmp}" + $SUDO chmod 0600 "${_TN_REG_FILE}" + info "Registry mirror ${_TN_REG_HOST} already configured" + else + $SUDO chmod 0600 "${_tn_reg_tmp}" + $SUDO mv -f "${_tn_reg_tmp}" "${_TN_REG_FILE}" || { rm -f "${_tn_reg_tmp}"; fatal "Cannot write ${_TN_REG_FILE}"; } + info "Registry mirror ${_TN_REG_HOST} configured (${_TN_REG_FILE})" + fi + unset _tn_reg_tmp _tn_reg_m +fi +unset _TN_REG_USER _TN_REG_PASS download k3s_installer.sh https://get.k3s.io $SUDO chmod +x k3s_installer.sh From 52e9426009662ae7f656d5ea619c1fb5a4b9771e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20=C3=85slund?= Date: Thu, 22 Oct 2026 13:32:21 +0000 Subject: [PATCH 2/3] fix(bootstrap): accept the registry-mirror playbook's marker in registries.yaml A device configured by playbooks/registry-mirror.yml (# managed by teknoir) and later re-bootstrapped with --registry-mirror kept the old file with a possibly stale credential and only got a warning; --registry-mirror none did not remove it. Either marker is ours now. Co-Authored-By: Claude Opus 5.5 --- docs/commands/bootstrap.md | 6 +++-- pkg/templates/partials/registries.sh.tmpl | 6 +++-- pkg/templates/registries_test.go | 24 +++++++++++++++++++ pkg/templates/testdata/agent.golden.sh | 6 +++-- pkg/templates/testdata/agent.roles.golden.sh | 6 +++-- pkg/templates/testdata/server.golden.sh | 6 +++-- pkg/templates/testdata/server.roles.golden.sh | 6 +++-- 7 files changed, 48 insertions(+), 12 deletions(-) diff --git a/docs/commands/bootstrap.md b/docs/commands/bootstrap.md index 16d7e06..f12c84a 100644 --- a/docs/commands/bootstrap.md +++ b/docs/commands/bootstrap.md @@ -172,8 +172,10 @@ tnctl bootstrap edge-01 --registry-mirror harbor.teknoir.cloud --registry-creden - **What is written.** The server and agent scripts write `/etc/rancher/k3s/registries.yaml` (mode `0600`, first line `# managed by tnctl bootstrap`) before the k3s installer runs, so the first start already uses it. On a re-run the installer restarts k3s. The file is written only when it differs. -- **A file you wrote is never touched.** A `registries.yaml` without the marker line is kept, with a - warning, and the mirror is not configured. `--registry-mirror none` removes a file that the scripts wrote +- **A file you wrote is never touched.** A `registries.yaml` without the marker line (`# managed by tnctl + bootstrap`, or `# managed by teknoir`, which the Ansible playbook `registry-mirror.yml` writes) is kept, with + a warning, and the mirror is not configured. A file with either marker is replaced, so a re-bootstrapped + robot never keeps a stale credential. `--registry-mirror none` removes a file that the scripts wrote (and keeps yours). Without `--registry-mirror` the scripts leave the file alone. - **The credential** is a Harbor robot account that can only pull from the proxy projects. It is embedded in the scripts (like the device keys, so keep the scripts private) and the scripts never print it. The file diff --git a/pkg/templates/partials/registries.sh.tmpl b/pkg/templates/partials/registries.sh.tmpl index 4173564..c37690a 100644 --- a/pkg/templates/partials/registries.sh.tmpl +++ b/pkg/templates/partials/registries.sh.tmpl @@ -6,13 +6,15 @@ # never touched. The credential is never printed. _TN_REG_FILE="${K3S_CONFIG_PATH:-/etc/rancher/k3s}/registries.yaml" _TN_REG_MARKER='# managed by tnctl bootstrap' +# playbooks/registry-mirror.yml (ansible) marks its file differently; it writes the same content, so either is ours. +_TN_REG_MARKER_PB='# managed by teknoir' _TN_REG_HOST='{{.RegistryMirror}}' {{if ne .RegistryMirror "none"}}_TN_REG_USER='{{.RegistryUsername}}' _TN_REG_PASS='{{.RegistryPassword}}' # A double-quoted YAML scalar: backslash and double quote escaped. _tn_reg_q() { printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'; } -{{end}}if [ -e "${_TN_REG_FILE}" ] && ! $SUDO grep -qxF "${_TN_REG_MARKER}" "${_TN_REG_FILE}"; then - warn "${_TN_REG_FILE} exists and was not written by tnctl bootstrap: keeping it, the registry mirror is not configured" +{{end}}if [ -e "${_TN_REG_FILE}" ] && ! $SUDO grep -qxF -e "${_TN_REG_MARKER}" -e "${_TN_REG_MARKER_PB}" "${_TN_REG_FILE}"; then + warn "${_TN_REG_FILE} exists and was not written by tnctl bootstrap or registry-mirror.yml: keeping it, the registry mirror is not configured" {{- if eq .RegistryMirror "none"}} elif [ -e "${_TN_REG_FILE}" ]; then $SUDO rm -f "${_TN_REG_FILE}" diff --git a/pkg/templates/registries_test.go b/pkg/templates/registries_test.go index 584d85d..f3f8f0f 100644 --- a/pkg/templates/registries_test.go +++ b/pkg/templates/registries_test.go @@ -221,6 +221,30 @@ func TestRegistries_ForeignFileIsKept(t *testing.T) { } } +// playbooks/registry-mirror.yml (ansible) writes the same content under its own first line; tnctl treats +// that file as its own: it replaces it (a stale credential must not stay) and `none` removes it. +func TestRegistries_AcceptsThePlaybooksMarker(t *testing.T) { + const pbMarker = "# managed by teknoir" + dir := t.TempDir() + p := filepath.Join(dir, "registries.yaml") + if err := os.WriteFile(p, []byte(pbMarker+"\nmirrors: {}\nconfigs:\n \"old.example\":\n auth:\n password: \"stale\"\n"), 0o600); err != nil { + t.Fatal(err) + } + r := runRegistries(t, dir, regData("harbor.example.test")).mustOK(t) + got := readFile(t, p) + if strings.Contains(r.output, "[WARN]") || strings.Contains(got, "stale") || !strings.HasPrefix(got, regMarker+"\n") || !strings.Contains(got, regPassword) { + t.Errorf("the playbook's file was not replaced:\n%s\n%s", r.output, got) + } + if err := os.WriteFile(p, []byte(pbMarker+"\nmirrors: {}\n"), 0o600); err != nil { + t.Fatal(err) + } + r = runRegistries(t, dir, regData("none")).mustOK(t) + if _, err := os.Stat(p); !os.IsNotExist(err) || !strings.Contains(r.output, "Registry mirror removed") { + t.Errorf("none did not remove the playbook's file: %v\n%s", err, r.output) + } + noPassword(t, "output", r.output) +} + func TestRegistries_NoneRemovesAManagedFile(t *testing.T) { dir := t.TempDir() r := runRegistries(t, dir, regData("harbor.example.test")).mustOK(t) diff --git a/pkg/templates/testdata/agent.golden.sh b/pkg/templates/testdata/agent.golden.sh index 5a89abf..8bf3669 100644 --- a/pkg/templates/testdata/agent.golden.sh +++ b/pkg/templates/testdata/agent.golden.sh @@ -762,13 +762,15 @@ info "Install Rancher K3s Agent" # never touched. The credential is never printed. _TN_REG_FILE="${K3S_CONFIG_PATH:-/etc/rancher/k3s}/registries.yaml" _TN_REG_MARKER='# managed by tnctl bootstrap' +# playbooks/registry-mirror.yml (ansible) marks its file differently; it writes the same content, so either is ours. +_TN_REG_MARKER_PB='# managed by teknoir' _TN_REG_HOST='harbor.example.test' _TN_REG_USER='robot$golden' _TN_REG_PASS='TEST-ONLY-registry-password' # A double-quoted YAML scalar: backslash and double quote escaped. _tn_reg_q() { printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'; } -if [ -e "${_TN_REG_FILE}" ] && ! $SUDO grep -qxF "${_TN_REG_MARKER}" "${_TN_REG_FILE}"; then - warn "${_TN_REG_FILE} exists and was not written by tnctl bootstrap: keeping it, the registry mirror is not configured" +if [ -e "${_TN_REG_FILE}" ] && ! $SUDO grep -qxF -e "${_TN_REG_MARKER}" -e "${_TN_REG_MARKER_PB}" "${_TN_REG_FILE}"; then + warn "${_TN_REG_FILE} exists and was not written by tnctl bootstrap or registry-mirror.yml: keeping it, the registry mirror is not configured" else $SUDO mkdir -p "$(dirname "${_TN_REG_FILE}")" _tn_reg_tmp=$(umask 077; mktemp "${_TN_REG_FILE}.XXXXXX") || fatal "Cannot create a temporary file next to ${_TN_REG_FILE}" diff --git a/pkg/templates/testdata/agent.roles.golden.sh b/pkg/templates/testdata/agent.roles.golden.sh index 6fabad4..957f0d0 100644 --- a/pkg/templates/testdata/agent.roles.golden.sh +++ b/pkg/templates/testdata/agent.roles.golden.sh @@ -914,13 +914,15 @@ info "Install Rancher K3s Agent" # never touched. The credential is never printed. _TN_REG_FILE="${K3S_CONFIG_PATH:-/etc/rancher/k3s}/registries.yaml" _TN_REG_MARKER='# managed by tnctl bootstrap' +# playbooks/registry-mirror.yml (ansible) marks its file differently; it writes the same content, so either is ours. +_TN_REG_MARKER_PB='# managed by teknoir' _TN_REG_HOST='harbor.example.test' _TN_REG_USER='robot$golden' _TN_REG_PASS='TEST-ONLY-registry-password' # A double-quoted YAML scalar: backslash and double quote escaped. _tn_reg_q() { printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'; } -if [ -e "${_TN_REG_FILE}" ] && ! $SUDO grep -qxF "${_TN_REG_MARKER}" "${_TN_REG_FILE}"; then - warn "${_TN_REG_FILE} exists and was not written by tnctl bootstrap: keeping it, the registry mirror is not configured" +if [ -e "${_TN_REG_FILE}" ] && ! $SUDO grep -qxF -e "${_TN_REG_MARKER}" -e "${_TN_REG_MARKER_PB}" "${_TN_REG_FILE}"; then + warn "${_TN_REG_FILE} exists and was not written by tnctl bootstrap or registry-mirror.yml: keeping it, the registry mirror is not configured" else $SUDO mkdir -p "$(dirname "${_TN_REG_FILE}")" _tn_reg_tmp=$(umask 077; mktemp "${_TN_REG_FILE}.XXXXXX") || fatal "Cannot create a temporary file next to ${_TN_REG_FILE}" diff --git a/pkg/templates/testdata/server.golden.sh b/pkg/templates/testdata/server.golden.sh index 485bc82..a30c50e 100644 --- a/pkg/templates/testdata/server.golden.sh +++ b/pkg/templates/testdata/server.golden.sh @@ -2014,13 +2014,15 @@ sysctl --system # never touched. The credential is never printed. _TN_REG_FILE="${K3S_CONFIG_PATH:-/etc/rancher/k3s}/registries.yaml" _TN_REG_MARKER='# managed by tnctl bootstrap' +# playbooks/registry-mirror.yml (ansible) marks its file differently; it writes the same content, so either is ours. +_TN_REG_MARKER_PB='# managed by teknoir' _TN_REG_HOST='harbor.example.test' _TN_REG_USER='robot$golden' _TN_REG_PASS='TEST-ONLY-registry-password' # A double-quoted YAML scalar: backslash and double quote escaped. _tn_reg_q() { printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'; } -if [ -e "${_TN_REG_FILE}" ] && ! $SUDO grep -qxF "${_TN_REG_MARKER}" "${_TN_REG_FILE}"; then - warn "${_TN_REG_FILE} exists and was not written by tnctl bootstrap: keeping it, the registry mirror is not configured" +if [ -e "${_TN_REG_FILE}" ] && ! $SUDO grep -qxF -e "${_TN_REG_MARKER}" -e "${_TN_REG_MARKER_PB}" "${_TN_REG_FILE}"; then + warn "${_TN_REG_FILE} exists and was not written by tnctl bootstrap or registry-mirror.yml: keeping it, the registry mirror is not configured" else $SUDO mkdir -p "$(dirname "${_TN_REG_FILE}")" _tn_reg_tmp=$(umask 077; mktemp "${_TN_REG_FILE}.XXXXXX") || fatal "Cannot create a temporary file next to ${_TN_REG_FILE}" diff --git a/pkg/templates/testdata/server.roles.golden.sh b/pkg/templates/testdata/server.roles.golden.sh index 2e34200..36d35a3 100644 --- a/pkg/templates/testdata/server.roles.golden.sh +++ b/pkg/templates/testdata/server.roles.golden.sh @@ -2167,13 +2167,15 @@ sysctl --system # never touched. The credential is never printed. _TN_REG_FILE="${K3S_CONFIG_PATH:-/etc/rancher/k3s}/registries.yaml" _TN_REG_MARKER='# managed by tnctl bootstrap' +# playbooks/registry-mirror.yml (ansible) marks its file differently; it writes the same content, so either is ours. +_TN_REG_MARKER_PB='# managed by teknoir' _TN_REG_HOST='harbor.example.test' _TN_REG_USER='robot$golden' _TN_REG_PASS='TEST-ONLY-registry-password' # A double-quoted YAML scalar: backslash and double quote escaped. _tn_reg_q() { printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'; } -if [ -e "${_TN_REG_FILE}" ] && ! $SUDO grep -qxF "${_TN_REG_MARKER}" "${_TN_REG_FILE}"; then - warn "${_TN_REG_FILE} exists and was not written by tnctl bootstrap: keeping it, the registry mirror is not configured" +if [ -e "${_TN_REG_FILE}" ] && ! $SUDO grep -qxF -e "${_TN_REG_MARKER}" -e "${_TN_REG_MARKER_PB}" "${_TN_REG_FILE}"; then + warn "${_TN_REG_FILE} exists and was not written by tnctl bootstrap or registry-mirror.yml: keeping it, the registry mirror is not configured" else $SUDO mkdir -p "$(dirname "${_TN_REG_FILE}")" _tn_reg_tmp=$(umask 077; mktemp "${_TN_REG_FILE}.XXXXXX") || fatal "Cannot create a temporary file next to ${_TN_REG_FILE}" From 294e07a46bbd6b1dfc485a3bf114a6e58fe5f134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20=C3=85slund?= Date: Thu, 22 Oct 2026 16:13:38 +0000 Subject: [PATCH 3/3] fix(bootstrap): write the bootstrap scripts with mode 0600, they hold the Harbor robot password Co-Authored-By: Claude Opus 5.5 --- cmd/bootstrap.go | 24 ++++++++++++------- cmd/bootstrap_perm_test.go | 48 ++++++++++++++++++++++++++++++++++++++ docs/commands/bootstrap.md | 3 ++- 3 files changed, 66 insertions(+), 9 deletions(-) create mode 100644 cmd/bootstrap_perm_test.go diff --git a/cmd/bootstrap.go b/cmd/bootstrap.go index 2e9d9ba..daccb45 100644 --- a/cmd/bootstrap.go +++ b/cmd/bootstrap.go @@ -103,14 +103,7 @@ var bootstrapCmd = &cobra.Command{ serverData, agentData := nodes.apply(data) - generate := func(tmpl *template.Template, filename string, data templates.TemplateData) error { - f, err := os.Create(filename) - if err != nil { - return err - } - defer f.Close() - return tmpl.Execute(f, data) - } + generate := writeBootstrapScript agentFile := fmt.Sprintf("bootstrap_agent_%s.sh", deviceName) serverFile := fmt.Sprintf("bootstrap_server_%s.sh", deviceName) @@ -221,3 +214,18 @@ func init() { "file with the Harbor credential for --registry-mirror, one line username:password (a robot account of the teamspace), readable by you only. The password goes into the generated scripts and is never printed") rootCmd.AddCommand(bootstrapCmd) } + +// writeBootstrapScript renders a bootstrap script to filename with mode 0600: the scripts hold the device +// password, the k3s token and the teamspace's Harbor robot password. A file that already exists (written with +// 0644 by an earlier tnctl) is chmodded too, because OpenFile's mode applies to new files only. +func writeBootstrapScript(tmpl *template.Template, filename string, data templates.TemplateData) error { + f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + if err != nil { + return err + } + defer f.Close() + if err := f.Chmod(0o600); err != nil { + return err + } + return tmpl.Execute(f, data) +} diff --git a/cmd/bootstrap_perm_test.go b/cmd/bootstrap_perm_test.go new file mode 100644 index 0000000..6f9a3a6 --- /dev/null +++ b/cmd/bootstrap_perm_test.go @@ -0,0 +1,48 @@ +package cmd + +import ( + "os" + "path/filepath" + "runtime" + "testing" + "text/template" + + "github.com/teknoir/cli/pkg/templates" +) + +// The bootstrap scripts hold the teamspace's Harbor robot password: new files are 0600, and a file an earlier +// tnctl wrote with 0644 is tightened and truncated, not appended to. +func TestWriteBootstrapScriptIsPrivate(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX modes") + } + tmpl := template.Must(template.New("t").Parse("password={{.RegistryPassword}}\n")) + data := templates.TemplateData{RegistryPassword: "s3cret"} + dir := t.TempDir() + + fresh := filepath.Join(dir, "fresh.sh") + if err := writeBootstrapScript(tmpl, fresh, data); err != nil { + t.Fatal(err) + } + if fi, err := os.Stat(fresh); err != nil || fi.Mode().Perm() != 0o600 { + t.Errorf("fresh file: %v %v, want 0600", fi.Mode().Perm(), err) + } + + old := filepath.Join(dir, "old.sh") + if err := os.WriteFile(old, []byte("an older, much longer script body that must not survive\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Chmod(old, 0o644); err != nil { // umask may have cleared bits + t.Fatal(err) + } + if err := writeBootstrapScript(tmpl, old, data); err != nil { + t.Fatal(err) + } + fi, err := os.Stat(old) + if err != nil || fi.Mode().Perm() != 0o600 { + t.Errorf("overwritten file: %v %v, want 0600", fi.Mode().Perm(), err) + } + if b, _ := os.ReadFile(old); string(b) != "password=s3cret\n" { + t.Errorf("content %q", b) + } +} diff --git a/docs/commands/bootstrap.md b/docs/commands/bootstrap.md index f12c84a..66d213e 100644 --- a/docs/commands/bootstrap.md +++ b/docs/commands/bootstrap.md @@ -178,7 +178,8 @@ tnctl bootstrap edge-01 --registry-mirror harbor.teknoir.cloud --registry-creden robot never keeps a stale credential. `--registry-mirror none` removes a file that the scripts wrote (and keeps yours). Without `--registry-mirror` the scripts leave the file alone. - **The credential** is a Harbor robot account that can only pull from the proxy projects. It is embedded in - the scripts (like the device keys, so keep the scripts private) and the scripts never print it. The file + the scripts (like the device keys, so keep the scripts private) and the scripts never print it. `tnctl bootstrap` writes the + scripts with mode `0600` (and tightens a script that an earlier run left at `0644`); the file on the robot must not be readable by group or others. - The Docker-mode script (`bootstrap_docker_server_.sh`) is not changed. - For a robot that is already running, the Ansible playbook `registry-mirror.yml` (repository `ansible`)