diff --git a/cmd/bootstrap.go b/cmd/bootstrap.go index 298ce94..daccb45 100644 --- a/cmd/bootstrap.go +++ b/cmd/bootstrap.go @@ -23,6 +23,12 @@ var bootstrapCmd = &cobra.Command{ if err := templates.ValidateK3SVersion(k3sVersion); err != nil { return err } + registryMirror, _ := cmd.Flags().GetString("registry-mirror") + registryCredFile, _ := cmd.Flags().GetString("registry-credential-file") + // Fail on a bad mirror or an unusable credential file before the platform is asked for anything. + if err := checkRegistryFlags(registryMirror, registryCredFile); err != nil { + return err + } nodeOpts, err := nodeOptionsFromFlags(cmd) if err != nil { return err @@ -66,6 +72,11 @@ var bootstrapCmd = &cobra.Command{ return fmt.Errorf("--force only applies together with --k3s-upgrade") } + registry, err := registryFromFlags(registryMirror, registryCredFile, bl.registry) + if err != nil { + return err + } + data := templates.TemplateData{ Domain: domain, Namespace: namespace, @@ -81,6 +92,9 @@ var bootstrapCmd = &cobra.Command{ K3SVersionExplicit: cmd.Flags().Changed("k3s-version"), K3SUpgrade: k3sUpgrade, K3SForce: force, + RegistryMirror: registry.Mirror, + RegistryUsername: registry.Username, + RegistryPassword: registry.Password, } if err := templates.ValidateTemplateData(data); err != nil { @@ -89,14 +103,7 @@ var bootstrapCmd = &cobra.Command{ serverData, agentData := nodes.apply(data) - generate := func(tmpl *template.Template, filename string, data templates.TemplateData) error { - f, err := os.Create(filename) - if err != nil { - return err - } - defer f.Close() - return tmpl.Execute(f, data) - } + generate := writeBootstrapScript agentFile := fmt.Sprintf("bootstrap_agent_%s.sh", deviceName) serverFile := fmt.Sprintf("bootstrap_server_%s.sh", deviceName) @@ -117,6 +124,13 @@ var bootstrapCmd = &cobra.Command{ fmt.Printf("- %s\n", serverFile) fmt.Printf("- %s\n", dockerServerFile) fmt.Print(nodes.describe(agentFile, serverFile)) + switch registry.Mirror { + case "": + case templates.RegistryMirrorNone: + fmt.Printf("%s and %s remove a registries.yaml that an earlier script wrote (registry mirror none)\n", agentFile, serverFile) + default: + fmt.Printf("%s and %s configure the registry mirror %s (/etc/rancher/k3s/registries.yaml); they contain its credential, keep them private\n", agentFile, serverFile, registry.Mirror) + } skipUpload, _ := cmd.Flags().GetBool("skip-upload") if !skipUpload { @@ -130,6 +144,8 @@ var bootstrapCmd = &cobra.Command{ // bootstrapLogin is the decoded login of a device that the bootstrap scripts install. type bootstrapLogin struct { rsaPrivate, rsaPublic, username, password, publicSSHKey string + // registry is the registry mirror the platform offers the device, nil when it sends none. + registry *api.FleetRegistry } // loadBootstrapLogin asks the platform for the login of a device and decodes it. The platform gives @@ -170,6 +186,7 @@ func loadBootstrapLogin(ctx context.Context, fl *api.Fleet, namespace, deviceNam if bl.publicSSHKey, err = decode("publicsshkey", c.PublicSSHKey); err != nil { return nil, err } + bl.registry = c.Registry return &bl, nil } @@ -191,5 +208,24 @@ func init() { "with --k3s-upgrade: allow an upgrade over more than one minor version (TN_K3S_FORCE=true is the default in the scripts)") addNodeFlags(bootstrapCmd) bootstrapCmd.Flags().String("docker-secret", "", "The contents of the .dockerconfigjson") + bootstrapCmd.Flags().String("registry-mirror", "", + "pull images through this Harbor host (pull-through cache projects dockerhub, ghcr, gcr, quay, k8s, nvcr), with the upstream registry as the fallback: the scripts write /etc/rancher/k3s/registries.yaml before the k3s installer. \"none\" removes a file the scripts wrote. Without it the file is left alone (or follows the registry that the platform sends with the device credentials, when it sends one)") + bootstrapCmd.Flags().String("registry-credential-file", "", + "file with the Harbor credential for --registry-mirror, one line username:password (a robot account of the teamspace), readable by you only. The password goes into the generated scripts and is never printed") rootCmd.AddCommand(bootstrapCmd) } + +// writeBootstrapScript renders a bootstrap script to filename with mode 0600: the scripts hold the device +// password, the k3s token and the teamspace's Harbor robot password. A file that already exists (written with +// 0644 by an earlier tnctl) is chmodded too, because OpenFile's mode applies to new files only. +func writeBootstrapScript(tmpl *template.Template, filename string, data templates.TemplateData) error { + f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + if err != nil { + return err + } + defer f.Close() + if err := f.Chmod(0o600); err != nil { + return err + } + return tmpl.Execute(f, data) +} diff --git a/cmd/bootstrap_perm_test.go b/cmd/bootstrap_perm_test.go new file mode 100644 index 0000000..6f9a3a6 --- /dev/null +++ b/cmd/bootstrap_perm_test.go @@ -0,0 +1,48 @@ +package cmd + +import ( + "os" + "path/filepath" + "runtime" + "testing" + "text/template" + + "github.com/teknoir/cli/pkg/templates" +) + +// The bootstrap scripts hold the teamspace's Harbor robot password: new files are 0600, and a file an earlier +// tnctl wrote with 0644 is tightened and truncated, not appended to. +func TestWriteBootstrapScriptIsPrivate(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX modes") + } + tmpl := template.Must(template.New("t").Parse("password={{.RegistryPassword}}\n")) + data := templates.TemplateData{RegistryPassword: "s3cret"} + dir := t.TempDir() + + fresh := filepath.Join(dir, "fresh.sh") + if err := writeBootstrapScript(tmpl, fresh, data); err != nil { + t.Fatal(err) + } + if fi, err := os.Stat(fresh); err != nil || fi.Mode().Perm() != 0o600 { + t.Errorf("fresh file: %v %v, want 0600", fi.Mode().Perm(), err) + } + + old := filepath.Join(dir, "old.sh") + if err := os.WriteFile(old, []byte("an older, much longer script body that must not survive\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Chmod(old, 0o644); err != nil { // umask may have cleared bits + t.Fatal(err) + } + if err := writeBootstrapScript(tmpl, old, data); err != nil { + t.Fatal(err) + } + fi, err := os.Stat(old) + if err != nil || fi.Mode().Perm() != 0o600 { + t.Errorf("overwritten file: %v %v, want 0600", fi.Mode().Perm(), err) + } + if b, _ := os.ReadFile(old); string(b) != "password=s3cret\n" { + t.Errorf("content %q", b) + } +} diff --git a/cmd/bootstrap_registry.go b/cmd/bootstrap_registry.go new file mode 100644 index 0000000..803ae20 --- /dev/null +++ b/cmd/bootstrap_registry.go @@ -0,0 +1,116 @@ +package cmd + +import ( + "fmt" + "os" + "runtime" + "strings" + + "github.com/teknoir/cli/pkg/api" + "github.com/teknoir/cli/pkg/templates" +) + +// registrySettings is what `tnctl bootstrap` puts into TemplateData for the registry mirror. +type registrySettings struct { + Mirror, Username, Password string +} + +// registryFromFlags decides the registry mirror of the generated scripts (plans/harbor-proxy-staging.md): +// +// - --registry-mirror none removes a registries.yaml that an earlier script wrote; no credential; +// - --registry-mirror HOST uses HOST with the credential of --registry-credential-file +// (one line username:password), or the platform's credential when the device's credentials +// response names the same host; +// - no --registry-mirror: the platform's registry of the device credentials response when it sends +// one (not yet), otherwise nothing: registries.yaml is left alone. +// +// Errors never contain the credential. +func registryFromFlags(mirrorFlag, credFile string, platform *api.FleetRegistry) (registrySettings, error) { + mirror := strings.TrimSpace(mirrorFlag) + if err := templates.ValidateRegistryMirror(mirror); err != nil { + return registrySettings{}, err + } + fromPlatform := false + if mirror == "" && platform != nil && platform.Host != "" { + mirror = platform.Host + fromPlatform = true + if err := templates.ValidateRegistryMirror(mirror); err != nil { + return registrySettings{}, fmt.Errorf("the registry mirror that the platform sent is unusable: %w", err) + } + } + switch mirror { + case "": + if credFile != "" { + return registrySettings{}, fmt.Errorf("--registry-credential-file needs --registry-mirror HOST") + } + return registrySettings{}, nil + case templates.RegistryMirrorNone: + return registrySettings{Mirror: mirror}, nil + } + s := registrySettings{Mirror: mirror} + switch { + case credFile != "": + user, pass, err := readRegistryCredentialFile(credFile) + if err != nil { + return registrySettings{}, err + } + s.Username, s.Password = user, pass + case platform != nil && platform.Host == mirror && platform.Username != "" && platform.Password != "": + s.Username, s.Password = platform.Username, platform.Password + default: + what := "--registry-mirror " + mirror + if fromPlatform { + what = "the registry mirror " + mirror + " that the platform sent" + } + return registrySettings{}, fmt.Errorf("%s needs a credential: pass --registry-credential-file FILE (one line username:password; the Harbor proxy projects are private)", what) + } + return s, nil +} + +// checkRegistryFlags checks the two flags on their own, before the platform is asked for anything: the +// mirror syntax and, when given, the credential file. Whether a mirror and a credential are both there +// is decided by registryFromFlags once the platform's answer is known. +func checkRegistryFlags(mirrorFlag, credFile string) error { + if err := templates.ValidateRegistryMirror(strings.TrimSpace(mirrorFlag)); err != nil { + return err + } + if credFile != "" { + if _, _, err := readRegistryCredentialFile(credFile); err != nil { + return err + } + } + return nil +} + +// readRegistryCredentialFile reads `username:password` from the first line of path. Where file modes +// mean something, a file that group or others can read is refused: it holds a password. The password +// is never part of an error. +func readRegistryCredentialFile(path string) (user, pass string, err error) { + fi, err := os.Stat(path) + if err != nil { + return "", "", fmt.Errorf("--registry-credential-file: %w", err) + } + if !fi.Mode().IsRegular() { + return "", "", fmt.Errorf("--registry-credential-file %s is not a regular file", path) + } + if runtime.GOOS != "windows" && fi.Mode().Perm()&0o077 != 0 { + return "", "", fmt.Errorf("--registry-credential-file %s has mode %04o: it holds a password, make it readable by you only (chmod 600)", path, fi.Mode().Perm()) + } + b, err := os.ReadFile(path) + if err != nil { + return "", "", fmt.Errorf("--registry-credential-file: %w", err) + } + if len(strings.TrimSpace(string(b))) == 0 { + return "", "", fmt.Errorf("--registry-credential-file %s is empty", path) + } + lines := strings.Split(strings.TrimRight(string(b), "\r\n"), "\n") + if len(lines) != 1 { + return "", "", fmt.Errorf("--registry-credential-file %s must hold exactly one line, username:password", path) + } + line := strings.TrimSuffix(lines[0], "\r") + user, pass, ok := strings.Cut(line, ":") + if !ok || user == "" || pass == "" { + return "", "", fmt.Errorf("--registry-credential-file %s must hold one line username:password", path) + } + return user, pass, nil +} diff --git a/cmd/bootstrap_registry_test.go b/cmd/bootstrap_registry_test.go new file mode 100644 index 0000000..47a5b14 --- /dev/null +++ b/cmd/bootstrap_registry_test.go @@ -0,0 +1,185 @@ +package cmd + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/teknoir/cli/pkg/api" + "github.com/teknoir/cli/pkg/templates" +) + +const testRegPassword = "S3cretRobotPassw0rdFORtest" + +func credFile(t *testing.T, content string, mode os.FileMode) string { + t.Helper() + p := filepath.Join(t.TempDir(), "cred") + if err := os.WriteFile(p, []byte(content), mode); err != nil { + t.Fatal(err) + } + if err := os.Chmod(p, mode); err != nil { + t.Fatal(err) + } + return p +} + +func TestBootstrapRegistryFlagsExist(t *testing.T) { + for _, name := range []string{"registry-mirror", "registry-credential-file"} { + f := bootstrapCmd.Flags().Lookup(name) + if f == nil { + t.Fatalf("no --%s flag", name) + } + if f.DefValue != "" { + t.Errorf("--%s defaults to %q: without it registries.yaml must be left alone", name, f.DefValue) + } + } +} + +func TestRegistryFromFlags_NothingGivenChangesNothing(t *testing.T) { + s, err := registryFromFlags("", "", nil) + if err != nil || s != (registrySettings{}) { + t.Fatalf("%+v %v", s, err) + } + // a platform without a registry field, or with an empty one, changes nothing either + for _, p := range []*api.FleetRegistry{nil, {}} { + if s, err := registryFromFlags("", "", p); err != nil || s != (registrySettings{}) { + t.Fatalf("%+v %v", s, err) + } + } +} + +func TestRegistryFromFlags_HostWithCredentialFile(t *testing.T) { + f := credFile(t, "robot$nodes:"+testRegPassword+"\n", 0o600) + s, err := registryFromFlags("harbor.teknoir.cloud", f, nil) + if err != nil { + t.Fatal(err) + } + if s != (registrySettings{Mirror: "harbor.teknoir.cloud", Username: "robot$nodes", Password: testRegPassword}) { + t.Errorf("%+v", s) + } + // CRLF files and a colon inside the password + f = credFile(t, "u:pa:ss\r\n", 0o600) + s, err = registryFromFlags("harbor.teknoir.cloud", f, nil) + if err != nil || s.Username != "u" || s.Password != "pa:ss" { + t.Errorf("%+v %v", s, err) + } +} + +func TestRegistryFromFlags_None(t *testing.T) { + s, err := registryFromFlags("none", "", nil) + if err != nil || s != (registrySettings{Mirror: "none"}) { + t.Fatalf("%+v %v", s, err) + } + // none carries no credential, even when the platform sends one + s, err = registryFromFlags("none", "", &api.FleetRegistry{Host: "h.example", Username: "u", Password: "p"}) + if err != nil || s != (registrySettings{Mirror: "none"}) { + t.Fatalf("%+v %v", s, err) + } +} + +func TestRegistryFromFlags_PlatformDefault(t *testing.T) { + p := &api.FleetRegistry{Host: "harbor.example.test", Username: "robot$ts-demo", Password: testRegPassword} + s, err := registryFromFlags("", "", p) + if err != nil || s != (registrySettings{Mirror: "harbor.example.test", Username: "robot$ts-demo", Password: testRegPassword}) { + t.Fatalf("%+v %v", s, err) + } + // the same host given explicitly takes the platform's credential + s, err = registryFromFlags("harbor.example.test", "", p) + if err != nil || s.Password != testRegPassword { + t.Fatalf("%+v %v", s, err) + } + // a file wins over the platform's credential + f := credFile(t, "mine:filepw\n", 0o600) + s, err = registryFromFlags("", f, p) + if err != nil || s.Username != "mine" || s.Mirror != "harbor.example.test" { + t.Fatalf("%+v %v", s, err) + } + // another host explicitly: the platform's credential is for its own host only + if _, err = registryFromFlags("other.example.test", "", p); err == nil { + t.Fatal("the platform's credential must not be sent to another host") + } + // the platform's mirror without a credential still needs one + if _, err = registryFromFlags("", "", &api.FleetRegistry{Host: "harbor.example.test"}); err == nil || !strings.Contains(err.Error(), "that the platform sent") { + t.Fatalf("%v", err) + } + // an unusable host from the platform is refused, not rendered + if _, err = registryFromFlags("", "", &api.FleetRegistry{Host: "bad host", Username: "u", Password: "p"}); err == nil { + t.Fatal("a bad host from the platform was accepted") + } +} + +func TestRegistryFromFlags_Rejects(t *testing.T) { + good := credFile(t, "u:"+testRegPassword+"\n", 0o600) + cases := map[string]struct { + mirror, file string + }{ + "host without a credential": {"harbor.example.test", ""}, + "credential without a host": {"", good}, + "invalid host": {"bad host", good}, + "host with scheme": {"https://harbor.example.test", good}, + "host with injection": {"a$(id).example", good}, + "missing file": {"harbor.example.test", filepath.Join(t.TempDir(), "nope")}, + "empty file": {"harbor.example.test", credFile(t, "", 0o600)}, + "blank file": {"harbor.example.test", credFile(t, "\n\n", 0o600)}, + "no colon": {"harbor.example.test", credFile(t, testRegPassword+"\n", 0o600)}, + "no user": {"harbor.example.test", credFile(t, ":"+testRegPassword+"\n", 0o600)}, + "no password": {"harbor.example.test", credFile(t, "user:\n", 0o600)}, + "two lines": {"harbor.example.test", credFile(t, "u:"+testRegPassword+"\nv:w\n", 0o600)}, + "directory": {"harbor.example.test", t.TempDir()}, + } + if runtime.GOOS != "windows" { + cases["group readable"] = struct{ mirror, file string }{"harbor.example.test", credFile(t, "u:"+testRegPassword+"\n", 0o640)} + cases["world readable"] = struct{ mirror, file string }{"harbor.example.test", credFile(t, "u:"+testRegPassword+"\n", 0o644)} + } + for name, c := range cases { + s, err := registryFromFlags(c.mirror, c.file, nil) + if err == nil { + t.Errorf("%s: accepted (%+v)", name, s) + continue + } + if strings.Contains(err.Error(), testRegPassword) { + t.Errorf("%s: the error contains the password: %v", name, err) + } + } +} + +func TestCheckRegistryFlags(t *testing.T) { + if err := checkRegistryFlags("", ""); err != nil { + t.Fatal(err) + } + // a credential file without a host is fine here: the platform may still name the host + if err := checkRegistryFlags("", credFile(t, "u:p\n", 0o600)); err != nil { + t.Fatal(err) + } + if err := checkRegistryFlags("bad host", ""); err == nil { + t.Fatal("bad host accepted") + } + if err := checkRegistryFlags("harbor.example.test", filepath.Join(t.TempDir(), "nope")); err == nil { + t.Fatal("missing file accepted") + } +} + +// What registryFromFlags returns renders into a script that ValidateTemplateData accepts, and the +// robot$ user name needs no escaping by the user. +func TestRegistrySettingsRender(t *testing.T) { + f := credFile(t, "robot$nodes:"+testRegPassword+"\n", 0o600) + s, err := registryFromFlags("harbor.teknoir.cloud", f, nil) + if err != nil { + t.Fatal(err) + } + d := templates.TemplateData{Domain: "example.test", Namespace: "ns", DeviceID: "dev-1", K3SToken: "tok", + RegistryMirror: s.Mirror, RegistryUsername: s.Username, RegistryPassword: s.Password} + for name, render := range map[string]func() ([]byte, error){ + "server": func() ([]byte, error) { return templates.RenderServer(d) }, + } { + out, err := render() + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if !strings.Contains(string(out), "_TN_REG_HOST='harbor.teknoir.cloud'") || !strings.Contains(string(out), "_TN_REG_USER='robot$nodes'") { + t.Errorf("%s: the registry block is missing", name) + } + } +} diff --git a/docs/commands/bootstrap.md b/docs/commands/bootstrap.md index 4b741a1..66d213e 100644 --- a/docs/commands/bootstrap.md +++ b/docs/commands/bootstrap.md @@ -19,6 +19,8 @@ The device may be given as an argument, via the `--device` global flag, or selec | `--k3s-upgrade` | `false` | Let the scripts replace an installed k3s with `--k3s-version` (sets `TN_K3S_UPGRADE=true` as the script's default). Without it a re-run **keeps** the installed k3s. See [Re-running a script](#re-running-a-script-the-installed-k3s-is-kept). | | `--force` | `false` | Only with `--k3s-upgrade`: allow an upgrade over more than one minor version (`TN_K3S_FORCE=true`). A downgrade is never allowed. | | `--docker-secret` | `""` | Contents of the `.dockerconfigjson` to embed. | +| `--registry-mirror HOST\|none` | `""` | Pull images through the Harbor host `HOST` (its proxy-cache projects), with the upstream registry as the fallback. The server and agent scripts write `/etc/rancher/k3s/registries.yaml` before the k3s installer. `none` removes a file the scripts wrote. Empty leaves `registries.yaml` alone. See [Registry mirror](#registry-mirror-harbor-pull-through-cache). | +| `--registry-credential-file FILE` | `""` | The Harbor credential for `--registry-mirror`: one line `username:password` (a robot account of the teamspace), readable by you only. It goes into the generated scripts and is never printed. | | `--robot` | `""` | Name of the robot (the multi-computer cluster) the computers belong to. Becomes the node label `teknoir.org/robot` and, together with a role, the node name `-`. See [Robots with several computers](#robots-with-several-computers). | | `--role` | `""` | Node role of the **server** script (a DNS label such as `vision` or `io`). Becomes the node label `teknoir.org/node-role`. | | `--node-name` | `""` | Node name of the server script. Default: `-` with `--robot` and `--role`, otherwise `teknoir-master` as before. | @@ -151,6 +153,42 @@ Without any of these flags the scripts are exactly what they were before (`tekno server, the host name for the agent, no labels). The docker server script (`k3s < 1.24`) does not take node roles. +## Registry mirror (Harbor pull-through cache) + +With `--registry-mirror HOST` the robot pulls its images through the platform's Harbor instead of straight +from the internet: Docker Hub, GitHub (`ghcr.io`), `gcr.io`, `quay.io`, `registry.k8s.io` and `nvcr.io` map +to the Harbor projects `dockerhub`, `ghcr`, `gcr`, `quay`, `k8s` and `nvcr`. Private `ghcr.io/teknoir/*` +images then reach the robot without a GitHub token on the device, and a popular image is downloaded once +per site instead of once per robot. + +```bash +printf 'robot$ts-my-teamspace:%s\n' "$HARBOR_ROBOT_SECRET" > harbor-pull && chmod 600 harbor-pull +tnctl bootstrap edge-01 --registry-mirror harbor.teknoir.cloud --registry-credential-file harbor-pull +``` + +- **Fallback.** k3s always tries the upstream registry last. When Harbor is down or slow to fail, public + images still come from the internet; a private image that is not on the device yet does not (the robots + have no upstream credential). +- **What is written.** The server and agent scripts write `/etc/rancher/k3s/registries.yaml` (mode `0600`, + first line `# managed by tnctl bootstrap`) before the k3s installer runs, so the first start already uses + it. On a re-run the installer restarts k3s. The file is written only when it differs. +- **A file you wrote is never touched.** A `registries.yaml` without the marker line (`# managed by tnctl + bootstrap`, or `# managed by teknoir`, which the Ansible playbook `registry-mirror.yml` writes) is kept, with + a warning, and the mirror is not configured. A file with either marker is replaced, so a re-bootstrapped + robot never keeps a stale credential. `--registry-mirror none` removes a file that the scripts wrote + (and keeps yours). Without `--registry-mirror` the scripts leave the file alone. +- **The credential** is a Harbor robot account that can only pull from the proxy projects. It is embedded in + the scripts (like the device keys, so keep the scripts private) and the scripts never print it. `tnctl bootstrap` writes the + scripts with mode `0600` (and tightens a script that an earlier run left at `0644`); the file on the robot + must not be readable by group or others. +- The Docker-mode script (`bootstrap_docker_server_.sh`) is not changed. +- For a robot that is already running, the Ansible playbook `registry-mirror.yml` (repository `ansible`) + does the same, with a probe before anything changes and a rollback. + +The platform may later send the registry and the credential together with the device login; `tnctl` then +uses them as the default of `--registry-mirror` and its credential (a `registry` field in the device +credentials response, ignored when absent). + ## The k3s version The server and agent scripts install a **pinned** k3s release: `INSTALL_K3S_VERSION` is set from @@ -312,6 +350,9 @@ tnctl bootstrap --docker-secret "$(cat dockerconfig.json)" # Generate but do not upload tnctl bootstrap edge-01 --skip-upload +# Pull images through Harbor (credential in a file, never printed) +tnctl bootstrap edge-01 --registry-mirror harbor.teknoir.cloud --registry-credential-file harbor-pull + # A single-computer robot with a GPU: the server script gets the role vision tnctl bootstrap vision-station-01 --robot vision-station-01 --role vision --label teknoir.org/gpu=nvidia diff --git a/pkg/api/fleet_devices.go b/pkg/api/fleet_devices.go index 4ed3bfe..3e259d5 100644 --- a/pkg/api/fleet_devices.go +++ b/pkg/api/fleet_devices.go @@ -216,6 +216,18 @@ type FleetDeviceCredentials struct { RSAPrivate string `json:"rsaPrivate,omitempty"` RSAPublic string `json:"rsaPublic,omitempty"` PublicSSHKey string `json:"publicSshKey,omitempty"` + // Registry is the image registry mirror (Harbor pull-through cache) of the device's teamspace, when the + // platform sends one. It is the default of `tnctl bootstrap --registry-mirror` and its credential. Nothing + // sends it yet (planned, plans/harbor-proxy-staging.md phase 1b): it is ignored when absent. + Registry *FleetRegistry `json:"registry,omitempty"` +} + +// FleetRegistry is the registry mirror a platform offers a device: the Harbor host and a pull-only robot +// credential, both plain text (not base64). A secret: never log or print it. +type FleetRegistry struct { + Host string `json:"host"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` } // FleetDeviceCredentialsBulk is the answer of the bulk credentials route. diff --git a/pkg/api/fleet_devices_test.go b/pkg/api/fleet_devices_test.go index b0b2763..e1fd04e 100644 --- a/pkg/api/fleet_devices_test.go +++ b/pkg/api/fleet_devices_test.go @@ -378,3 +378,26 @@ func TestNoCatalogCallsInTheSource(t *testing.T) { t.Errorf("code that calls the catalog: %v", hits) } } + +// The registry field of the device credentials is optional (phase 1b of plans/harbor-proxy-staging.md): +// absent it is nil, present it decodes as plain text. +func TestFleetDeviceCredentials_RegistryIsOptional(t *testing.T) { + var c FleetDeviceCredentials + if err := json.Unmarshal([]byte(`{"name":"d","namespace":"ns","username":"dQ=="}`), &c); err != nil { + t.Fatal(err) + } + if c.Registry != nil { + t.Fatalf("registry %+v without the field", c.Registry) + } + if err := json.Unmarshal([]byte(`{"name":"d","namespace":"ns","registry":{"host":"harbor.example.test","username":"robot$ts-ns","password":"pw"}}`), &c); err != nil { + t.Fatal(err) + } + if c.Registry == nil || c.Registry.Host != "harbor.example.test" || c.Registry.Username != "robot$ts-ns" || c.Registry.Password != "pw" { + t.Fatalf("%+v", c.Registry) + } + // unchanged encoding when there is none + b, _ := json.Marshal(FleetDeviceCredentials{Name: "d"}) + if strings.Contains(string(b), "registry") { + t.Fatalf("empty registry encoded: %s", b) + } +} diff --git a/pkg/templates/agent.sh.tmpl b/pkg/templates/agent.sh.tmpl index 0eb8c08..070da3e 100644 --- a/pkg/templates/agent.sh.tmpl +++ b/pkg/templates/agent.sh.tmpl @@ -45,7 +45,7 @@ $SUDO cp ${CONFIG_PATH}/rsa_public.pem /usr/local/share/ca-certificates/${_DEVIC $SUDO update-ca-certificates info "Install Rancher K3s Agent" -download k3s_installer.sh https://get.k3s.io +{{template "registries.sh.tmpl" .}}download k3s_installer.sh https://get.k3s.io $SUDO chmod +x k3s_installer.sh if [ "${OS_BUILD}" ] || [ "${INSECURE}" ]; then diff --git a/pkg/templates/bootstrap_templates.go b/pkg/templates/bootstrap_templates.go index d94f358..95db99c 100644 --- a/pkg/templates/bootstrap_templates.go +++ b/pkg/templates/bootstrap_templates.go @@ -60,6 +60,32 @@ type TemplateData struct { Server string // TLSSANs are extra --tls-san values of the server script, after the two it always adds. TLSSANs []string + // RegistryMirror is the Harbor host the device pulls images through (`tnctl bootstrap --registry-mirror`): + // the scripts write /etc/rancher/k3s/registries.yaml with the six proxy-cache mirrors (docker.io, + // ghcr.io, gcr.io, quay.io, registry.k8s.io, nvcr.io) and the upstream registry as the fallback. + // "none" removes a file that the scripts wrote. Empty leaves registries.yaml alone and renders no code. + RegistryMirror string + // RegistryUsername and RegistryPassword are the Harbor robot credential for RegistryMirror. They are + // embedded in single quotes in the script and never printed by it; ValidateTemplateData allows a `$` + // here (a Harbor robot is called robot$name) but no quote, backslash, newline or control character. + RegistryUsername string + RegistryPassword string +} + +// registryHostRE is a Harbor host name, optionally with a port: what the k3s mirror endpoint +// https:// can hold. +var registryHostRE = regexp.MustCompile(`^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?(:[0-9]{1,5})?$`) + +// RegistryMirrorNone is the value of RegistryMirror that removes a registries.yaml written by the scripts. +const RegistryMirrorNone = "none" + +// ValidateRegistryMirror accepts "" (no registry mirror), "none", or a host name with an optional port. +// The value ends up in a shell script and in a YAML file, so nothing else is let through. +func ValidateRegistryMirror(v string) error { + if v == "" || v == RegistryMirrorNone || registryHostRE.MatchString(v) { + return nil + } + return fmt.Errorf("invalid registry mirror %q: want a host name such as harbor.teknoir.cloud (optionally with :port), or %q", v, RegistryMirrorNone) } func (d TemplateData) k3sVersion() string { diff --git a/pkg/templates/golden_data_test.go b/pkg/templates/golden_data_test.go index 926726b..920afa3 100644 --- a/pkg/templates/golden_data_test.go +++ b/pkg/templates/golden_data_test.go @@ -14,5 +14,9 @@ func goldenData() TemplateData { PublicSSHKey: "ssh-ed25519 TEST-ONLY-ssh-public golden@example.test", K3SToken: "TEST-ONLY-k3s-token", DockerSecret: "TEST-ONLY-docker-secret", + // the registry mirror, so the golden files cover registries.sh.tmpl + RegistryMirror: "harbor.example.test", + RegistryUsername: "robot$golden", + RegistryPassword: "TEST-ONLY-registry-password", } } diff --git a/pkg/templates/partials/registries.sh.tmpl b/pkg/templates/partials/registries.sh.tmpl new file mode 100644 index 0000000..c37690a --- /dev/null +++ b/pkg/templates/partials/registries.sh.tmpl @@ -0,0 +1,52 @@ +{{define "registries.sh.tmpl"}}{{if .RegistryMirror}}# The registry mirror (tnctl bootstrap --registry-mirror): pull images through the Harbor proxy-cache +# projects, with the upstream registry as the fallback (k3s always tries its default endpoint last, so +# a Harbor outage never blocks public images). It runs before the k3s installer, so a first start already +# uses it; a re-run restarts k3s through the installer. The file is written only when it differs, at +# mode 0600 (it holds the Harbor credential), and a registries.yaml that this script did not write is +# never touched. The credential is never printed. +_TN_REG_FILE="${K3S_CONFIG_PATH:-/etc/rancher/k3s}/registries.yaml" +_TN_REG_MARKER='# managed by tnctl bootstrap' +# playbooks/registry-mirror.yml (ansible) marks its file differently; it writes the same content, so either is ours. +_TN_REG_MARKER_PB='# managed by teknoir' +_TN_REG_HOST='{{.RegistryMirror}}' +{{if ne .RegistryMirror "none"}}_TN_REG_USER='{{.RegistryUsername}}' +_TN_REG_PASS='{{.RegistryPassword}}' +# A double-quoted YAML scalar: backslash and double quote escaped. +_tn_reg_q() { printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'; } +{{end}}if [ -e "${_TN_REG_FILE}" ] && ! $SUDO grep -qxF -e "${_TN_REG_MARKER}" -e "${_TN_REG_MARKER_PB}" "${_TN_REG_FILE}"; then + warn "${_TN_REG_FILE} exists and was not written by tnctl bootstrap or registry-mirror.yml: keeping it, the registry mirror is not configured" +{{- if eq .RegistryMirror "none"}} +elif [ -e "${_TN_REG_FILE}" ]; then + $SUDO rm -f "${_TN_REG_FILE}" + info "Registry mirror removed (${_TN_REG_FILE}); pulls go to the upstream registries again" +fi +{{- else}} +else + $SUDO mkdir -p "$(dirname "${_TN_REG_FILE}")" + _tn_reg_tmp=$(umask 077; mktemp "${_TN_REG_FILE}.XXXXXX") || fatal "Cannot create a temporary file next to ${_TN_REG_FILE}" + { + printf '%s\n' "${_TN_REG_MARKER}" + printf '# k3s private registry mirrors: pull-through cache on %s. containerd asks Harbor first and\n' "${_TN_REG_HOST}" + printf '# falls back to the upstream registry. Written by tnctl bootstrap --registry-mirror; do not edit.\n' + printf 'mirrors:\n' + for _tn_reg_m in docker.io=dockerhub ghcr.io=ghcr gcr.io=gcr quay.io=quay registry.k8s.io=k8s nvcr.io=nvcr; do + printf ' %s:\n endpoint:\n - "https://%s"\n rewrite:\n "^(.*)$": "%s/$1"\n' \ + "${_tn_reg_m%%=*}" "${_TN_REG_HOST}" "${_tn_reg_m#*=}" + done + printf 'configs:\n "%s":\n auth:\n username: "%s"\n password: "%s"\n' \ + "${_TN_REG_HOST}" "$(_tn_reg_q "${_TN_REG_USER}")" "$(_tn_reg_q "${_TN_REG_PASS}")" + } > "${_tn_reg_tmp}" || { rm -f "${_tn_reg_tmp}"; fatal "Cannot write ${_TN_REG_FILE}"; } + if [ -f "${_TN_REG_FILE}" ] && cmp -s "${_tn_reg_tmp}" "${_TN_REG_FILE}"; then + rm -f "${_tn_reg_tmp}" + $SUDO chmod 0600 "${_TN_REG_FILE}" + info "Registry mirror ${_TN_REG_HOST} already configured" + else + $SUDO chmod 0600 "${_tn_reg_tmp}" + $SUDO mv -f "${_tn_reg_tmp}" "${_TN_REG_FILE}" || { rm -f "${_tn_reg_tmp}"; fatal "Cannot write ${_TN_REG_FILE}"; } + info "Registry mirror ${_TN_REG_HOST} configured (${_TN_REG_FILE})" + fi + unset _tn_reg_tmp _tn_reg_m +fi +unset _TN_REG_USER _TN_REG_PASS +{{- end}} +{{end}}{{end}} diff --git a/pkg/templates/registries_test.go b/pkg/templates/registries_test.go new file mode 100644 index 0000000..f3f8f0f --- /dev/null +++ b/pkg/templates/registries_test.go @@ -0,0 +1,427 @@ +package templates + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" +) + +const ( + regPassword = "S3cretRobotPassw0rdFORtest" + regUsername = "robot$nodes" + regMarker = "# managed by tnctl bootstrap" +) + +// registriesBlock renders the registries partial for d, the way server.sh.tmpl and agent.sh.tmpl include it. +func registriesBlock(t *testing.T, d TemplateData) string { + t.Helper() + var b strings.Builder + if err := ServerTemplate.ExecuteTemplate(&b, "registries.sh.tmpl", d); err != nil { + t.Fatal(err) + } + return b.String() +} + +func regData(mirror string) TemplateData { + d := goldenData() + d.RegistryMirror = mirror + d.RegistryUsername = regUsername + d.RegistryPassword = regPassword + if mirror == "none" || mirror == "" { + d.RegistryUsername, d.RegistryPassword = "", "" + } + return d +} + +type regRun struct { + dir string // K3S_CONFIG_PATH + output string // combined stdout and stderr + err error +} + +func (r regRun) file() string { return filepath.Join(r.dir, "registries.yaml") } + +// runRegistries runs the partial in dir under `set -e` and `umask 027` (the header of the bootstrap +// script sets both), with info/warn/fatal as in the header. SUDO is empty, as for root. +func runRegistries(t *testing.T, dir string, d TemplateData) regRun { + t.Helper() + script := "set -e\numask 027\n" + + "info() { echo \"[INFO] $*\"; }\nwarn() { echo \"[WARN] $*\" >&2; }\nfatal() { echo \"[ERROR] $*\" >&2; exit 1; }\n" + + "SUDO=\nK3S_CONFIG_PATH=" + dir + "\n" + registriesBlock(t, d) + "echo done\n" + var out bytes.Buffer + cmd := exec.Command("sh", "-c", script) + cmd.Stdout, cmd.Stderr = &out, &out + err := cmd.Run() + return regRun{dir: dir, output: out.String(), err: err} +} + +func (r regRun) mustOK(t *testing.T) regRun { + t.Helper() + if r.err != nil || !strings.HasSuffix(r.output, "done\n") { + t.Fatalf("script failed: %v\n%s", r.err, r.output) + } + return r +} + +func readFile(t *testing.T, p string) string { + t.Helper() + b, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func inode(t *testing.T, p string) uint64 { + t.Helper() + fi, err := os.Stat(p) + if err != nil { + t.Fatal(err) + } + return fi.Sys().(*syscall.Stat_t).Ino +} + +func noPassword(t *testing.T, what, s string) { + t.Helper() + if strings.Contains(s, regPassword) { + t.Errorf("%s contains the password", what) + } +} + +const wantFresh = regMarker + ` +# k3s private registry mirrors: pull-through cache on harbor.example.test. containerd asks Harbor first and +# falls back to the upstream registry. Written by tnctl bootstrap --registry-mirror; do not edit. +mirrors: + docker.io: + endpoint: + - "https://harbor.example.test" + rewrite: + "^(.*)$": "dockerhub/$1" + ghcr.io: + endpoint: + - "https://harbor.example.test" + rewrite: + "^(.*)$": "ghcr/$1" + gcr.io: + endpoint: + - "https://harbor.example.test" + rewrite: + "^(.*)$": "gcr/$1" + quay.io: + endpoint: + - "https://harbor.example.test" + rewrite: + "^(.*)$": "quay/$1" + registry.k8s.io: + endpoint: + - "https://harbor.example.test" + rewrite: + "^(.*)$": "k8s/$1" + nvcr.io: + endpoint: + - "https://harbor.example.test" + rewrite: + "^(.*)$": "nvcr/$1" +configs: + "harbor.example.test": + auth: + username: "robot$nodes" + password: "` + regPassword + `" +` + +func TestRegistries_Fresh(t *testing.T) { + dir := t.TempDir() + r := runRegistries(t, dir, regData("harbor.example.test")).mustOK(t) + if got := readFile(t, r.file()); got != wantFresh { + t.Fatalf("registries.yaml:\n%s\nwant:\n%s", got, wantFresh) + } + fi, _ := os.Stat(r.file()) + if fi.Mode().Perm() != 0o600 { + t.Errorf("mode %v, want 0600 (under umask 027)", fi.Mode().Perm()) + } + if !strings.Contains(r.output, "Registry mirror harbor.example.test configured") { + t.Errorf("output: %s", r.output) + } + noPassword(t, "output", r.output) + if left, _ := filepath.Glob(filepath.Join(dir, "registries.yaml.*")); len(left) != 0 { + t.Errorf("temp files left behind: %v", left) + } +} + +func TestRegistries_RerunIsANoOp(t *testing.T) { + dir := t.TempDir() + r := runRegistries(t, dir, regData("harbor.example.test")).mustOK(t) + ino, before := inode(t, r.file()), readFile(t, r.file()) + if err := os.Chmod(r.file(), 0o644); err != nil { // loosened by someone: repaired, content untouched + t.Fatal(err) + } + r2 := runRegistries(t, dir, regData("harbor.example.test")).mustOK(t) + if !strings.Contains(r2.output, "already configured") { + t.Errorf("output: %s", r2.output) + } + if inode(t, r.file()) != ino || readFile(t, r.file()) != before { + t.Error("a re-run rewrote the file") + } + if fi, _ := os.Stat(r.file()); fi.Mode().Perm() != 0o600 { + t.Errorf("mode %v after a re-run", fi.Mode().Perm()) + } + noPassword(t, "output", r2.output) + if left, _ := filepath.Glob(filepath.Join(dir, "registries.yaml.*")); len(left) != 0 { + t.Errorf("temp files left behind: %v", left) + } +} + +func TestRegistries_ChangedCredentialAndHostAreWritten(t *testing.T) { + dir := t.TempDir() + r := runRegistries(t, dir, regData("harbor.example.test")).mustOK(t) + d := regData("harbor.example.test") + d.RegistryPassword = "RotatedPassw0rd" + runRegistries(t, dir, d).mustOK(t) + if got := readFile(t, r.file()); !strings.Contains(got, `password: "RotatedPassw0rd"`) || strings.Contains(got, regPassword) { + t.Errorf("rotated password not written:\n%s", got) + } + d = regData("other.example.test:8443") + runRegistries(t, dir, d).mustOK(t) + if got := readFile(t, r.file()); !strings.Contains(got, `"https://other.example.test:8443"`) || strings.Contains(got, "harbor.example.test") { + t.Errorf("new host not written:\n%s", got) + } +} + +func TestRegistries_ForeignFileIsKept(t *testing.T) { + dir := t.TempDir() + foreign := "mirrors:\n docker.io:\n endpoint:\n - \"https://mine.example\"\n" + p := filepath.Join(dir, "registries.yaml") + if err := os.WriteFile(p, []byte(foreign), 0o640); err != nil { + t.Fatal(err) + } + for _, mirror := range []string{"harbor.example.test", "none"} { + r := runRegistries(t, dir, regData(mirror)).mustOK(t) + if readFile(t, p) != foreign { + t.Errorf("%s: the foreign file was changed", mirror) + } + if fi, _ := os.Stat(p); fi.Mode().Perm() != 0o640 { + t.Errorf("%s: mode of the foreign file changed to %v", mirror, fi.Mode().Perm()) + } + if !strings.Contains(r.output, "[WARN]") || !strings.Contains(r.output, "was not written by tnctl bootstrap") { + t.Errorf("%s: no warning:\n%s", mirror, r.output) + } + noPassword(t, "output", r.output) + } + // a comment that only contains the marker text is not the marker + if err := os.WriteFile(p, []byte("# not "+regMarker+"\nmirrors: {}\n"), 0o600); err != nil { + t.Fatal(err) + } + r := runRegistries(t, dir, regData("harbor.example.test")).mustOK(t) + if !strings.Contains(r.output, "[WARN]") || strings.Contains(readFile(t, p), "harbor.example.test") { + t.Errorf("a file with the marker text inside a comment was taken as ours:\n%s", r.output) + } +} + +// playbooks/registry-mirror.yml (ansible) writes the same content under its own first line; tnctl treats +// that file as its own: it replaces it (a stale credential must not stay) and `none` removes it. +func TestRegistries_AcceptsThePlaybooksMarker(t *testing.T) { + const pbMarker = "# managed by teknoir" + dir := t.TempDir() + p := filepath.Join(dir, "registries.yaml") + if err := os.WriteFile(p, []byte(pbMarker+"\nmirrors: {}\nconfigs:\n \"old.example\":\n auth:\n password: \"stale\"\n"), 0o600); err != nil { + t.Fatal(err) + } + r := runRegistries(t, dir, regData("harbor.example.test")).mustOK(t) + got := readFile(t, p) + if strings.Contains(r.output, "[WARN]") || strings.Contains(got, "stale") || !strings.HasPrefix(got, regMarker+"\n") || !strings.Contains(got, regPassword) { + t.Errorf("the playbook's file was not replaced:\n%s\n%s", r.output, got) + } + if err := os.WriteFile(p, []byte(pbMarker+"\nmirrors: {}\n"), 0o600); err != nil { + t.Fatal(err) + } + r = runRegistries(t, dir, regData("none")).mustOK(t) + if _, err := os.Stat(p); !os.IsNotExist(err) || !strings.Contains(r.output, "Registry mirror removed") { + t.Errorf("none did not remove the playbook's file: %v\n%s", err, r.output) + } + noPassword(t, "output", r.output) +} + +func TestRegistries_NoneRemovesAManagedFile(t *testing.T) { + dir := t.TempDir() + r := runRegistries(t, dir, regData("harbor.example.test")).mustOK(t) + r2 := runRegistries(t, dir, regData("none")).mustOK(t) + if _, err := os.Stat(r.file()); !os.IsNotExist(err) { + t.Errorf("the managed file is still there: %v", err) + } + if !strings.Contains(r2.output, "Registry mirror removed") { + t.Errorf("output: %s", r2.output) + } + // without a file, none is a quiet no-op + r3 := runRegistries(t, dir, regData("none")).mustOK(t) + if strings.TrimSpace(r3.output) != "done" { + t.Errorf("none without a file printed: %q", r3.output) + } + noPassword(t, "output", r2.output+r3.output) + // none never carries a credential into the script + if blk := registriesBlock(t, regData("none")); strings.Contains(blk, "_TN_REG_PASS") || strings.Contains(blk, regPassword) { + t.Error("the none variant mentions a credential") + } +} + +func TestRegistries_EmptyMirrorRendersNothingAndLeavesTheFileAlone(t *testing.T) { + d := regData("") + if blk := registriesBlock(t, d); blk != "" { + t.Fatalf("an empty RegistryMirror must render nothing, got:\n%s", blk) + } + for _, tpl := range []string{"server", "agent"} { + var b bytes.Buffer + var err error + if tpl == "server" { + err = ServerTemplate.Execute(&b, d) + } else { + err = AgentTemplate.Execute(&b, d) + } + if err != nil { + t.Fatal(err) + } + if strings.Contains(b.String(), "registries.yaml") || strings.Contains(b.String(), "_TN_REG_") { + t.Errorf("%s script without a mirror mentions the registry code", tpl) + } + } + dir := t.TempDir() + p := filepath.Join(dir, "registries.yaml") + for _, content := range []string{"mine: true\n", regMarker + "\nmirrors: {}\n"} { + if err := os.WriteFile(p, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + runRegistries(t, dir, d).mustOK(t) + if readFile(t, p) != content { + t.Errorf("an empty mirror changed %q", content) + } + } +} + +func TestRegistries_IncludedBeforeTheInstaller(t *testing.T) { + d := goldenData() + for name, render := range map[string]func() (string, error){ + "server": func() (string, error) { + var b bytes.Buffer + err := ServerTemplate.Execute(&b, d) + return b.String(), err + }, + "agent": func() (string, error) { + var b bytes.Buffer + err := AgentTemplate.Execute(&b, d) + return b.String(), err + }, + } { + out, err := render() + if err != nil { + t.Fatal(err) + } + reg, inst := strings.Index(out, "_TN_REG_FILE="), strings.Index(out, "$SUDO ./k3s_installer.sh") + if reg < 0 || inst < 0 || reg > inst { + t.Errorf("%s: the registry block (%d) must come before the k3s installer (%d)", name, reg, inst) + } + if strings.Count(out, "_TN_REG_FILE=") != 1 { + t.Errorf("%s: the registry block appears %d times", name, strings.Count(out, "_TN_REG_FILE=")) + } + } + var b bytes.Buffer + if err := DockerServerTemplate.Execute(&b, d); err != nil { + t.Fatal(err) + } + if strings.Contains(b.String(), "registries.yaml") { + t.Error("the Docker-mode script must not write registries.yaml") + } +} + +// A password with characters that need care (a `#` and a `:` in YAML, `!` and `%` in printf) round-trips +// byte for byte, and a `$` in the user name is not expanded by the shell. +func TestRegistries_SpecialCharactersSurvive(t *testing.T) { + dir := t.TempDir() + d := regData("harbor.example.test") + d.RegistryUsername = "robot$a$HOME" + d.RegistryPassword = "p#a:s!s%s%d-_.~" + if err := ValidateTemplateData(d); err != nil { + t.Fatal(err) + } + r := runRegistries(t, dir, d).mustOK(t) + got := readFile(t, r.file()) + if !strings.Contains(got, `username: "robot$a$HOME"`) || !strings.Contains(got, `password: "p#a:s!s%s%d-_.~"`) { + t.Errorf("credential mangled:\n%s", got) + } +} + +// The password is in the script (it has to be), in single quotes, so a value that tries to close the +// quote or run something is refused by the validation before a script exists. +func TestRegistries_ValidateRegistryData(t *testing.T) { + ok := func(mod func(*TemplateData)) error { + d := regData("harbor.example.test") + mod(&d) + return ValidateTemplateData(d) + } + if err := ok(func(*TemplateData) {}); err != nil { + t.Fatalf("the robot$nodes credential must be accepted: %v", err) + } + for _, h := range []string{"", "none", "harbor.teknoir.cloud", "harbor.example.test:8443", "localhost", "10.0.0.5"} { + if err := ValidateRegistryMirror(h); err != nil { + t.Errorf("host %q rejected: %v", h, err) + } + } + for _, h := range []string{"a b", "a/b", `a"b`, "a'b", "a$b", "a;b", "https://harbor.example.test", "-x", "a:b:c", "x:99999999", "harbor.example.test\n", "$(id)"} { + if err := ValidateRegistryMirror(h); err == nil { + t.Errorf("host %q accepted", h) + } + } + bad := map[string]func(*TemplateData){ + "quote closes": func(d *TemplateData) { d.RegistryPassword = "x'; touch /tmp/pwn; '" }, + "single quote": func(d *TemplateData) { d.RegistryPassword = "a'b" }, + "double quote": func(d *TemplateData) { d.RegistryPassword = `a"b` }, + "backslash": func(d *TemplateData) { d.RegistryPassword = `a\b` }, + "backtick": func(d *TemplateData) { d.RegistryPassword = "a`id`b" }, + "newline": func(d *TemplateData) { d.RegistryPassword = "a\nb" }, + "carriage return": func(d *TemplateData) { d.RegistryPassword = "a\rb" }, + "space": func(d *TemplateData) { d.RegistryPassword = "a b" }, + "nul": func(d *TemplateData) { d.RegistryPassword = "a\x00b" }, + "del": func(d *TemplateData) { d.RegistryPassword = "a\x7fb" }, + "too long": func(d *TemplateData) { d.RegistryPassword = strings.Repeat("a", 513) }, + "user quote": func(d *TemplateData) { d.RegistryUsername = "robot'x" }, + "user newline": func(d *TemplateData) { d.RegistryUsername = "robot\nx" }, + "user empty": func(d *TemplateData) { d.RegistryUsername = "" }, + "password empty": func(d *TemplateData) { d.RegistryPassword = "" }, + "credential w/o host": func(d *TemplateData) { d.RegistryMirror = "" }, + "credential on none": func(d *TemplateData) { d.RegistryMirror = "none" }, + "host with quote": func(d *TemplateData) { d.RegistryMirror = "a'b" }, + "host with dollar": func(d *TemplateData) { d.RegistryMirror = "a$b.example" }, + } + for name, mod := range bad { + err := ok(mod) + if err == nil { + t.Errorf("%s: accepted", name) + continue + } + noPassword(t, name+" error", err.Error()) + for _, leak := range []string{"touch", "a'b", "a\nb"} { + if strings.Contains(err.Error(), leak) { + t.Errorf("%s: error %q leaks %q", name, err, leak) + } + } + if _, rerr := RenderServer(func() TemplateData { d := regData("harbor.example.test"); mod(&d); return d }()); rerr == nil { + t.Errorf("%s: RenderServer accepted it", name) + } + } +} + +// A credential hidden in a quote-closing value must never become shell: run the rendered block with a +// value that the validation would refuse, to prove why it must (the test documents the single-quote context). +func TestRegistries_TheValidationIsWhatKeepsTheScriptSafe(t *testing.T) { + d := regData("harbor.example.test") + d.RegistryPassword = "x'; echo INJECTED; '" + if ValidateTemplateData(d) == nil { + t.Fatal("validation must refuse a quote in the password") + } + blk := registriesBlock(t, d) // unvalidated render: the quote does close the string + if !strings.Contains(blk, "_TN_REG_PASS='x'; echo INJECTED; ''") { + t.Fatalf("test premise changed:\n%s", blk) + } +} diff --git a/pkg/templates/render.go b/pkg/templates/render.go index 00326bf..99a0628 100644 --- a/pkg/templates/render.go +++ b/pkg/templates/render.go @@ -60,6 +60,7 @@ func ValidateTemplateData(d TemplateData) error { {"Node.Robot", d.Node.Robot, false}, {"Node.Name", d.Node.Name, false}, {"Server", d.Server, false}, + {"RegistryMirror", d.RegistryMirror, false}, } for _, l := range d.Node.Labels { fields = append(fields, field{"Node.Labels", l, false}) @@ -78,12 +79,45 @@ func ValidateTemplateData(d TemplateData) error { return fmt.Errorf("template data field %s must not contain a newline", f.name) } } + if err := validateRegistry(d); err != nil { + return err + } if d.Username != "" && (!loginNameRe.MatchString(d.Username) || d.Username == "root") { return fmt.Errorf("template data field Username must be a lowercase login name (a-z, 0-9, _ and -, starting with a letter or _, at most 32 characters, not root)") } return nil } +// validateRegistry checks the registry mirror and its credential. The credential is embedded in +// single quotes (see partials/registries.sh.tmpl), so a `$` is fine; a single quote, a backslash, a +// double quote (it ends up inside a double-quoted YAML scalar after escaping, but the shell variable +// is the place to refuse), whitespace and control characters are not. Errors never contain the value. +func validateRegistry(d TemplateData) error { + if ValidateRegistryMirror(d.RegistryMirror) != nil { + return fmt.Errorf("template data field RegistryMirror must be a host name (optionally with :port) or %q", RegistryMirrorNone) + } + if d.RegistryMirror == "" || d.RegistryMirror == RegistryMirrorNone { + if d.RegistryUsername != "" || d.RegistryPassword != "" { + return fmt.Errorf("template data fields RegistryUsername and RegistryPassword need a RegistryMirror host") + } + return nil + } + if d.RegistryUsername == "" || d.RegistryPassword == "" { + return fmt.Errorf("template data fields RegistryUsername and RegistryPassword must both be set for RegistryMirror %s", d.RegistryMirror) + } + for _, f := range []field{{"RegistryUsername", d.RegistryUsername, false}, {"RegistryPassword", d.RegistryPassword, false}} { + if len(f.value) > 512 { + return fmt.Errorf("template data field %s is longer than 512 characters", f.name) + } + for _, r := range f.value { + if r == '\'' || r == '"' || r == '\\' || r == '`' || r <= ' ' || r == 0x7f { + return fmt.Errorf("template data field %s contains a character that is unsafe in a shell script (quote, backtick, backslash, whitespace or control character)", f.name) + } + } + } + return nil +} + // RenderServer validates d and renders the server (k3s server node) bootstrap // script. It produces the same bytes as ServerTemplate.Execute for valid input. func RenderServer(d TemplateData) ([]byte, error) { diff --git a/pkg/templates/server.sh.tmpl b/pkg/templates/server.sh.tmpl index 41d524d..093aafa 100644 --- a/pkg/templates/server.sh.tmpl +++ b/pkg/templates/server.sh.tmpl @@ -124,7 +124,7 @@ fs.inotify.max_user_instances=1024 EOF sysctl --system -download k3s_installer.sh https://get.k3s.io +{{template "registries.sh.tmpl" .}}download k3s_installer.sh https://get.k3s.io $SUDO chmod +x k3s_installer.sh if [ ${OS_BUILD} ] || [ ${INSECURE} ]; then diff --git a/pkg/templates/testdata/agent.golden.sh b/pkg/templates/testdata/agent.golden.sh index 7c74599..8bf3669 100644 --- a/pkg/templates/testdata/agent.golden.sh +++ b/pkg/templates/testdata/agent.golden.sh @@ -754,6 +754,50 @@ $SUDO cp ${CONFIG_PATH}/rsa_public.pem /usr/local/share/ca-certificates/${_DEVIC $SUDO update-ca-certificates info "Install Rancher K3s Agent" +# The registry mirror (tnctl bootstrap --registry-mirror): pull images through the Harbor proxy-cache +# projects, with the upstream registry as the fallback (k3s always tries its default endpoint last, so +# a Harbor outage never blocks public images). It runs before the k3s installer, so a first start already +# uses it; a re-run restarts k3s through the installer. The file is written only when it differs, at +# mode 0600 (it holds the Harbor credential), and a registries.yaml that this script did not write is +# never touched. The credential is never printed. +_TN_REG_FILE="${K3S_CONFIG_PATH:-/etc/rancher/k3s}/registries.yaml" +_TN_REG_MARKER='# managed by tnctl bootstrap' +# playbooks/registry-mirror.yml (ansible) marks its file differently; it writes the same content, so either is ours. +_TN_REG_MARKER_PB='# managed by teknoir' +_TN_REG_HOST='harbor.example.test' +_TN_REG_USER='robot$golden' +_TN_REG_PASS='TEST-ONLY-registry-password' +# A double-quoted YAML scalar: backslash and double quote escaped. +_tn_reg_q() { printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'; } +if [ -e "${_TN_REG_FILE}" ] && ! $SUDO grep -qxF -e "${_TN_REG_MARKER}" -e "${_TN_REG_MARKER_PB}" "${_TN_REG_FILE}"; then + warn "${_TN_REG_FILE} exists and was not written by tnctl bootstrap or registry-mirror.yml: keeping it, the registry mirror is not configured" +else + $SUDO mkdir -p "$(dirname "${_TN_REG_FILE}")" + _tn_reg_tmp=$(umask 077; mktemp "${_TN_REG_FILE}.XXXXXX") || fatal "Cannot create a temporary file next to ${_TN_REG_FILE}" + { + printf '%s\n' "${_TN_REG_MARKER}" + printf '# k3s private registry mirrors: pull-through cache on %s. containerd asks Harbor first and\n' "${_TN_REG_HOST}" + printf '# falls back to the upstream registry. Written by tnctl bootstrap --registry-mirror; do not edit.\n' + printf 'mirrors:\n' + for _tn_reg_m in docker.io=dockerhub ghcr.io=ghcr gcr.io=gcr quay.io=quay registry.k8s.io=k8s nvcr.io=nvcr; do + printf ' %s:\n endpoint:\n - "https://%s"\n rewrite:\n "^(.*)$": "%s/$1"\n' \ + "${_tn_reg_m%%=*}" "${_TN_REG_HOST}" "${_tn_reg_m#*=}" + done + printf 'configs:\n "%s":\n auth:\n username: "%s"\n password: "%s"\n' \ + "${_TN_REG_HOST}" "$(_tn_reg_q "${_TN_REG_USER}")" "$(_tn_reg_q "${_TN_REG_PASS}")" + } > "${_tn_reg_tmp}" || { rm -f "${_tn_reg_tmp}"; fatal "Cannot write ${_TN_REG_FILE}"; } + if [ -f "${_TN_REG_FILE}" ] && cmp -s "${_tn_reg_tmp}" "${_TN_REG_FILE}"; then + rm -f "${_tn_reg_tmp}" + $SUDO chmod 0600 "${_TN_REG_FILE}" + info "Registry mirror ${_TN_REG_HOST} already configured" + else + $SUDO chmod 0600 "${_tn_reg_tmp}" + $SUDO mv -f "${_tn_reg_tmp}" "${_TN_REG_FILE}" || { rm -f "${_tn_reg_tmp}"; fatal "Cannot write ${_TN_REG_FILE}"; } + info "Registry mirror ${_TN_REG_HOST} configured (${_TN_REG_FILE})" + fi + unset _tn_reg_tmp _tn_reg_m +fi +unset _TN_REG_USER _TN_REG_PASS download k3s_installer.sh https://get.k3s.io $SUDO chmod +x k3s_installer.sh diff --git a/pkg/templates/testdata/agent.roles.golden.sh b/pkg/templates/testdata/agent.roles.golden.sh index 796c214..957f0d0 100644 --- a/pkg/templates/testdata/agent.roles.golden.sh +++ b/pkg/templates/testdata/agent.roles.golden.sh @@ -906,6 +906,50 @@ $SUDO cp ${CONFIG_PATH}/rsa_public.pem /usr/local/share/ca-certificates/${_DEVIC $SUDO update-ca-certificates info "Install Rancher K3s Agent" +# The registry mirror (tnctl bootstrap --registry-mirror): pull images through the Harbor proxy-cache +# projects, with the upstream registry as the fallback (k3s always tries its default endpoint last, so +# a Harbor outage never blocks public images). It runs before the k3s installer, so a first start already +# uses it; a re-run restarts k3s through the installer. The file is written only when it differs, at +# mode 0600 (it holds the Harbor credential), and a registries.yaml that this script did not write is +# never touched. The credential is never printed. +_TN_REG_FILE="${K3S_CONFIG_PATH:-/etc/rancher/k3s}/registries.yaml" +_TN_REG_MARKER='# managed by tnctl bootstrap' +# playbooks/registry-mirror.yml (ansible) marks its file differently; it writes the same content, so either is ours. +_TN_REG_MARKER_PB='# managed by teknoir' +_TN_REG_HOST='harbor.example.test' +_TN_REG_USER='robot$golden' +_TN_REG_PASS='TEST-ONLY-registry-password' +# A double-quoted YAML scalar: backslash and double quote escaped. +_tn_reg_q() { printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'; } +if [ -e "${_TN_REG_FILE}" ] && ! $SUDO grep -qxF -e "${_TN_REG_MARKER}" -e "${_TN_REG_MARKER_PB}" "${_TN_REG_FILE}"; then + warn "${_TN_REG_FILE} exists and was not written by tnctl bootstrap or registry-mirror.yml: keeping it, the registry mirror is not configured" +else + $SUDO mkdir -p "$(dirname "${_TN_REG_FILE}")" + _tn_reg_tmp=$(umask 077; mktemp "${_TN_REG_FILE}.XXXXXX") || fatal "Cannot create a temporary file next to ${_TN_REG_FILE}" + { + printf '%s\n' "${_TN_REG_MARKER}" + printf '# k3s private registry mirrors: pull-through cache on %s. containerd asks Harbor first and\n' "${_TN_REG_HOST}" + printf '# falls back to the upstream registry. Written by tnctl bootstrap --registry-mirror; do not edit.\n' + printf 'mirrors:\n' + for _tn_reg_m in docker.io=dockerhub ghcr.io=ghcr gcr.io=gcr quay.io=quay registry.k8s.io=k8s nvcr.io=nvcr; do + printf ' %s:\n endpoint:\n - "https://%s"\n rewrite:\n "^(.*)$": "%s/$1"\n' \ + "${_tn_reg_m%%=*}" "${_TN_REG_HOST}" "${_tn_reg_m#*=}" + done + printf 'configs:\n "%s":\n auth:\n username: "%s"\n password: "%s"\n' \ + "${_TN_REG_HOST}" "$(_tn_reg_q "${_TN_REG_USER}")" "$(_tn_reg_q "${_TN_REG_PASS}")" + } > "${_tn_reg_tmp}" || { rm -f "${_tn_reg_tmp}"; fatal "Cannot write ${_TN_REG_FILE}"; } + if [ -f "${_TN_REG_FILE}" ] && cmp -s "${_tn_reg_tmp}" "${_TN_REG_FILE}"; then + rm -f "${_tn_reg_tmp}" + $SUDO chmod 0600 "${_TN_REG_FILE}" + info "Registry mirror ${_TN_REG_HOST} already configured" + else + $SUDO chmod 0600 "${_tn_reg_tmp}" + $SUDO mv -f "${_tn_reg_tmp}" "${_TN_REG_FILE}" || { rm -f "${_tn_reg_tmp}"; fatal "Cannot write ${_TN_REG_FILE}"; } + info "Registry mirror ${_TN_REG_HOST} configured (${_TN_REG_FILE})" + fi + unset _tn_reg_tmp _tn_reg_m +fi +unset _TN_REG_USER _TN_REG_PASS download k3s_installer.sh https://get.k3s.io $SUDO chmod +x k3s_installer.sh diff --git a/pkg/templates/testdata/server.golden.sh b/pkg/templates/testdata/server.golden.sh index c1a2ed7..a30c50e 100644 --- a/pkg/templates/testdata/server.golden.sh +++ b/pkg/templates/testdata/server.golden.sh @@ -2006,6 +2006,50 @@ fs.inotify.max_user_instances=1024 EOF sysctl --system +# The registry mirror (tnctl bootstrap --registry-mirror): pull images through the Harbor proxy-cache +# projects, with the upstream registry as the fallback (k3s always tries its default endpoint last, so +# a Harbor outage never blocks public images). It runs before the k3s installer, so a first start already +# uses it; a re-run restarts k3s through the installer. The file is written only when it differs, at +# mode 0600 (it holds the Harbor credential), and a registries.yaml that this script did not write is +# never touched. The credential is never printed. +_TN_REG_FILE="${K3S_CONFIG_PATH:-/etc/rancher/k3s}/registries.yaml" +_TN_REG_MARKER='# managed by tnctl bootstrap' +# playbooks/registry-mirror.yml (ansible) marks its file differently; it writes the same content, so either is ours. +_TN_REG_MARKER_PB='# managed by teknoir' +_TN_REG_HOST='harbor.example.test' +_TN_REG_USER='robot$golden' +_TN_REG_PASS='TEST-ONLY-registry-password' +# A double-quoted YAML scalar: backslash and double quote escaped. +_tn_reg_q() { printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'; } +if [ -e "${_TN_REG_FILE}" ] && ! $SUDO grep -qxF -e "${_TN_REG_MARKER}" -e "${_TN_REG_MARKER_PB}" "${_TN_REG_FILE}"; then + warn "${_TN_REG_FILE} exists and was not written by tnctl bootstrap or registry-mirror.yml: keeping it, the registry mirror is not configured" +else + $SUDO mkdir -p "$(dirname "${_TN_REG_FILE}")" + _tn_reg_tmp=$(umask 077; mktemp "${_TN_REG_FILE}.XXXXXX") || fatal "Cannot create a temporary file next to ${_TN_REG_FILE}" + { + printf '%s\n' "${_TN_REG_MARKER}" + printf '# k3s private registry mirrors: pull-through cache on %s. containerd asks Harbor first and\n' "${_TN_REG_HOST}" + printf '# falls back to the upstream registry. Written by tnctl bootstrap --registry-mirror; do not edit.\n' + printf 'mirrors:\n' + for _tn_reg_m in docker.io=dockerhub ghcr.io=ghcr gcr.io=gcr quay.io=quay registry.k8s.io=k8s nvcr.io=nvcr; do + printf ' %s:\n endpoint:\n - "https://%s"\n rewrite:\n "^(.*)$": "%s/$1"\n' \ + "${_tn_reg_m%%=*}" "${_TN_REG_HOST}" "${_tn_reg_m#*=}" + done + printf 'configs:\n "%s":\n auth:\n username: "%s"\n password: "%s"\n' \ + "${_TN_REG_HOST}" "$(_tn_reg_q "${_TN_REG_USER}")" "$(_tn_reg_q "${_TN_REG_PASS}")" + } > "${_tn_reg_tmp}" || { rm -f "${_tn_reg_tmp}"; fatal "Cannot write ${_TN_REG_FILE}"; } + if [ -f "${_TN_REG_FILE}" ] && cmp -s "${_tn_reg_tmp}" "${_TN_REG_FILE}"; then + rm -f "${_tn_reg_tmp}" + $SUDO chmod 0600 "${_TN_REG_FILE}" + info "Registry mirror ${_TN_REG_HOST} already configured" + else + $SUDO chmod 0600 "${_tn_reg_tmp}" + $SUDO mv -f "${_tn_reg_tmp}" "${_TN_REG_FILE}" || { rm -f "${_tn_reg_tmp}"; fatal "Cannot write ${_TN_REG_FILE}"; } + info "Registry mirror ${_TN_REG_HOST} configured (${_TN_REG_FILE})" + fi + unset _tn_reg_tmp _tn_reg_m +fi +unset _TN_REG_USER _TN_REG_PASS download k3s_installer.sh https://get.k3s.io $SUDO chmod +x k3s_installer.sh diff --git a/pkg/templates/testdata/server.roles.golden.sh b/pkg/templates/testdata/server.roles.golden.sh index 076a8ec..36d35a3 100644 --- a/pkg/templates/testdata/server.roles.golden.sh +++ b/pkg/templates/testdata/server.roles.golden.sh @@ -2159,6 +2159,50 @@ fs.inotify.max_user_instances=1024 EOF sysctl --system +# The registry mirror (tnctl bootstrap --registry-mirror): pull images through the Harbor proxy-cache +# projects, with the upstream registry as the fallback (k3s always tries its default endpoint last, so +# a Harbor outage never blocks public images). It runs before the k3s installer, so a first start already +# uses it; a re-run restarts k3s through the installer. The file is written only when it differs, at +# mode 0600 (it holds the Harbor credential), and a registries.yaml that this script did not write is +# never touched. The credential is never printed. +_TN_REG_FILE="${K3S_CONFIG_PATH:-/etc/rancher/k3s}/registries.yaml" +_TN_REG_MARKER='# managed by tnctl bootstrap' +# playbooks/registry-mirror.yml (ansible) marks its file differently; it writes the same content, so either is ours. +_TN_REG_MARKER_PB='# managed by teknoir' +_TN_REG_HOST='harbor.example.test' +_TN_REG_USER='robot$golden' +_TN_REG_PASS='TEST-ONLY-registry-password' +# A double-quoted YAML scalar: backslash and double quote escaped. +_tn_reg_q() { printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'; } +if [ -e "${_TN_REG_FILE}" ] && ! $SUDO grep -qxF -e "${_TN_REG_MARKER}" -e "${_TN_REG_MARKER_PB}" "${_TN_REG_FILE}"; then + warn "${_TN_REG_FILE} exists and was not written by tnctl bootstrap or registry-mirror.yml: keeping it, the registry mirror is not configured" +else + $SUDO mkdir -p "$(dirname "${_TN_REG_FILE}")" + _tn_reg_tmp=$(umask 077; mktemp "${_TN_REG_FILE}.XXXXXX") || fatal "Cannot create a temporary file next to ${_TN_REG_FILE}" + { + printf '%s\n' "${_TN_REG_MARKER}" + printf '# k3s private registry mirrors: pull-through cache on %s. containerd asks Harbor first and\n' "${_TN_REG_HOST}" + printf '# falls back to the upstream registry. Written by tnctl bootstrap --registry-mirror; do not edit.\n' + printf 'mirrors:\n' + for _tn_reg_m in docker.io=dockerhub ghcr.io=ghcr gcr.io=gcr quay.io=quay registry.k8s.io=k8s nvcr.io=nvcr; do + printf ' %s:\n endpoint:\n - "https://%s"\n rewrite:\n "^(.*)$": "%s/$1"\n' \ + "${_tn_reg_m%%=*}" "${_TN_REG_HOST}" "${_tn_reg_m#*=}" + done + printf 'configs:\n "%s":\n auth:\n username: "%s"\n password: "%s"\n' \ + "${_TN_REG_HOST}" "$(_tn_reg_q "${_TN_REG_USER}")" "$(_tn_reg_q "${_TN_REG_PASS}")" + } > "${_tn_reg_tmp}" || { rm -f "${_tn_reg_tmp}"; fatal "Cannot write ${_TN_REG_FILE}"; } + if [ -f "${_TN_REG_FILE}" ] && cmp -s "${_tn_reg_tmp}" "${_TN_REG_FILE}"; then + rm -f "${_tn_reg_tmp}" + $SUDO chmod 0600 "${_TN_REG_FILE}" + info "Registry mirror ${_TN_REG_HOST} already configured" + else + $SUDO chmod 0600 "${_tn_reg_tmp}" + $SUDO mv -f "${_tn_reg_tmp}" "${_TN_REG_FILE}" || { rm -f "${_tn_reg_tmp}"; fatal "Cannot write ${_TN_REG_FILE}"; } + info "Registry mirror ${_TN_REG_HOST} configured (${_TN_REG_FILE})" + fi + unset _tn_reg_tmp _tn_reg_m +fi +unset _TN_REG_USER _TN_REG_PASS download k3s_installer.sh https://get.k3s.io $SUDO chmod +x k3s_installer.sh