diff --git a/cmd/quarry-sysupdate/transfers.go b/cmd/quarry-sysupdate/transfers.go new file mode 100644 index 0000000..fe5024e --- /dev/null +++ b/cmd/quarry-sysupdate/transfers.go @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Amutable GmbH + +package sysupdate + +import ( + "context" + "errors" + "fmt" + "log/slog" + "maps" + "slices" + + "go.amutable.dev/quarry/internal/systemdcmd" + "go.amutable.dev/quarry/internal/transferlayout" + "go.amutable.dev/quarry/internal/xsysupdate" +) + +// sysupdateRunArgs returns the systemd-run arguments that run systemd-sysupdate +// with the given arguments in a transient service that sees the staged +// definitions through read-only bind mounts (see +// [xsysupdate.StagedTransfers.BindPaths]). +func sysupdateRunArgs(bindPaths []string, args ...string) []string { + runArgs := []string{"--collect", "--pipe"} + for _, bind := range bindPaths { + runArgs = append(runArgs, "-p", "BindReadOnlyPaths="+bind) + } + runArgs = append(runArgs, systemdcmd.FindCmd("sysupdate")) + return append(runArgs, args...) +} + +// stageAndUpdate stages the definitions of every component with transfer +// definitions for the given machine tags and updates them in one +// systemd-sysupdate run (see [updateStaged]). +func stageAndUpdate(ctx context.Context, ext *xsysupdate.TransferFileExtension, machineTags []string) error { + // Only components with transfer definitions are updated. A component + // without any is not managed by us. + transfers := ext.Transfers() + if len(transfers) == 0 { + slog.Info("No transfer definitions found -- nothing to update.") + return nil + } + + var errs []error + components := slices.Sorted(maps.Keys(transfers)) + slog.Info("Staging transfer definitions for 'systemd-sysupdate update'...", + "components", len(components), "machineTags", machineTags) + var staged []*xsysupdate.StagedTransfers + for _, component := range components { + componentDir := transferlayout.ComponentDir(component) // for log messages + sc, err := ext.Stage(ctx, transfers[component], xsysupdate.StageOptions{MachineTags: machineTags}) + if err != nil { + slog.Error("Staging the component's transfer definitions failed.", + "err", err.Error(), "component", componentDir) + // FIXME: If we hit an error, continue on because we don't have + // a decent revert story at the moment. + errs = append(errs, err) + continue + } + if sc == nil { + continue // not enabled on this machine + } + slog.Info("Staged transfer definitions.", + "component", componentDir, "path", sc.Path(), "enabledFeatures", sc.EnabledFeatures) + staged = append(staged, sc) + } + if len(staged) > 0 { + // There is one run for everything staged. The hook units merge + // the extensions and relink the UKI after each component. + // TODO: Make sure we control this better to only do it at the end + // after everything was successfully updated. + slog.Info("Spawning 'systemd-sysupdate update' with the staged transfer definitions...", + "components", len(staged)) + if err := updateStaged(ctx, staged); err != nil { + slog.Error("sysupdate failed.", "err", err.Error()) + errs = append(errs, err) + } + } + if err := errors.Join(errs...); err != nil { + return fmt.Errorf("sysupdate failed: %w", err) + } + slog.Info("'systemd-sysupdate update' completed.") + return nil +} + +// updateStaged runs systemd-sysupdate once over all staged components. Their +// definitions are bind-mounted into a transient service, where +// "--component-all update" updates the default component and each enabled +// component. +func updateStaged(ctx context.Context, staged []*xsysupdate.StagedTransfers) (Err error) { + var bindPaths []string + for _, sc := range staged { + bindPaths = append(bindPaths, sc.BindPaths()...) + } + defer func() { + failed := Err != nil + for _, sc := range staged { + if failed { + // Keep the staged definitions for debugging. They are pruned + // at the start of the next run. + slog.Warn("Keeping staged transfer definitions of failed update.", + "component", transferlayout.ComponentDir(sc.Component), "path", sc.Path()) + Err = errors.Join(Err, sc.Close()) + } else { + Err = errors.Join(Err, sc.Remove()) + } + } + }() + + // TODO: Pass --cleanup=yes to garbage-collect files that are no longer + // covered by this component's transfers, and also explicitly run + // "systemd-sysupdate -C cleanup" for components that are no longer + // enabled (or have no transfer definitions anymore) on the bare host, + // which deletes everything recorded in their installdb. Both are blocked + // on the sysupdate garbage collector taking "bootctl link" references + // into account, as otherwise it would remove files still referenced from + // the ESP. + // + // systemd-run --collect --pipe -p BindReadOnlyPaths=... systemd-sysupdate --component-all update + return systemdcmd.Call(ctx, "run", sysupdateRunArgs(bindPaths, "--component-all", "update")...) +} diff --git a/cmd/quarry-sysupdate/transfers_test.go b/cmd/quarry-sysupdate/transfers_test.go new file mode 100644 index 0000000..3025e6b --- /dev/null +++ b/cmd/quarry-sysupdate/transfers_test.go @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Amutable GmbH + +package sysupdate + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "go.amutable.dev/quarry/internal/systemdcmd" +) + +func TestSysupdateRunArgs(t *testing.T) { + binds := []string{ + "/var/lib/quarry-sysupdate/xsysupdate/transfers/sysupdate.foo.d/sysupdate.foo.d:/run/sysupdate.foo.d", + "/var/lib/quarry-sysupdate/xsysupdate/transfers/sysupdate.foo.d/sysupdate.foo.component:/run/sysupdate.foo.component", + } + sysupdate := systemdcmd.FindCmd("sysupdate") + assert.Equal(t, []string{ + "--collect", "--pipe", + "-p", "BindReadOnlyPaths=" + binds[0], + "-p", "BindReadOnlyPaths=" + binds[1], + sysupdate, "--component-all", "update", + }, sysupdateRunArgs(binds, "--component-all", "update")) + + // There are no -p arguments at all without bind paths. + assert.Equal(t, []string{"--collect", "--pipe", sysupdate, "update"}, sysupdateRunArgs(nil, "update")) +} diff --git a/cmd/quarry-sysupdate/update.go b/cmd/quarry-sysupdate/update.go index 9d178dc..8157544 100644 --- a/cmd/quarry-sysupdate/update.go +++ b/cmd/quarry-sysupdate/update.go @@ -5,6 +5,7 @@ package sysupdate import ( "context" + "errors" "fmt" "log/slog" "slices" @@ -24,10 +25,11 @@ var updateCommand = withExtensionDirFlag(withQuarryUserFlag(withQuarryProxyFlag( Name: "update", Usage: "extended 'systemd-sysupdate update' wrapper", Action: func(ctx context.Context, _ *cli.Command) (Err error) { + transfersExt := &xsysupdate.TransferFileExtension{ + OverrideSourcePathURL: ctxQuarryProxyURL(ctx), + } exts := xsysupdate.ExtensionSet{ - &xsysupdate.TransferFileExtension{ - OverrideSourcePathURL: ctxQuarryProxyURL(ctx), - }, + transfersExt, } ctx, err := exts.DoInit(ctx) if err != nil { @@ -35,6 +37,26 @@ var updateCommand = withExtensionDirFlag(withQuarryUserFlag(withQuarryProxyFlag( } slog.Info("Initialised extensions.", "extensions", exts.String()) + // A failure before the components are updated leaves nothing + // installed, so the extensions are aborted (removing the staging + // directory). Afterwards the state is kept (see the FIXME below) and + // the extensions are only closed. + updating, closed := false, false + defer func() { + switch { + case Err == nil || closed: + case !updating: + // A failed Do* call has aborted already, and aborting again + // does nothing. + if err := exts.DoAbort(context.WithoutCancel(ctx), Err); err != nil { + Err = errors.Join(Err, fmt.Errorf("abort extensions: %w", err)) + } + default: + if err := exts.Close(); err != nil { + Err = errors.Join(Err, fmt.Errorf("post-update extension cleanup: %w", err)) + } + } + }() // Legacy-prefixed targets are only applied after all other targets so // we can allow them to be overridden by the non-legacy prefix. // TODO: Drop this. @@ -95,40 +117,36 @@ var updateCommand = withExtensionDirFlag(withQuarryUserFlag(withQuarryProxyFlag( // FIXME FIXME FIXME // // If these updates error out we are kind of in a messy position. - // Because we need to call sysupdate multiple times to update each - // component, it is possible that one component installation fails. We + // sysupdate updates the components one after the other, so one + // component installation can fail after another succeeded. We // would ideally never want to allow execution with mixed repo states, // but fixing this requires having logic for cleanup so we can abort // the extension application, clean up the successfully installed // components, and force-downgrade to the old versions. - // A refreshed confext may ship machine-tags.d(5) files, and features gated on - // those tags were evaluated against the old tags. Loop until the tags - // settle. + // The machine tags decide which components and features are enabled + // while staging the definitions, because sysupdate does not enable + // suggested ones by itself (see xsysupdate.TransferFileExtension.Stage). + tags, err := hostnamed.CurrentTags(ctx, "") + if err != nil { + return fmt.Errorf("fetch machine tags before update: %w", err) + } + + // A refreshed confext may ship machine-tags.d(5) files, and features gated + // on those tags were staged with the old tags. Loop until the tags + // settle, staging the definitions again with the new tags for each pass. // TODO: The real signal to stop looping is whether the set of // installable features stopped changing (rather than the tags), but // there isn't a really nice way for us to detect that so we use tags // as a proxy. In future, sysupdate will handle the loop for us (and // thus be responsible for this detection) so this can all be removed // then. - tags, err := hostnamed.CurrentTags(ctx, "") - if err != nil { - return fmt.Errorf("fetch machine tags before update: %w", err) - } - - // Pick up tags the user set by hand, not only ones a confext ships. - if err := enableSuggested(ctx); err != nil { - return err - } - if err := sysupdateAll(ctx); err != nil { - return err - } - - if err := exts.Close(); err != nil { - return fmt.Errorf("post-update extension cleanup: %w", err) - } - + updating = true for iteration := 1; ; iteration++ { + if err := stageAndUpdate(ctx, transfersExt, tags); err != nil { + return err + } + // Do online-reload of sysexts and confexts. slog.Info("Trigger online-reload of new components.") if err := sysupdateRefresh(ctx); err != nil { @@ -140,22 +158,22 @@ var updateCommand = withExtensionDirFlag(withQuarryUserFlag(withQuarryProxyFlag( return fmt.Errorf("fetch machine tags after refresh: %w", err) } if slices.Equal(tags, newTags) { - return nil + break } if iteration >= maxUpdateIterations { slog.Warn("Maximum number of update iterations reached, not looking for further updates.", "max", maxUpdateIterations, "before", tags, "after", newTags) - return nil + break } slog.Info("Machine tags changed by refresh, running another update pass.", "iteration", iteration, "before", tags, "after", newTags) tags = newTags - if err := enableSuggested(ctx); err != nil { - return err - } - if err := sysupdateAll(ctx); err != nil { - return err - } } + + closed = true + if err := exts.Close(); err != nil { + return fmt.Errorf("post-update extension cleanup: %w", err) + } + return nil }, }))) diff --git a/cmd/quarry-sysupdate/utils_sysupdate.go b/cmd/quarry-sysupdate/utils_sysupdate.go index 6522cd1..0c5c490 100644 --- a/cmd/quarry-sysupdate/utils_sysupdate.go +++ b/cmd/quarry-sysupdate/utils_sysupdate.go @@ -6,87 +6,10 @@ package sysupdate import ( "context" "errors" - "fmt" - "log/slog" - "os/exec" - "go.amutable.dev/quarry/internal/jsonutils" "go.amutable.dev/quarry/internal/systemdcmd" ) -const defaultComponent = "" // copied from `systemd-sysupdate components` - -func listComponents(ctx context.Context) ([]string, error) { - sysupdatePath := systemdcmd.FindCmd("sysupdate") - // systemd-sysupdate --json=short components - cmd := exec.CommandContext(ctx, - sysupdatePath, "--json=short", "components") - - slog.Info(fmt.Sprintf("[exec] %s", cmd)) - data, err := cmd.Output() - if err != nil { - return nil, fmt.Errorf("get list of sysupdate components (%s): %w", cmd, err) - } - - type outputType struct { - Default bool `json:"default"` - Components []string `json:"components"` - } - out, err := jsonutils.Parse[outputType](data) - if err != nil { - return nil, fmt.Errorf("(%s) produced invalid json: %w", cmd, err) - } - components := out.Components - if out.Default { - components = append(components, defaultComponent) - } - return components, nil -} - -func sysupdate(ctx context.Context, component string) error { - // systemd-sysupdate [--component=$component] update - args := make([]string, 0, 2) - if component != defaultComponent { - args = append(args, "--component="+component) - } - args = append(args, "update") - return systemdcmd.Call(ctx, "sysupdate", args...) -} - -// sysupdateAll runs "systemd-sysupdate update" for every component. -func sysupdateAll(ctx context.Context) error { - components, err := listComponents(ctx) - if err != nil { - return fmt.Errorf("get list of sysupdate components: %w", err) - } - slog.Info("Spawning 'systemd-sysupdate update'...", - "components", components) - var errs []error - for _, component := range components { - if err := sysupdate(ctx, component); err != nil { - slog.Error("sysupdate for component failed.", - "err", err.Error(), "component", component) - // FIXME: If we hit an error, continue on because we don't have - // a decent revert story at the moment. - errs = append(errs, err) - } - } - if err := errors.Join(errs...); err != nil { - return fmt.Errorf("sysupdate failed: %w", err) - } - slog.Info("'systemd-sysupdate update' completed.") - return nil -} - -// enableSuggested enables the components and features whose SuggestOn* -// conditions now hold, e.g. after the machine tags changed. -func enableSuggested(ctx context.Context) error { - if err := systemdcmd.Call(ctx, "systemctl", "start", "systemd-sysupdate-auto-enable.service"); err != nil { - return fmt.Errorf("failed to trigger systemd-sysupdate-auto-enable service: %w", err) - } - return nil -} - func sysupdateRefresh(ctx context.Context) error { errs := []error{ // systemd-confext refresh diff --git a/internal/pathrsext/write_linux.go b/internal/pathrsext/write_linux.go new file mode 100644 index 0000000..779d16d --- /dev/null +++ b/internal/pathrsext/write_linux.go @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Amutable GmbH + +package pathrsext + +import ( + "errors" + "fmt" + "io/fs" + "path/filepath" + "strings" + + "cyphar.com/go-pathrs" + "golang.org/x/sys/unix" + + "go.amutable.dev/quarry/internal/third_party/funchelpers" +) + +// WriteFile writes data to path inside root, creating parent directories as +// needed. The data is written to an unlinked temporary file first and only +// attached under its name once synced. An existing file is unlinked before +// that, so the replacement is not atomic. A failed attach leaves no file. +func WriteFile(root *pathrs.Root, path string, data []byte) (Err error) { + if dir, _ := filepath.Split(path); dir != "" { //nolint:forbidigo // lexical path to be passed to libpathrs + dir = strings.TrimSuffix(dir, "/") + handle, err := root.MkdirAll(dir, 0o755) + if err != nil { + return fmt.Errorf("mkdir %s: %w", dir, err) + } + if err := handle.Close(); err != nil { + return fmt.Errorf("close %s: %w", dir, err) + } + } + tmpFile, err := root.Create(".", unix.O_TMPFILE|unix.O_RDWR|unix.O_NOFOLLOW, 0o644) + if err != nil { + return fmt.Errorf("create tmpfile for %s: %w", path, err) + } + defer funchelpers.VerifyClose(&Err, tmpFile) + if _, err := tmpFile.Write(data); err != nil { + return fmt.Errorf("write tmpfile for %s: %w", path, err) + } + if err := tmpFile.Sync(); err != nil { + return fmt.Errorf("sync tmpfile for %s: %w", path, err) + } + // linkat(2) does not replace, so make room for the new file first. + if err := root.RemoveFile(path); err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("remove old %s: %w", path, err) + } + if err := AttachIntoRoot(root, path, tmpFile); err != nil { + return fmt.Errorf("attach tmpfile as %s: %w", path, err) + } + return nil +} diff --git a/internal/xsysupdate/suggest.go b/internal/xsysupdate/suggest.go new file mode 100644 index 0000000..b5b660b --- /dev/null +++ b/internal/xsysupdate/suggest.go @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Amutable GmbH + +package xsysupdate + +import ( + "fmt" + "maps" + "path" + "slices" + "strings" +) + +// suggestOnMachineTag is the only SuggestOn*= condition that quarry evaluates. +const suggestOnMachineTag = "SuggestOnMachineTag" + +// assignment is one "Key=Value" line of a definition. +type assignment struct{ key, value string } + +// sectionAssignments returns the assignments in the [section] sections of a +// systemd-style definition in file order. Unlike [loadINI], it keeps empty +// assignments because they reset a list. +func sectionAssignments(data []byte, section string) []assignment { + var ( + assignments []assignment + inSection bool + ) + for line := range strings.SplitSeq(string(data), "\n") { + line = strings.TrimSpace(line) + switch { + case line == "" || line[0] == '#' || line[0] == ';': + case line[0] == '[': + inSection = line == "["+section+"]" + case inSection: + if key, value, ok := strings.Cut(line, "="); ok { + assignments = append(assignments, assignment{key: strings.TrimSpace(key), value: strings.TrimSpace(value)}) + } + } + } + return assignments +} + +// parseBoolean parses a boolean value the way systemd does. ok is false if +// the value is no boolean. +func parseBoolean(value string) (b, ok bool) { + switch strings.ToLower(value) { + case "1", "yes", "y", "true", "t", "on": + return true, true + case "0", "no", "n", "false", "f", "off": + return false, true + } + return false, false +} + +// definitionEnabled returns the Enabled= setting of the [section] of a +// component or feature definition. It is def if the definition sets none. As +// in systemd, the last valid assignment wins and invalid ones are ignored. +func definitionEnabled(data []byte, section string, def bool) bool { + enabled := def + for _, a := range sectionAssignments(data, section) { + if a.key != "Enabled" { + continue + } + if b, ok := parseBoolean(a.value); ok { + enabled = b + } + } + return enabled +} + +// definitionSuggested returns whether the [section] of a component or feature +// definition is suggested for a machine with the given tags. The +// SuggestOnMachineTag= conditions are evaluated like ConditionMachineTag= +// (see [machineTagConditionsHold]). Without any condition the definition is +// not suggested. quarry does not evaluate Suggest= or the other SuggestOn*= +// conditions. Suggest= is ignored, and an error is returned for a definition +// that uses another SuggestOn*= condition. +func definitionSuggested(data []byte, section string, machineTags []string) (bool, error) { + conditions := map[string][]string{} + for _, a := range sectionAssignments(data, section) { + if !strings.HasPrefix(a.key, "SuggestOn") { + continue + } + // An empty value resets the conditions of this key. + if a.value == "" { + delete(conditions, a.key) + } else { + conditions[a.key] = append(conditions[a.key], a.value) + } + } + for _, key := range slices.Sorted(maps.Keys(conditions)) { + if key != suggestOnMachineTag { + return false, fmt.Errorf("quarry does not evaluate %s=", key) + } + } + values := conditions[suggestOnMachineTag] + if len(values) == 0 { + return false, nil + } + return machineTagConditionsHold(values, machineTags), nil +} + +// machineTagConditionsHold evaluates the values of SuggestOnMachineTag= as a +// systemd condition list of ConditionMachineTag=. A condition holds if its +// shell glob pattern matches any of the machine tags ("key" or "key=value"). +// A leading "!" negates the condition. A leading "|" makes it a triggering +// condition. The list holds if all non-triggering conditions hold and, if +// there are triggering conditions, at least one of them holds. +func machineTagConditionsHold(values, machineTags []string) bool { + const whitespace = " \t\n\r" + hasTrigger, triggered := false, false + for _, value := range values { + trigger := strings.HasPrefix(value, "|") + if trigger { + value = strings.TrimLeft(value[1:], whitespace) + } + negate := strings.HasPrefix(value, "!") + if negate { + value = strings.TrimLeft(value[1:], whitespace) + } + // Tags contain no "/", so path.Match behaves like fnmatch(3) with no + // flags, as systemd uses it. An invalid pattern matches nothing. + matched := slices.ContainsFunc(machineTags, func(tag string) bool { + ok, err := path.Match(value, tag) + return err == nil && ok + }) + result := matched != negate + switch { + case trigger: + hasTrigger = true + triggered = triggered || result + case !result: + return false + } + } + return !hasTrigger || triggered +} diff --git a/internal/xsysupdate/suggest_test.go b/internal/xsysupdate/suggest_test.go new file mode 100644 index 0000000..62cbc36 --- /dev/null +++ b/internal/xsysupdate/suggest_test.go @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Amutable GmbH + +package xsysupdate + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDefinitionSuggested(t *testing.T) { + tags := []string{"amutable.os", "amutable.ext.docker", "acp.site=berlin"} + for _, tc := range []struct { + name string + section string + text string + suggested bool + wantErr bool + }{ + {name: "no keys", text: "[Feature]\nEnabled=false\n"}, + {name: "no section", text: "[Transfer]\nSuggestOnMachineTag=amutable.os\n"}, + {name: "other section", text: "[Feature]\n[Other]\nSuggestOnMachineTag=amutable.os\n"}, + {name: "matching tag", text: "[Feature]\nSuggestOnMachineTag=amutable.ext.docker\n", suggested: true}, + {name: "missing tag", text: "[Feature]\nSuggestOnMachineTag=amutable.ext.podman\n"}, + {name: "no prefix match", text: "[Feature]\nSuggestOnMachineTag=amutable\n"}, + {name: "glob", text: "[Feature]\nSuggestOnMachineTag=amutable.ext.*\n", suggested: true}, + {name: "glob class", text: "[Feature]\nSuggestOnMachineTag=amutable.[eo]s\n", suggested: true}, + {name: "key and value", text: "[Feature]\nSuggestOnMachineTag=acp.site=berlin\n", suggested: true}, + {name: "key of tag with value", text: "[Feature]\nSuggestOnMachineTag=acp.site\n"}, + {name: "key glob of tag with value", text: "[Feature]\nSuggestOnMachineTag=acp.site=*\n", suggested: true}, + {name: "invalid pattern", text: "[Feature]\nSuggestOnMachineTag=amutable.[\n"}, + {name: "all must hold", text: "[Feature]\nSuggestOnMachineTag=amutable.os\nSuggestOnMachineTag=amutable.k8s\n"}, + {name: "all hold", text: "[Feature]\nSuggestOnMachineTag=amutable.os\nSuggestOnMachineTag=amutable.ext.docker\n", suggested: true}, + {name: "negated", text: "[Feature]\nSuggestOnMachineTag=!amutable.os\n"}, + {name: "negated missing", text: "[Feature]\nSuggestOnMachineTag=!amutable.k8s\n", suggested: true}, + {name: "negated with space", text: "[Feature]\nSuggestOnMachineTag=! amutable.k8s\n", suggested: true}, + {name: "one trigger holds", text: "[Feature]\nSuggestOnMachineTag=|amutable.k8s\nSuggestOnMachineTag=|amutable.os\n", suggested: true}, + {name: "no trigger holds", text: "[Feature]\nSuggestOnMachineTag=|amutable.k8s\nSuggestOnMachineTag=|amutable.gw\n"}, + {name: "trigger and failing condition", text: "[Feature]\nSuggestOnMachineTag=|amutable.os\nSuggestOnMachineTag=amutable.k8s\n"}, + {name: "negated trigger", text: "[Feature]\nSuggestOnMachineTag=| !amutable.k8s\nSuggestOnMachineTag=|amutable.gw\n", suggested: true}, + {name: "reset", text: "[Feature]\nSuggestOnMachineTag=amutable.k8s\nSuggestOnMachineTag=\nSuggestOnMachineTag=amutable.os\n", suggested: true}, + {name: "reset to nothing", text: "[Feature]\nSuggestOnMachineTag=amutable.os\nSuggestOnMachineTag=\n"}, + {name: "suggest ignored", text: "[Feature]\nSuggest=yes\nSuggestOnMachineTag=amutable.k8s\n"}, + {name: "unsupported condition", text: "[Feature]\nSuggestOnMachineTag=amutable.os\nSuggestOnArchitecture=x86-64\n", wantErr: true}, + {name: "unsupported condition reset", text: "[Feature]\nSuggestOnArchitecture=x86-64\nSuggestOnArchitecture=\nSuggestOnMachineTag=amutable.os\n", suggested: true}, + {name: "comments and spaces", text: "# x\n[Feature]\n; SuggestOnMachineTag=amutable.k8s\n SuggestOnMachineTag = amutable.os \n", suggested: true}, + {name: "component", section: "Component", text: "[Component]\nSuggestOnMachineTag=amutable.os\n", suggested: true}, + } { + section := tc.section + if section == "" { + section = "Feature" + } + suggested, err := definitionSuggested([]byte(tc.text), section, tags) + if tc.wantErr { + require.Error(t, err, tc.name) + continue + } + require.NoError(t, err, tc.name) + assert.Equal(t, tc.suggested, suggested, tc.name) + } +} + +func TestDefinitionEnabled(t *testing.T) { + for _, tc := range []struct { + text string + def bool + expected bool + }{ + {text: "[Component]\n", def: true, expected: true}, + {text: "[Component]\n", def: false, expected: false}, + {text: "[Component]\nEnabled=false\n", def: true, expected: false}, + {text: "[Component]\nEnabled=On\n", def: false, expected: true}, + {text: "[Component]\nEnabled=1\nEnabled=no\n", def: false, expected: false}, + {text: "[Component]\nEnabled=yes\nEnabled=bogus\n", def: false, expected: true}, + {text: "[Component]\nEnabled=\n", def: true, expected: true}, + {text: "[Other]\nEnabled=yes\n", def: false, expected: false}, + } { + assert.Equal(t, tc.expected, definitionEnabled([]byte(tc.text), "Component", tc.def), tc.text) + } +} diff --git a/internal/xsysupdate/transfer.go b/internal/xsysupdate/transfer.go index 12f7725..e1924d7 100644 --- a/internal/xsysupdate/transfer.go +++ b/internal/xsysupdate/transfer.go @@ -4,66 +4,121 @@ package xsysupdate import ( + "bytes" "context" "errors" "fmt" "io" "io/fs" "log/slog" - "os" + "maps" "path/filepath" + "slices" "strings" - "time" "cyphar.com/go-pathrs" - "github.com/schollz/progressbar/v3" "golang.org/x/sys/unix" "gopkg.in/ini.v1" - "go.amutable.dev/quarry/internal/ctxext" "go.amutable.dev/quarry/internal/pathrsext" "go.amutable.dev/quarry/internal/third_party/funchelpers" + "go.amutable.dev/quarry/internal/transferlayout" "go.amutable.dev/quarry/internal/tufclient" ) const ( // TransferFilePrefix is the prefix used for our "auto-applied transfer files" - // extension to sysupdate. Any target file in a repository with this prefix - // will be installed as /etc/sysupdate.* (with the [Source] Path= adjusted to - // point to the local quarry-http-client server). + // extension to sysupdate. A repository ships the sysupdate definitions of + // a component as the target files + // + // .quarry/sysupdate.d/ (default component) + // .quarry/sysupdate..d/ (component) + // + // where is a *.transfer, *.feature or .component + // definition or a drop-in. [TransferFileExtension.Stage] lays them out for + // a single systemd-sysupdate run (with the [Source] Path= adjusted to point + // to the local quarry-http-client server). TransferFilePrefix = ExtensionTargetPrefix + transferFileName // transferFileName is [TransferFilePrefix] without the // [ExtensionTargetPrefix]. transferFileName = "sysupdate." - liveLink = "live" - lastLink = "last" -) + // maxDefinitionFileSize bounds how much of a definition file is read. + maxDefinitionFileSize = 1 << 20 // 1 MiB -// These are variables so that we can modify them in tests. -var ( - // transferInstallDir is the directory where the transfer file *symlinks* - // will be installed. This needs to be one of systemd-sysupdate's search - // directories, be writable, and be persistent across reboot. - // - // On AmutableOS, only /etc fits the bill. - transferInstallDir = "/etc" // .../sysupdate{.*,}.d/ - // extensionInstallDir is where sysext and confexts are linked by default - // using CurrentSymlink. - // FIXME: This is only needed for the systemd bug workaround in Init. - extensionInstallDir = transferInstallDir + "/extensions" + // bindMountDir is where the staged definitions are bind-mounted for + // systemd-sysupdate. It is /run rather than /etc because systemd creates + // missing mount points on the host filesystem (the service only has a + // private mount table), and on /run (a tmpfs in sysupdate's search path) + // these empty placeholders do not persist. + bindMountDir = "/run" ) -// transferInstallPatterns are a set of globs that will match the symlink names -// in /etc and in /var/lib/quarry-sysupdate/xsysupdate/transfers/. -var transferInstallPatterns = [...]string{ - "sysupdate.*.d", // component sysupdate directories - "sysupdate.d/*", // base system sysupdate files +// hostDefinitionsDir is the highest-precedence sysupdate search directory on +// the host. Definitions there take precedence over our bind mounts, so we warn +// about them. This is a variable so that tests can override it. +var hostDefinitionsDir = "/etc" + +// TransferDir is the definitions directory of one sysupdate component as +// shipped by the repositories (see [TransferFilePrefix]). +type TransferDir struct { + // Component is the sysupdate component name ("" for the default + // component). + Component string + + // files are the definition files (relative to the directory) and the + // targets providing them. + files map[string]*tufclient.TargetInfo + // path is the target path of the directory, for log messages. + path string +} + +// Files returns the sorted list of definition files in the directory. +func (dir *TransferDir) Files() []string { + return slices.Sorted(maps.Keys(dir.files)) +} + +// String returns the target path of the directory. +func (dir *TransferDir) String() string { return dir.path } + +// componentFileName returns the name of the component definition file of a +// non-default component ("sysupdate..component"). +func componentFileName(component string) string { + return "sysupdate." + component + ".component" +} + +// ParseTransferPath parses a target path of the transfer definition directory +// extension into the component ("" for the default component) and the file +// path relative to the directory. ok is false if the target is not part of +// this extension at all. err is non-nil if it is part of it but is malformed. +func ParseTransferPath(path string) (component, file string, ok bool, err error) { + rest, _, ok := CutExtensionTargetPrefix(path) + if !ok || !strings.HasPrefix(rest, transferFileName) { + return "", "", false, nil + } + dirName, file, found := strings.Cut(rest, "/") + if !found || file == "" { + return "", "", true, fmt.Errorf("transfer target %s has no file component", path) + } + for part := range strings.SplitSeq(file, "/") { + if !transferlayout.ValidPathComponent(part) { + return "", "", true, fmt.Errorf("transfer target %s has an invalid file path %q", path, file) + } + } + component, err = transferlayout.ParseComponentDir(dirName) + if err != nil { + return "", "", true, fmt.Errorf("transfer target %s: %w", path, err) + } + return component, file, true, nil } // TransferFileExtension is an [Extension] that implements auto-installation of // target files from Quarry repositories. +// It collects the transfer definition directories (see [TransferFilePrefix]). +// quarry-sysupdate stages each component from +// [TransferFileExtension.Transfers] for systemd-sysupdate with +// [TransferFileExtension.Stage]. type TransferFileExtension struct { // OverrideSourcePathURL (if non-nil) is a URL which replaces all [Source] // Path entries in transfer files when persisting them to disk. This is @@ -71,17 +126,11 @@ type TransferFileExtension struct { // sysupdate requests. OverrideSourcePathURL *string - // storeDir is the storage for transfer files in general. + // storeDir is the storage for the staged transfer files. storeDir *pathrs.Root - // subdirectory in storeDir where the previous update's files live. - oldDirSubpath string - // subdirectory in which our new transfer files live. - newDir *pathrs.Root - newDirSubpath string - - // modifiedEtc is set to true once BeforeUpdate has been called and we have - // modified the global /etc (meaning on Abort we need to undo our changes). - modifiedEtc bool + + // dirs are the collected transfer directories, keyed by component. + dirs map[string]*TransferDir } var _ Extension = &TransferFileExtension{} @@ -89,79 +138,10 @@ var _ Extension = &TransferFileExtension{} // Name returns the name of this extension. func (ext *TransferFileExtension) Name() string { return "transfers" } -func (ext *TransferFileExtension) unlinkTransfers() error { - storeDirPrefix := ext.storeDir.IntoFile().Name() + "/" - for _, pattern := range transferInstallPatterns { - pattern = filepath.Join(transferInstallDir, pattern) //nolint:forbidigo // host-controlled directory - matches, err := filepath.Glob(pattern) //nolint:forbidigo // host-controlled directory - if err != nil { - return fmt.Errorf("could not glob %q pattern: %w", pattern, err) - } - for _, match := range matches { - linkTarget, err := os.Readlink(match) - if err != nil { - if errors.Is(err, unix.EINVAL) { - // Not a symlink. - slog.Info("[xsysupdate transfers] Skipping non-symlink transfer file.", - "transfer", match) - } else { - // Some other error. - slog.Warn("[xsysupdate transfers] Skipping otherwise-invalid transfer file.", - "transfer", match, "err", err.Error()) - } - continue - } - if !strings.HasPrefix(linkTarget, storeDirPrefix) { - slog.Info("[xsysupdate transfers] Skipping non-quarry transfer file link.", - "transfer", match, "linkTarget", linkTarget, "wantPrefix", storeDirPrefix) - continue - } - if err := os.Remove(match); err != nil { //nolint:forbidigo // host-controlled directory with fixed patterns - return fmt.Errorf("remove old quarry transfer file link %s: %w", match, err) - } - } - } - return nil -} - -// linkTransfers takes an xsysypdate/transfers/* directory full of transfer -// files and links them into [transferInstallDir]. -func (ext *TransferFileExtension) linkTransfers(subdir string) error { - srcTransferDir := filepath.Join(ext.storeDir.IntoFile().Name(), subdir) //nolint:forbidigo // TODO: Implement pathrsext.Glob. - slog.Info("[xsysupdate transfers] Linking auto-enable transfers into host sysupdate directory.", - "transferDir", srcTransferDir, "installDir", transferInstallDir) - for _, pattern := range transferInstallPatterns { - pattern = filepath.Join(srcTransferDir, pattern) //nolint:forbidigo // TODO: Implement pathrsext.Glob. - matches, err := filepath.Glob(pattern) //nolint:forbidigo // TODO: Implement pathrsext.Glob. - if err != nil { - return fmt.Errorf("could not glob %q pattern: %w", pattern, err) - } - for _, match := range matches { - transferName, hasPrefix := strings.CutPrefix(match, srcTransferDir) - if !hasPrefix { - // Should never happen! - return fmt.Errorf("match %q is missing transfer dir prefix %q", match, srcTransferDir) - } - // TODO: Ideally we would do this through a pathrs.Root handle to - // /etc, to avoid all of these fairly concerning path operations. - // But in practice, the danger of writing to the wrong /etc file is - // just as bad as writing outside of /etc... - linkname := filepath.Join(transferInstallDir, transferName) //nolint:forbidigo // restricted lexical pathname in host-controlled directory - if err := os.MkdirAll(filepath.Dir(linkname), 0o755); err != nil { //nolint:forbidigo // host-controlled directory // TODO: We should do this with pathrs.Root anyway... - return fmt.Errorf("make transfer %s directory: %w", match, err) - } - if err := os.Symlink(match, linkname); err != nil { //nolint:forbidigo // host-controlled directory // TODO: We should do this with pathrs.Root anyway... - return fmt.Errorf("link transfer %s: %w", match, err) - } - slog.Info("[xsysupdate transfers] Linked auto-enable transfer.", - "transfer", match, "link", linkname) - } - } - return nil -} - // Init constructs the storage area for the auto-enabled transfer files. -func (ext *TransferFileExtension) Init(ctx context.Context) (_ context.Context, Err error) { +// Staging directories left behind by earlier runs (they are kept on failure, +// for debugging) are removed. +func (ext *TransferFileExtension) Init(ctx context.Context) (context.Context, error) { ctx, err := makeExtStoreDir(ctx, ext) if err != nil { return nil, fmt.Errorf("make %s extension store: %w", ext.Name(), err) @@ -169,174 +149,147 @@ func (ext *TransferFileExtension) Init(ctx context.Context) (_ context.Context, ext.storeDir = ctxExtStoreDir(ctx, ext) // storeDir will be closed by Abort on error. - // FIXME: sysupdate does not auto-create the parent directory of - // CurrentSymlink=, which causes update failures.See - // . - if err := os.MkdirAll(extensionInstallDir, 0o755); err != nil { //nolint:forbidigo // fixed host-controlled path - return nil, err - } - - // Get the current "live" directory subpath for recovery purposes in Abort. - // If there is no such symlink, ignore the error as that is the initial - // state and we just need to skip linkTransfers in the Abort path. - if subpath, err := ext.storeDir.Readlink(liveLink); errors.Is(err, fs.ErrNotExist) { - slog.Info("[xsysupdate transfers init] No existing 'live' link found -- Abort will not restore anything.", - "err", err.Error()) - } else if err != nil { - return nil, fmt.Errorf("old %s link is invalid: %w", liveLink, err) - } else { - ext.oldDirSubpath = subpath - // Move the "live" link to "last", for abort purposes. - if err := ext.storeDir.Rename(liveLink, lastLink, 0); err != nil { - return nil, fmt.Errorf("swap %s link to %s link: %w", liveLink, lastLink, err) - } - } - - // Create a new time-based directory path. - refTime, _ := ctxext.RefTime(ctx) - ext.newDirSubpath = fmt.Sprintf("update-%s.%d", refTime.Format(time.DateOnly), refTime.UnixMilli()) - - newDirHandle, err := ext.storeDir.MkdirAll(ext.newDirSubpath, 0o755) - if err != nil { - return nil, fmt.Errorf("create new update subdir: %w", err) - } - defer funchelpers.VerifyClose(&Err, newDirHandle) - - ext.newDir, err = pathrs.RootFromFile(newDirHandle.IntoFile()) - if err != nil { - return nil, fmt.Errorf("convert update subdir to root: %w", err) + if err := ext.pruneStore(); err != nil { + return nil, fmt.Errorf("prune old staging directories: %w", err) } - // newDir will be closed by Abort on error. + ext.dirs = map[string]*TransferDir{} return ctx, nil } -// patchTransferFile writes out a transfer file, after rewriting the source -// path with OverrideSourcePathURL (if set). -func (ext *TransferFileExtension) patchTransferFile(wtr io.Writer, rdr io.Reader) error { - transfer, err := ini.Load(rdr) +// pruneStore removes everything in the store directory. Staged definitions +// are only needed while their update runs, so anything present at the start +// of a run is a leftover from a failed (or interrupted) one. +func (ext *TransferFileExtension) pruneStore() (Err error) { + dir, err := ext.storeDir.OpenFile(".", unix.O_DIRECTORY) if err != nil { - return fmt.Errorf("load transfer file ini: %w", err) + return fmt.Errorf("open store: %w", err) } - if overrideURL := ext.OverrideSourcePathURL; overrideURL != nil { - // This allocates a new Source section and Path value if missing. - transfer.Section("Source").Key("Path").SetValue(*overrideURL) + defer funchelpers.VerifyClose(&Err, dir) + + entries, err := dir.ReadDir(-1) + if err != nil { + return fmt.Errorf("list store: %w", err) } - if _, err := transfer.WriteTo(wtr); err != nil { - return fmt.Errorf("serialise transfer file: %w", err) + for _, entry := range entries { + slog.Info("[xsysupdate transfers] Removing staging directory.", + "name", entry.Name()) + if err := ext.storeDir.RemoveAll(entry.Name()); err != nil { + return fmt.Errorf("remove staging directory %s: %w", entry.Name(), err) + } } return nil } -// ApplyTarget saves the transfer file from the repository to the per-update -// transfer directory. -func (ext *TransferFileExtension) ApplyTarget(ctx context.Context, info *tufclient.TargetInfo) (_ bool, Err error) { - transferPath, _, ok := CutExtensionTargetPrefix(info.Path) - if !ok || !strings.HasPrefix(transferPath, transferFileName) { - return false, nil // not for this extension +// dir returns the (possibly new) transfer directory record for the given +// component, whose target path is path. +func (ext *TransferFileExtension) dir(component, path string) *TransferDir { + dir, ok := ext.dirs[component] + if !ok { + dir = &TransferDir{ + Component: component, + files: map[string]*tufclient.TargetInfo{}, + path: path, + } + ext.dirs[component] = dir } + return dir +} - slog.Info("[xsysupdate transfers target] Applying transfer file.", - "target", info.Path, "repository", info.Repo.Name) - - srcTransferFile, err := info.Fetch(ctx) +// readTarget fetches and verifies a (small) target file. +func readTarget(ctx context.Context, info *tufclient.TargetInfo, maxSize int64) (_ []byte, Err error) { + file, err := info.Fetch(ctx) if err != nil { - return false, fmt.Errorf("fetch transfer file %s: %w", info.Path, err) + return nil, fmt.Errorf("fetch target %s: %w", info.Path, err) } - defer funchelpers.VerifyClose(&Err, srcTransferFile) + defer funchelpers.VerifyClose(&Err, file) - dstTransferTmpFile, err := ext.newDir.Create(".", unix.O_TMPFILE|unix.O_RDWR|unix.O_NOFOLLOW, 0o644) + data, err := io.ReadAll(io.LimitReader(file, maxSize+1)) if err != nil { - return false, fmt.Errorf("create tmpfile for transfer file %s: %w", info.Path, err) - } - defer funchelpers.VerifyClose(&Err, dstTransferTmpFile) - - srcRdr := progressbar.NewReader(srcTransferFile, - progressbar.DefaultBytes(info.Length, info.Path)) - if err := ext.patchTransferFile(dstTransferTmpFile, &srcRdr); err != nil { - return false, fmt.Errorf("write transfer file %s to tmpfile: %w", info.Path, err) + return nil, fmt.Errorf("read target %s: %w", info.Path, err) } - if err := srcTransferFile.Close(); err != nil { - return false, fmt.Errorf("transfer file %s close check failed: %w", info.Path, err) + if int64(len(data)) > maxSize { + return nil, fmt.Errorf("target %s is too large (limit %d)", info.Path, maxSize) } - if err := dstTransferTmpFile.Sync(); err != nil { - return false, fmt.Errorf("sync writes to transfer tmpfile %s: %w", info.Path, err) + // The stream is hash-verified on Close, so this must succeed before the + // data is used for anything. + if err := file.Close(); err != nil { + return nil, fmt.Errorf("verify target %s: %w", info.Path, err) } + return data, nil +} - transferPathDir, _ := filepath.Split(transferPath) //nolint:forbidigo // lexical path to be passed to libpathrs - if handle, err := ext.newDir.MkdirAll(transferPathDir, 0o755); err != nil { - return false, fmt.Errorf("mkdir transfer file parent dir %s: %w", transferPathDir, err) - } else if err := handle.Close(); err != nil { - return false, fmt.Errorf("close transfer file parent dir %s: %w", transferPathDir, err) +// ApplyTarget collects the target into its transfer directory. A malformed +// target is ignored with a warning rather than aborting the update, so that +// one bad entry cannot block the updates of other components. +func (ext *TransferFileExtension) ApplyTarget(_ context.Context, info *tufclient.TargetInfo) (bool, error) { + component, file, ok, err := ParseTransferPath(info.Path) + if !ok { + return false, nil // not for this extension } + if err != nil { + slog.Warn("[xsysupdate transfers target] Ignoring invalid transfer target.", + "target", info.Path, "repository", info.Repo.Name, "err", err.Error()) + return true, nil + } + dir := ext.dir(component, strings.TrimSuffix(info.Path, "/"+file)) - if err := pathrsext.AttachIntoRoot(ext.newDir, transferPath, dstTransferTmpFile); err != nil { - return false, fmt.Errorf("attach transfer tmpfile %s into %s/%s: %w", info.Path, ext.newDir.IntoFile().Name(), transferPath, err) + // For equal paths (e.g. from two repositories) the first one seen wins. + if _, seen := dir.files[file]; seen { + slog.Info("[xsysupdate transfers target] Ignoring duplicate transfer definition.", + "target", info.Path, "repository", info.Repo.Name) + return true, nil } + slog.Info("[xsysupdate transfers target] Collected transfer definition.", + "target", info.Path, "repository", info.Repo.Name, "component", transferlayout.ComponentDir(component)) + dir.files[file] = info return true, nil } -// BeforeUpdate adds /etc symlinks for all of the transfer files installed -// during this update, and removes any old quarry-managed transfer files. -func (ext *TransferFileExtension) BeforeUpdate(_ context.Context) (Err error) { - if err := ext.storeDir.Symlink(ext.newDirSubpath, liveLink); err != nil { - return fmt.Errorf("add %s link to %s: %w", liveLink, ext.newDirSubpath, err) - } - - // We are now modifying the host. - ext.modifiedEtc = true +// Transfers returns the collected transfer directories, keyed by component. +func (ext *TransferFileExtension) Transfers() map[string]*TransferDir { + return ext.dirs +} - // Replace the /etc/sysupdate.* links. - if err := ext.unlinkTransfers(); err != nil { - return fmt.Errorf("unlink old transfer files: %w", err) - } - if err := ext.linkTransfers(ext.newDirSubpath); err != nil { - err = fmt.Errorf("link new transfer files from %s: %w", ext.newDirSubpath, err) - // Remove any partially-added transfer files. - slog.Warn("[xsysupdate transfers before-update] Removing partially-added transfer files on error.", - "newDir", ext.newDirSubpath, "err", err.Error()) - if err2 := ext.unlinkTransfers(); err2 != nil { - err = fmt.Errorf("%w (failure during removal of new files: %w)", err, err2) +// hostDefinitions returns the sysupdate definitions present in +// [hostDefinitionsDir], which take precedence over the staged ones. +func hostDefinitions() ([]string, error) { + var found []string + for _, pattern := range []string{"sysupdate.d/*", "sysupdate.*.d", "sysupdate.*.component"} { + pattern = filepath.Join(hostDefinitionsDir, pattern) //nolint:forbidigo // host-controlled directory + matches, err := filepath.Glob(pattern) //nolint:forbidigo // host-controlled directory + if err != nil { + return nil, fmt.Errorf("could not glob %q pattern: %w", pattern, err) } + found = append(found, matches...) + } + slices.Sort(found) + return found, nil +} + +// BeforeUpdate warns about sysupdate definitions on the host that would +// take precedence over the staged ones. +func (ext *TransferFileExtension) BeforeUpdate(_ context.Context) error { + found, err := hostDefinitions() + if err != nil { return err } + if len(found) > 0 { + slog.Warn("[xsysupdate transfers before-update] Host has sysupdate definitions that take precedence over the repository ones.", + "definitions", found) + } return nil } // Abort resets the transfer file state to before the update was started. +// Init left the store empty, so everything in it was staged by this run. func (ext *TransferFileExtension) Abort(_ context.Context, srcErr error) error { var errs []error - // Revert the old transfer files (if we modified the host). - if ext.modifiedEtc { - slog.Info("[xsysupdate transfers abort] Reverting transfer files to old reposistory state.", - "oldDir", ext.oldDirSubpath, "err", srcErr.Error()) - if err := ext.unlinkTransfers(); err != nil { - errs = append(errs, fmt.Errorf("unlink old transfer files: %w", err)) - } - if ext.oldDirSubpath != "" { - if err := ext.linkTransfers(ext.oldDirSubpath); err != nil { - errs = append(errs, fmt.Errorf("link old transfer files from %s: %w", ext.oldDirSubpath, err)) - } - } - } - // Kill the new update directory. - if ext.newDirSubpath != "" { - slog.Info("[xsysupdate transfers abort] Removing broken transfer file directory.", - "dir", ext.newDirSubpath, "err", srcErr.Error()) - if err := ext.storeDir.RemoveAll(ext.newDirSubpath); err != nil { - errs = append(errs, fmt.Errorf("remove new transfer dir %s: %w", ext.newDirSubpath, err)) - } - } - // Swap back "last" over "live". - if ext.oldDirSubpath != "" { - slog.Info("[xsysupdate transfers abort] Swapping back last link to live link.") - if err := ext.storeDir.Rename(lastLink, liveLink, 0); err != nil { - errs = append(errs, fmt.Errorf("swap back %s link to %s link: %w", lastLink, liveLink, err)) - } - } else if ext.modifiedEtc { - slog.Info("[xsysupdate transfers abort] Removing live link (no previous update).") - if err := ext.storeDir.Remove(liveLink); err != nil { - errs = append(errs, fmt.Errorf("remove %s link: %w", liveLink, err)) + if ext.storeDir != nil { + slog.Info("[xsysupdate transfers abort] Removing staged transfer definitions.", + "err", srcErr.Error()) + if err := ext.pruneStore(); err != nil { + errs = append(errs, fmt.Errorf("remove staged transfer definitions: %w", err)) } } if err := ext.Close(); err != nil { @@ -349,22 +302,296 @@ func (ext *TransferFileExtension) Abort(_ context.Context, srcErr error) error { return errors.Join(errs...) } -// Close clears any resources used by the extension. +// Close clears any resources used by the extension. Staged definitions that +// were not removed are left in place until the next Init. func (ext *TransferFileExtension) Close() error { - var errs []error + var err error if ext.storeDir != nil { - err := ext.storeDir.Close() - if err != nil && !errors.Is(err, fs.ErrClosed) { - errs = append(errs, err) + if err = ext.storeDir.Close(); errors.Is(err, fs.ErrClosed) { + err = nil } ext.storeDir = nil } - if ext.newDir != nil { - err := ext.newDir.Close() - if err != nil && !errors.Is(err, fs.ErrClosed) { - errs = append(errs, err) + ext.dirs = nil + return err +} + +func init() { + // Write "Key=Value" like the definitions we are given (and systemd + // itself), not the library's default "Key = Value". These are package + // globals of the ini library. Nothing else in quarry writes ini files. + ini.PrettyFormat = false + ini.PrettyEqual = false +} + +// loadINI parses a systemd-style definition file. Repeated keys are kept +// verbatim (sysupdate allows e.g. multiple MatchPattern= lines) and "#" and +// ";" only start a comment at the start of a line. +func loadINI(source any) (*ini.File, error) { + return ini.LoadSources(ini.LoadOptions{ + AllowShadows: true, + AllowDuplicateShadowValues: true, + IgnoreInlineComment: true, + }, source) +} + +// setKey sets the key to exactly one value, dropping any repeated (shadow) +// assignments, as systemd would only honour the last one anyway. +func setKey(section *ini.Section, name, value string) { + section.DeleteKey(name) + section.Key(name).SetValue(value) +} + +// rewriteINI parses a definition file (see [loadINI]), applies edit to it and +// serialises the result. +func rewriteINI(data []byte, edit func(*ini.File)) ([]byte, error) { + file, err := loadINI(data) + if err != nil { + return nil, fmt.Errorf("load ini: %w", err) + } + edit(file) + var out bytes.Buffer + if _, err := file.WriteTo(&out); err != nil { + return nil, fmt.Errorf("serialise ini: %w", err) + } + return out.Bytes(), nil +} + +// enableINI stamps Enabled=true into the [section] of a definition file. +func enableINI(data []byte, section string) ([]byte, error) { + return rewriteINI(data, func(def *ini.File) { + setKey(def.Section(section), "Enabled", "true") + }) +} + +// suggestedForMachine returns whether the [section] of a component or feature +// definition from the target is suggested for a machine with the given tags +// (see [definitionSuggested]). A definition that quarry cannot evaluate is +// not suggested, which is logged as a warning. +func suggestedForMachine(info *tufclient.TargetInfo, data []byte, section string, machineTags []string) bool { + suggested, err := definitionSuggested(data, section, machineTags) + if err != nil { + slog.Warn("[xsysupdate transfers stage] Treating definition as not suggested because quarry cannot evaluate it.", + "target", info.Path, "err", err.Error()) + return false + } + return suggested +} + +// isTransferDropIn returns whether the file of a definitions directory is a +// drop-in of a transfer file (".transfer.d/.conf"). +func isTransferDropIn(file string) bool { + dir, dropIn, ok := strings.Cut(file, "/") + return ok && strings.HasSuffix(dir, ".transfer.d") && !strings.Contains(dropIn, "/") && strings.HasSuffix(dropIn, ".conf") +} + +// setsSourcePath returns whether a definition assigns [Source] Path=. +func setsSourcePath(data []byte) bool { + return slices.ContainsFunc(sectionAssignments(data, "Source"), func(a assignment) bool { + return a.key == "Path" + }) +} + +// StageOptions are the options for [TransferFileExtension.Stage]. +type StageOptions struct { + // MachineTags are the machine tags currently set on the host, which + // decide the enablement of components and features (see + // [TransferFileExtension.Stage]). + MachineTags []string +} + +// StagedTransfers are the definitions of one component laid out for a +// systemd-sysupdate run, see [TransferFileExtension.Stage]. +type StagedTransfers struct { + // Component is the staged component. + Component string + // EnabledFeatures are the features the staging enabled. + EnabledFeatures []string + + // hasComponentFile records that a component file was staged. + hasComponentFile bool + // root is the staging directory of the component. + root *pathrs.Root + // removeFn removes the staging directory. + removeFn func() error +} + +// Path returns the absolute path of the staging directory. +func (staged *StagedTransfers) Path() string { + return staged.root.IntoFile().Name() +} + +// BindPaths returns the ":" bind mounts that make +// systemd-sysupdate see the staged definitions (as BindReadOnlyPaths= +// values), which are the definitions directory of the component and, if one +// was staged, its component file. +func (staged *StagedTransfers) BindPaths() []string { + names := []string{transferlayout.ComponentDir(staged.Component)} + if staged.hasComponentFile { + names = append(names, componentFileName(staged.Component)) + } + path := staged.Path() + binds := make([]string, 0, len(names)) + for _, name := range names { + binds = append(binds, path+"/"+name+":"+bindMountDir+"/"+name) + } + return binds +} + +// Close releases the staging directory handle but leaves the directory in +// place for debugging a failed run. It is removed at the start of the next +// run. +func (staged *StagedTransfers) Close() error { + if staged.root == nil { + return nil + } + err := staged.root.Close() + staged.root = nil + if err != nil && !errors.Is(err, fs.ErrClosed) { + return err + } + return nil +} + +// Remove deletes the staging directory. +func (staged *StagedTransfers) Remove() error { + return errors.Join(staged.Close(), staged.removeFn()) +} + +// Stage lays the definitions of the given component out for a +// systemd-sysupdate run, in a staging directory of its own: +// +// /sysupdate.d/sysupdate.d/* (default component) +// /sysupdate..d/sysupdate..d/* (component) +// /sysupdate..d/sysupdate..component +// +// Transfer files get their [Source] Path= replaced by OverrideSourcePathURL, +// and so do transfer drop-ins that set one. +// +// The machine tags decide the enablement because sysupdate does not enable +// suggested components and features by itself. The default component and a +// component without a component file are enabled as for sysupdate. A +// component file that does not enable its component gets [Component] +// Enabled=true if it is suggested for the machine (see +// [definitionSuggested]). Otherwise the component is skipped, and Stage +// returns nil without an error. A feature that is not enabled gets [Feature] +// Enabled=true if it is suggested for the machine. Otherwise it is staged as +// shipped. Only the component or feature file itself is evaluated for this. +// Enabled= and SuggestOn*= in its drop-ins are not, although sysupdate still +// applies an Enabled= from a drop-in. The caller removes the staging +// directory once the run succeeded (or closes it to keep it for debugging). +func (ext *TransferFileExtension) Stage(ctx context.Context, dir *TransferDir, opts StageOptions) (_ *StagedTransfers, Err error) { + if ext.storeDir == nil { + return nil, errors.New("transfer extension is not initialised") + } + + // Everything is read first because the component file decides whether + // the component is staged at all. + files := make(map[string][]byte, len(dir.files)) + for file, info := range dir.files { + data, err := readTarget(ctx, info, maxDefinitionFileSize) + if err != nil { + return nil, err } - ext.newDir = nil + files[file] = data } - return errors.Join(errs...) + var ( + componentFile []byte + hasComponentFile bool + ) + if dir.Component != "" { + // It is staged as a sibling of the definitions directory below. + name := dir.Component + ".component" + componentFile, hasComponentFile = files[name] + delete(files, name) + if hasComponentFile && !definitionEnabled(componentFile, "Component", true) { + if !suggestedForMachine(dir.files[name], componentFile, "Component", opts.MachineTags) { + slog.Info("[xsysupdate transfers stage] Skipping component not enabled on this machine.", + "directory", dir.path) + return nil, nil //nolint:nilnil // nil indicates a skipped component + } + var err error + if componentFile, err = enableINI(componentFile, "Component"); err != nil { + return nil, fmt.Errorf("patch %s: %w", dir.files[name].Path, err) + } + slog.Info("[xsysupdate transfers stage] Enabling suggested component.", + "directory", dir.path) + } + } + + subdir := transferlayout.ComponentDir(dir.Component) + if err := ext.storeDir.RemoveAll(subdir); err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("remove old staging dir %s: %w", subdir, err) + } + handle, err := ext.storeDir.MkdirAll(subdir, 0o755) + if err != nil { + return nil, fmt.Errorf("create staging dir %s: %w", subdir, err) + } + defer funchelpers.VerifyClose(&Err, handle) + root, err := pathrs.RootFromFile(handle.IntoFile()) + if err != nil { + return nil, fmt.Errorf("convert staging dir %s to root: %w", subdir, err) + } + staged := &StagedTransfers{ + Component: dir.Component, + hasComponentFile: hasComponentFile, + root: root, + removeFn: func() error { + if err := ext.storeDir.RemoveAll(subdir); err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("remove staging dir %s: %w", subdir, err) + } + return nil + }, + } + defer func() { + if Err != nil { + Err = errors.Join(Err, staged.Remove()) + } + }() + + slog.Info("[xsysupdate transfers stage] Staging transfer definitions.", + "directory", dir.path, "path", staged.Path()) + + for _, file := range slices.Sorted(maps.Keys(files)) { + info, data := dir.files[file], files[file] + var err error + switch { + case strings.HasSuffix(file, ".transfer"): + data, err = rewriteINI(data, func(transfer *ini.File) { + if overrideURL := ext.OverrideSourcePathURL; overrideURL != nil { + // This allocates a new Source section and Path value if missing. + setKey(transfer.Section("Source"), "Path", *overrideURL) + } + }) + case isTransferDropIn(file): + // sysupdate reads the drop-in after its transfer file, so a Path= + // here would bypass the proxy. Other drop-ins are staged verbatim. + if overrideURL := ext.OverrideSourcePathURL; overrideURL != nil && setsSourcePath(data) { + data, err = rewriteINI(data, func(dropIn *ini.File) { + setKey(dropIn.Section("Source"), "Path", *overrideURL) + }) + } + case strings.HasSuffix(file, ".feature"): + enabled := definitionEnabled(data, "Feature", false) + if !enabled && suggestedForMachine(info, data, "Feature", opts.MachineTags) { + data, err = enableINI(data, "Feature") + enabled = true + staged.EnabledFeatures = append(staged.EnabledFeatures, strings.TrimSuffix(file, ".feature")) + } + slog.Info("[xsysupdate transfers stage] Decided feature enablement.", + "feature", file, "enabled", enabled) + } + if err != nil { + return nil, fmt.Errorf("patch %s: %w", info.Path, err) + } + if err := pathrsext.WriteFile(root, subdir+"/"+file, data); err != nil { + return nil, fmt.Errorf("stage %s: %w", info.Path, err) + } + } + if hasComponentFile { + if err := pathrsext.WriteFile(root, componentFileName(dir.Component), componentFile); err != nil { + return nil, fmt.Errorf("stage component file of %s: %w", dir.path, err) + } + } + return staged, nil } diff --git a/internal/xsysupdate/transfer_test.go b/internal/xsysupdate/transfer_test.go index ff18a5b..1425589 100644 --- a/internal/xsysupdate/transfer_test.go +++ b/internal/xsysupdate/transfer_test.go @@ -6,35 +6,26 @@ package xsysupdate import ( - "bytes" "context" + "crypto/sha256" "errors" - "fmt" "io/fs" "os" "path/filepath" "strings" "testing" - "time" "cyphar.com/go-pathrs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" tufmetadata "github.com/theupdateframework/go-tuf/v2/metadata" - "gopkg.in/ini.v1" - "go.amutable.dev/quarry/internal/ctxext" - "go.amutable.dev/quarry/internal/testrepo" + "go.amutable.dev/quarry/internal/transferlayout" "go.amutable.dev/quarry/internal/tufclient" "go.amutable.dev/quarry/internal/tufext" ) -var fixedRefTime = time.Date(2026, 5, 24, 12, 0, 0, 0, time.UTC) - -// Mirrors the production format string in transfer.go's Init. -var fixedNewDirSubpath = fmt.Sprintf("update-%s.%d", fixedRefTime.Format(time.DateOnly), fixedRefTime.UnixMilli()) - -const priorSubpath = "update-prior" +var testRepo = &tufext.Repository{Name: "test.example.com/repo"} // testEnv's paths are EvalSymlinks'd so they match the canonicalised names // that libpathrs caches on the corresponding [pathrs.Root] handles. @@ -49,13 +40,9 @@ func setupTestEnv(t *testing.T) (*testEnv, context.Context) { etc, err := filepath.EvalSymlinks(t.TempDir()) //nolint:forbidigo // test code require.NoError(t, err) - origTransfer, origExtension := transferInstallDir, extensionInstallDir - transferInstallDir = etc - extensionInstallDir = filepath.Join(etc, "extensions") //nolint:forbidigo // test code - t.Cleanup(func() { - transferInstallDir = origTransfer - extensionInstallDir = origExtension - }) + origHostDir := hostDefinitionsDir + hostDefinitionsDir = etc + t.Cleanup(func() { hostDefinitionsDir = origHostDir }) rootPath, err := filepath.EvalSymlinks(t.TempDir()) //nolint:forbidigo // test code require.NoError(t, err) @@ -64,7 +51,6 @@ func setupTestEnv(t *testing.T) (*testEnv, context.Context) { t.Cleanup(func() { _ = root.Close() }) ctx := context.WithValue(context.Background(), RootDirCtxKey, root) - ctx = context.WithValue(ctx, ctxext.RefTimeCtxKey, fixedRefTime) return &testEnv{ rootPath: rootPath, @@ -73,23 +59,6 @@ func setupTestEnv(t *testing.T) (*testEnv, context.Context) { }, ctx } -func (env *testEnv) populateUpdateDir(t *testing.T, subdir string, files map[string][]byte) { - t.Helper() - base := filepath.Join(env.storeDir, subdir) //nolint:forbidigo // test code - require.NoError(t, os.MkdirAll(base, 0o755)) //nolint:forbidigo // test code - for name, content := range files { - full := filepath.Join(base, name) //nolint:forbidigo // test code - require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755)) //nolint:forbidigo // test code - require.NoError(t, os.WriteFile(full, content, 0o644)) //nolint:forbidigo // test code - } -} - -func (env *testEnv) makeLive(t *testing.T) { - t.Helper() - require.NoError(t, os.MkdirAll(env.storeDir, 0o755)) //nolint:forbidigo // test code - require.NoError(t, os.Symlink(priorSubpath, filepath.Join(env.storeDir, liveLink))) //nolint:forbidigo // test code -} - func initExt(ctx context.Context, t *testing.T) *TransferFileExtension { t.Helper() ext := &TransferFileExtension{} @@ -99,427 +68,450 @@ func initExt(ctx context.Context, t *testing.T) *TransferFileExtension { return ext } -// makeRepo returns the [tufext.Repository] backed by the given test repository -// server. -func makeRepo(t *testing.T, srv *testrepo.Server, name string) *tufext.Repository { - t.Helper() - cfg := testrepo.Config(t, srv.ConfigBlock(name)) - repo, ok := cfg.Repos[name] - require.True(t, ok, "testrepo config did not yield repository %q", name) - return repo.AsRepository() -} - -func TestPatchTransferFile_OverrideSourcePath(t *testing.T) { - u := "http://localhost:555/" - ext := &TransferFileExtension{OverrideSourcePathURL: &u} - - input := strings.TrimSpace(` -[Transfer] -ProtectVersion=%A - -[Source] -Type=url-file -Path=https://example.com/orig/ -MatchPattern=foo_@v.raw - -[Target] -Type=partition -MatchPattern=foo_@v -`) + "\n" - - var out bytes.Buffer - require.NoError(t, ext.patchTransferFile(&out, strings.NewReader(input))) - - cfg, err := ini.Load(out.Bytes()) - require.NoError(t, err) - assert.Equal(t, u, cfg.Section("Source").Key("Path").String()) - assert.Equal(t, "url-file", cfg.Section("Source").Key("Type").String()) - assert.Equal(t, "foo_@v.raw", cfg.Section("Source").Key("MatchPattern").String()) - assert.Equal(t, "%A", cfg.Section("Transfer").Key("ProtectVersion").String()) - assert.Equal(t, "partition", cfg.Section("Target").Key("Type").String()) +// inlineTarget returns a target whose data is embedded in its metadata, so +// that Fetch needs no repository server. +func inlineTarget(path string, data []byte) *tufclient.TargetInfo { + sum := sha256.Sum256(data) + target := &tufmetadata.TargetFiles{ + Path: path, + Length: int64(len(data)), + Hashes: tufmetadata.Hashes{"sha256": sum[:]}, + } + tufext.TargetFilesExt(target).WithInlineData(data) + // Embedding round-trips the struct through JSON, which drops the + // (json:"-") path a client otherwise takes from the metadata key. + target.Path = path + return &tufclient.TargetInfo{TargetFiles: target, Repo: testRepo} } -func TestPatchTransferFile_NoOverride(t *testing.T) { - ext := &TransferFileExtension{} - input := "[Source]\nPath=https://example.com/\n" - - var out bytes.Buffer - require.NoError(t, ext.patchTransferFile(&out, strings.NewReader(input))) - - cfg, err := ini.Load(out.Bytes()) - require.NoError(t, err) - assert.Equal(t, "https://example.com/", cfg.Section("Source").Key("Path").String()) +func applyAll(ctx context.Context, t *testing.T, ext *TransferFileExtension, infos ...*tufclient.TargetInfo) { + t.Helper() + for _, info := range infos { + applied, err := ext.ApplyTarget(ctx, info) + require.NoError(t, err, "target %s", info.Path) + require.True(t, applied, "target %s", info.Path) + } } -func TestPatchTransferFile_CreatesSourceIfMissing(t *testing.T) { - u := "http://proxy/" - ext := &TransferFileExtension{OverrideSourcePathURL: &u} - input := "[Target]\nType=partition\n" - - var out bytes.Buffer - require.NoError(t, ext.patchTransferFile(&out, strings.NewReader(input))) +func TestParseTransferPath(t *testing.T) { + for _, tc := range []struct { + path string + component, file string + ok, wantErr bool + }{ + {path: "regular/target.bin"}, + {path: ".quarry/machine-tags/amutable.foo"}, + {path: ".quarry/sysupdate-other/x"}, // another extension, not claimed + {path: ".quarry/sysupdate"}, + {path: ".quarry/sysupdate.d/10-usr.transfer", file: "10-usr.transfer", ok: true}, + {path: ".quarry/sysupdate.k8s.d/k8s.transfer", component: "k8s", file: "k8s.transfer", ok: true}, + {path: ".zzz-quarry-special/sysupdate.k8s.d/k8s.transfer", component: "k8s", file: "k8s.transfer", ok: true}, // legacy prefix + {path: ".quarry/sysupdate.k8s.d/k8s.component", component: "k8s", file: "k8s.component", ok: true}, + {path: ".quarry/sysupdate.k8s.d/k8s.transfer.d/10-x.conf", component: "k8s", file: "k8s.transfer.d/10-x.conf", ok: true}, + // These are malformed. They are part of the extension but rejected. + {path: ".quarry/sysupdate.d", ok: true, wantErr: true}, + {path: ".quarry/sysupdate.d/", ok: true, wantErr: true}, + {path: ".quarry/sysupdate.d/../x.transfer", ok: true, wantErr: true}, + {path: ".quarry/sysupdate.d/a//b", ok: true, wantErr: true}, + {path: ".quarry/sysupdate.d/./b", ok: true, wantErr: true}, + {path: ".quarry/sysupdate..d/x.transfer", ok: true, wantErr: true}, + {path: ".quarry/sysupdate.a.b.d/x.transfer", ok: true, wantErr: true}, + {path: ".quarry/sysupdate.transfer", ok: true, wantErr: true}, + {path: ".quarry/sysupdate.foo/x.transfer", ok: true, wantErr: true}, + } { + component, file, ok, err := ParseTransferPath(tc.path) + assert.Equal(t, tc.ok, ok, "path %q", tc.path) + if tc.wantErr { + require.Error(t, err, "path %q", tc.path) + continue + } + require.NoError(t, err, "path %q", tc.path) + assert.Equal(t, tc.component, component, "path %q", tc.path) + assert.Equal(t, tc.file, file, "path %q", tc.path) + } - cfg, err := ini.Load(out.Bytes()) - require.NoError(t, err) - assert.Equal(t, u, cfg.Section("Source").Key("Path").String()) - assert.Equal(t, "partition", cfg.Section("Target").Key("Type").String()) + // What hardhat writes (transferlayout) parses back to the same values. + for _, tc := range []struct{ component, file string }{ + {"", "12-usr.transfer"}, + {"", "docker.feature"}, + {"k8s", "k8s.component"}, + {"k8s", "k8s.transfer.d/10-x.conf"}, + } { + path, err := transferlayout.TargetPath(transferlayout.ComponentDir(tc.component), tc.file) + require.NoError(t, err) + component, file, ok, err := ParseTransferPath(path) + require.NoError(t, err, path) + assert.True(t, ok, path) + assert.Equal(t, tc.component, component, path) + assert.Equal(t, tc.file, file, path) + } } func TestInit_FreshStore(t *testing.T) { env, ctx := setupTestEnv(t) - - ext := &TransferFileExtension{} - _, err := ext.Init(ctx) - require.NoError(t, err) - t.Cleanup(func() { _ = ext.Close() }) + ext := initExt(ctx, t) require.NotNil(t, ext.storeDir) - require.NotNil(t, ext.newDir) - assert.Equal(t, fixedNewDirSubpath, ext.newDirSubpath) - assert.Empty(t, ext.oldDirSubpath) - assert.False(t, ext.modifiedEtc) - - info, err := os.Stat(filepath.Join(env.storeDir, fixedNewDirSubpath)) //nolint:forbidigo // test code - require.NoError(t, err) - assert.True(t, info.IsDir()) - - // /etc/extensions/ is a workaround for the systemd CurrentSymlink= bug. - info, err = os.Stat(filepath.Join(env.etcPath, "extensions")) //nolint:forbidigo // test code + entries, err := os.ReadDir(env.storeDir) require.NoError(t, err) - assert.True(t, info.IsDir()) - - _, err = os.Lstat(filepath.Join(env.storeDir, liveLink)) //nolint:forbidigo // test code - require.ErrorIs(t, err, fs.ErrNotExist) - _, err = os.Lstat(filepath.Join(env.storeDir, lastLink)) //nolint:forbidigo // test code - assert.ErrorIs(t, err, fs.ErrNotExist) + assert.Empty(t, entries) } -func TestInit_WithExistingLive(t *testing.T) { +// Staging directories of earlier runs (kept on failure) are removed. +func TestInit_PrunesStaleRuns(t *testing.T) { env, ctx := setupTestEnv(t) - env.populateUpdateDir(t, priorSubpath, nil) - env.makeLive(t) - - ext := &TransferFileExtension{} - _, err := ext.Init(ctx) - require.NoError(t, err) - t.Cleanup(func() { _ = ext.Close() }) + stale := filepath.Join(env.storeDir, "sysupdate.foo.d", "sysupdate.foo.d") //nolint:forbidigo // test code + require.NoError(t, os.MkdirAll(stale, 0o755)) //nolint:forbidigo // test code + require.NoError(t, os.WriteFile(filepath.Join(stale, "foo.transfer"), []byte("x"), 0o644)) //nolint:forbidigo // test code - assert.Equal(t, priorSubpath, ext.oldDirSubpath) - assert.Equal(t, fixedNewDirSubpath, ext.newDirSubpath) + initExt(ctx, t) - target, err := os.Readlink(filepath.Join(env.storeDir, lastLink)) //nolint:forbidigo // test code - require.NoError(t, err) - assert.Equal(t, priorSubpath, target) - - _, err = os.Lstat(filepath.Join(env.storeDir, liveLink)) //nolint:forbidigo // test code + _, err := os.Stat(filepath.Join(env.storeDir, "sysupdate.foo.d")) //nolint:forbidigo // test code assert.ErrorIs(t, err, fs.ErrNotExist) } -func TestInit_LiveIsNotSymlink(t *testing.T) { - env, ctx := setupTestEnv(t) - require.NoError(t, os.MkdirAll(env.storeDir, 0o755)) //nolint:forbidigo // test code - require.NoError(t, os.WriteFile(filepath.Join(env.storeDir, liveLink), nil, 0o644)) //nolint:forbidigo // test code - - ext := &TransferFileExtension{} - _, err := ext.Init(ctx) - require.Error(t, err) - // Mirror the orchestrator: abortOnError calls Abort, which also releases - // the partial storeDir handle. - require.NoError(t, ext.Abort(ctx, err)) -} - -func TestUnlinkTransfers_RemovesQuarryLinks(t *testing.T) { - env, ctx := setupTestEnv(t) - ext := initExt(ctx, t) - - target := filepath.Join(env.storeDir, fixedNewDirSubpath, "sysupdate.d", "quarry.transfer") //nolint:forbidigo // test code - require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o755)) //nolint:forbidigo // test code - require.NoError(t, os.WriteFile(target, []byte("dummy"), 0o644)) //nolint:forbidigo // test code - require.NoError(t, os.MkdirAll(filepath.Join(env.etcPath, "sysupdate.d"), 0o755)) //nolint:forbidigo // test code - link := filepath.Join(env.etcPath, "sysupdate.d", "quarry.transfer") //nolint:forbidigo // test code - require.NoError(t, os.Symlink(target, link)) //nolint:forbidigo // test code - - require.NoError(t, ext.unlinkTransfers()) - - _, err := os.Lstat(link) - assert.ErrorIs(t, err, fs.ErrNotExist) -} - -// Covers the `sysupdate.*.d` glob (whole-component-dir symlinks), which is -// linked and unlinked differently from base-pattern files. -func TestUnlinkTransfers_RemovesQuarryComponentDir(t *testing.T) { - env, ctx := setupTestEnv(t) +func TestApplyTarget_NotForUs(t *testing.T) { + _, ctx := setupTestEnv(t) ext := initExt(ctx, t) - target := filepath.Join(env.storeDir, fixedNewDirSubpath, "sysupdate.comp.d") //nolint:forbidigo // test code - require.NoError(t, os.MkdirAll(target, 0o755)) //nolint:forbidigo // test code - require.NoError(t, os.WriteFile(filepath.Join(target, "x.conf"), []byte("x"), 0o644)) //nolint:forbidigo // test code - link := filepath.Join(env.etcPath, "sysupdate.comp.d") //nolint:forbidigo // test code - require.NoError(t, os.Symlink(target, link)) //nolint:forbidigo // test code - - require.NoError(t, ext.unlinkTransfers()) - - _, err := os.Lstat(link) - assert.ErrorIs(t, err, fs.ErrNotExist) + applied, err := ext.ApplyTarget(ctx, inlineTarget("regular/target.bin", []byte("x"))) + require.NoError(t, err) + assert.False(t, applied) + applied, err = ext.ApplyTarget(ctx, inlineTarget(".quarry/machine-tags/amutable.foo", nil)) + require.NoError(t, err) + assert.False(t, applied) + assert.Empty(t, ext.Transfers()) } -func TestUnlinkTransfers_SkipsNonSymlinks(t *testing.T) { - env, ctx := setupTestEnv(t) +func TestApplyTarget_Collects(t *testing.T) { + _, ctx := setupTestEnv(t) ext := initExt(ctx, t) - require.NoError(t, os.MkdirAll(filepath.Join(env.etcPath, "sysupdate.d"), 0o755)) //nolint:forbidigo // test code - realFile := filepath.Join(env.etcPath, "sysupdate.d", "foo.transfer") //nolint:forbidigo // test code - require.NoError(t, os.WriteFile(realFile, []byte("sysadmin owned"), 0o644)) //nolint:forbidigo // test code - - require.NoError(t, ext.unlinkTransfers()) - - _, err := os.Stat(realFile) - assert.NoError(t, err) + transfer := []byte("[Transfer]\nProtectVersion=%A\n") + applyAll(ctx, t, ext, + inlineTarget(".quarry/sysupdate.d/10-usr.transfer", transfer), + inlineTarget(".quarry/sysupdate.d/docker.feature", []byte("[Feature]\n")), + inlineTarget(".quarry/sysupdate.foo.d/foo.transfer", transfer), + inlineTarget(".quarry/sysupdate.foo.d/foo.component", []byte("[Component]\n")), + // For duplicates (e.g. from a second repository), the first one wins. + inlineTarget(".quarry/sysupdate.d/10-usr.transfer", []byte("[Transfer]\nother=1\n")), + // Malformed paths are claimed (and ignored) rather than aborting. + inlineTarget(".quarry/sysupdate.bar/x.transfer", transfer), + inlineTarget(".quarry/sysupdate.d/../x.transfer", transfer), + ) + + transfers := ext.Transfers() + require.Len(t, transfers, 2) + assert.Empty(t, transfers[""].Component) + assert.Equal(t, []string{"10-usr.transfer", "docker.feature"}, transfers[""].Files()) + assert.Equal(t, ".quarry/sysupdate.d", transfers[""].String()) + data, err := tufext.TargetFilesExt(transfers[""].files["10-usr.transfer"].TargetFiles).InlineData() + require.NoError(t, err) + assert.Equal(t, transfer, data, "the first one wins") + assert.Equal(t, "foo", transfers["foo"].Component) + assert.Equal(t, []string{"foo.component", "foo.transfer"}, transfers["foo"].Files()) + assert.Equal(t, ".quarry/sysupdate.foo.d", transfers["foo"].String()) } -// Quarry must not delete symlinks it doesn't own. -func TestUnlinkTransfers_SkipsNonQuarrySymlinks(t *testing.T) { +func TestStage_DefaultComponent(t *testing.T) { env, ctx := setupTestEnv(t) - ext := initExt(ctx, t) - - require.NoError(t, os.MkdirAll(filepath.Join(env.etcPath, "sysupdate.d"), 0o755)) //nolint:forbidigo // test code - link := filepath.Join(env.etcPath, "sysupdate.d", "foo.transfer") //nolint:forbidigo // test code - require.NoError(t, os.Symlink("/somewhere/else/foo.transfer", link)) //nolint:forbidigo // test code + proxyURL := "http://localhost:9999/" + ext := &TransferFileExtension{OverrideSourcePathURL: &proxyURL} + _, err := ext.Init(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = ext.Close() }) - require.NoError(t, ext.unlinkTransfers()) + transfer := []byte(strings.TrimSpace(` +[Transfer] +ProtectVersion=%A - _, err := os.Lstat(link) - assert.NoError(t, err) -} +[Source] +Type=url-file +Path=https://upstream.example.com/orig/ +MatchPattern=**/foo_@v.raw -func TestLinkTransfers_BaseAndComponent(t *testing.T) { - env, ctx := setupTestEnv(t) - ext := initExt(ctx, t) +[Target] +Type=partition +MatchPattern=foo_@v +`) + "\n") + applyAll(ctx, t, ext, + inlineTarget(".quarry/sysupdate.d/10-usr.transfer", transfer), + inlineTarget(".quarry/sysupdate.d/docker.feature", []byte("[Feature]\nEnabled=false\nSuggestOnMachineTag=amutable.ext.docker\n")), + inlineTarget(".quarry/sysupdate.d/containerd.feature", []byte("[Feature]\nSuggestOnMachineTag=amutable.ext.containerd\n")), + inlineTarget(".quarry/sysupdate.d/10-usr.transfer.d/x.conf", []byte("[Transfer]\nVerify=no\n")), + inlineTarget(".quarry/sysupdate.d/10-usr.transfer.d/y.conf", []byte("[Source]\nPath=https://elsewhere.example.com/\n")), + ) - env.populateUpdateDir(t, fixedNewDirSubpath, map[string][]byte{ - "sysupdate.d/base.transfer": []byte("base"), - "sysupdate.comp.d/component.conf": []byte("component"), - }) + staged, err := ext.Stage(ctx, ext.Transfers()[""], StageOptions{MachineTags: []string{"amutable.ext.docker", "acp.x=1"}}) + require.NoError(t, err) + require.NotNil(t, staged, "the default component is always staged") + t.Cleanup(func() { _ = staged.Close() }) - require.NoError(t, ext.linkTransfers(fixedNewDirSubpath)) + stagePath := filepath.Join(env.storeDir, "sysupdate.d") //nolint:forbidigo // test code + assert.Equal(t, stagePath, staged.Path()) + assert.Empty(t, staged.Component) + assert.Equal(t, []string{"docker"}, staged.EnabledFeatures) + assert.Equal(t, []string{stagePath + "/sysupdate.d:/run/sysupdate.d"}, staged.BindPaths()) - baseLink := filepath.Join(env.etcPath, "sysupdate.d", "base.transfer") //nolint:forbidigo // test code - target, err := os.Readlink(baseLink) + out, err := os.ReadFile(filepath.Join(stagePath, "sysupdate.d", "10-usr.transfer")) //nolint:forbidigo // test code require.NoError(t, err) - assert.Equal(t, filepath.Join(env.storeDir, fixedNewDirSubpath, "sysupdate.d", "base.transfer"), target) //nolint:forbidigo // test code - // Stat after Readlink: guards against a regression where base files - // landed at /etc/ instead of /etc/sysupdate.d/. - _, err = os.Stat(baseLink) + cfg, err := loadINI(out) require.NoError(t, err) + assert.Equal(t, proxyURL, cfg.Section("Source").Key("Path").String()) + assert.Equal(t, "**/foo_@v.raw", cfg.Section("Source").Key("MatchPattern").String()) - // Component dirs are symlinked as a whole, not file-by-file. - compLink := filepath.Join(env.etcPath, "sysupdate.comp.d") //nolint:forbidigo // test code - target, err = os.Readlink(compLink) + out, err = os.ReadFile(filepath.Join(stagePath, "sysupdate.d", "docker.feature")) //nolint:forbidigo // test code require.NoError(t, err) - assert.Equal(t, filepath.Join(env.storeDir, fixedNewDirSubpath, "sysupdate.comp.d"), target) //nolint:forbidigo // test code -} - -func TestLinkTransfers_CreatesSysupdateDirIfMissing(t *testing.T) { - env, ctx := setupTestEnv(t) - ext := initExt(ctx, t) - - env.populateUpdateDir(t, fixedNewDirSubpath, map[string][]byte{ - "sysupdate.d/foo.transfer": []byte("foo"), - }) - - _, err := os.Stat(filepath.Join(env.etcPath, "sysupdate.d")) //nolint:forbidigo // test code - require.ErrorIs(t, err, fs.ErrNotExist) + cfg, err = loadINI(out) + require.NoError(t, err) + assert.Equal(t, "true", cfg.Section("Feature").Key("Enabled").String()) - require.NoError(t, ext.linkTransfers(fixedNewDirSubpath)) + // A feature the machine does not enable is left as shipped. Enabled= is + // only ever set to true and never written as false. + out, err = os.ReadFile(filepath.Join(stagePath, "sysupdate.d", "containerd.feature")) //nolint:forbidigo // test code + require.NoError(t, err) + assert.Equal(t, "[Feature]\nSuggestOnMachineTag=amutable.ext.containerd\n", string(out)) - info, err := os.Stat(filepath.Join(env.etcPath, "sysupdate.d")) //nolint:forbidigo // test code + // Other files are staged verbatim, nested paths included. + out, err = os.ReadFile(filepath.Join(stagePath, "sysupdate.d", "10-usr.transfer.d", "x.conf")) //nolint:forbidigo // test code require.NoError(t, err) - assert.True(t, info.IsDir()) -} + assert.Equal(t, "[Transfer]\nVerify=no\n", string(out)) -func TestLinkTransfers_FailsOnConflictingNonQuarryFile(t *testing.T) { - env, ctx := setupTestEnv(t) - ext := initExt(ctx, t) + // A transfer drop-in that sets Path= also points at the proxy. + out, err = os.ReadFile(filepath.Join(stagePath, "sysupdate.d", "10-usr.transfer.d", "y.conf")) //nolint:forbidigo // test code + require.NoError(t, err) + cfg, err = loadINI(out) + require.NoError(t, err) + assert.Equal(t, proxyURL, cfg.Section("Source").Key("Path").String()) - env.populateUpdateDir(t, fixedNewDirSubpath, map[string][]byte{ - "sysupdate.d/foo.transfer": []byte("new"), - }) - require.NoError(t, os.MkdirAll(filepath.Join(env.etcPath, "sysupdate.d"), 0o755)) //nolint:forbidigo // test code - require.NoError(t, os.WriteFile(filepath.Join(env.etcPath, "sysupdate.d", "foo.transfer"), []byte("sysadmin"), 0o644)) //nolint:forbidigo // test code + // The default component gets no component file. + entries, err := os.ReadDir(stagePath) + require.NoError(t, err) + require.Len(t, entries, 1) + assert.Equal(t, "sysupdate.d", entries[0].Name()) - err := ext.linkTransfers(fixedNewDirSubpath) - require.Error(t, err) - assert.ErrorIs(t, err, fs.ErrExist) + // Remove deletes the component's staging directory. + require.NoError(t, staged.Remove()) + _, err = os.Stat(stagePath) + assert.ErrorIs(t, err, fs.ErrNotExist) } -func TestBeforeUpdate_HappyPath(t *testing.T) { +func TestStage_Component(t *testing.T) { env, ctx := setupTestEnv(t) ext := initExt(ctx, t) - env.populateUpdateDir(t, fixedNewDirSubpath, map[string][]byte{ - "sysupdate.d/foo.transfer": []byte("transfer"), - }) + applyAll(ctx, t, ext, + inlineTarget(".quarry/sysupdate.foo.d/foo.transfer", []byte("[Transfer]\n\n[Source]\nType=url-file\nPath=https://example.com/\n")), + inlineTarget(".quarry/sysupdate.foo.d/foo.component", []byte("[Component]\nDescription=Foo\nEnabled=false\nSuggestOnMachineTag=amutable.foo\n")), + ) - require.NoError(t, ext.BeforeUpdate(ctx)) + staged, err := ext.Stage(ctx, ext.Transfers()["foo"], StageOptions{MachineTags: []string{"amutable.foo"}}) + require.NoError(t, err) + require.NotNil(t, staged) + t.Cleanup(func() { _ = staged.Close() }) - assert.True(t, ext.modifiedEtc) + stagePath := filepath.Join(env.storeDir, "sysupdate.foo.d") //nolint:forbidigo // test code + assert.Equal(t, []string{ + stagePath + "/sysupdate.foo.d:/run/sysupdate.foo.d", + stagePath + "/sysupdate.foo.component:/run/sysupdate.foo.component", + }, staged.BindPaths()) - liveTarget, err := os.Readlink(filepath.Join(env.storeDir, liveLink)) //nolint:forbidigo // test code + // The shipped component file is staged as a sibling of the definitions + // directory and not inside it. + out, err := os.ReadFile(filepath.Join(stagePath, "sysupdate.foo.component")) //nolint:forbidigo // test code require.NoError(t, err) - assert.Equal(t, fixedNewDirSubpath, liveTarget) - - foo := filepath.Join(env.etcPath, "sysupdate.d", "foo.transfer") //nolint:forbidigo // test code - target, err := os.Readlink(foo) + cfg, err := loadINI(out) require.NoError(t, err) - assert.Equal(t, filepath.Join(env.storeDir, fixedNewDirSubpath, "sysupdate.d", "foo.transfer"), target) //nolint:forbidigo // test code -} - -// Exercises the inner-unlink recovery path when linkTransfers fails partway: -// partial Quarry-managed links must be removed, but pre-existing non-Quarry -// files and the already-created `live` symlink must survive for Abort. -func TestBeforeUpdate_LinkPartialFailureCleansUp(t *testing.T) { - env, ctx := setupTestEnv(t) - ext := initExt(ctx, t) - - // filepath.Glob is sorted: "alpha" links first, then "conflict" collides - // with the pre-planted non-Quarry file. - env.populateUpdateDir(t, fixedNewDirSubpath, map[string][]byte{ - "sysupdate.d/alpha.transfer": []byte("alpha"), - "sysupdate.d/conflict.transfer": []byte("new"), - }) - require.NoError(t, os.MkdirAll(filepath.Join(env.etcPath, "sysupdate.d"), 0o755)) //nolint:forbidigo // test code - conflict := filepath.Join(env.etcPath, "sysupdate.d", "conflict.transfer") //nolint:forbidigo // test code - require.NoError(t, os.WriteFile(conflict, []byte("sysadmin"), 0o644)) //nolint:forbidigo // test code - - err := ext.BeforeUpdate(ctx) - require.Error(t, err) - require.ErrorIs(t, err, fs.ErrExist) - - _, err = os.Lstat(filepath.Join(env.etcPath, "sysupdate.d", "alpha.transfer")) //nolint:forbidigo // test code + assert.Equal(t, "Foo", cfg.Section("Component").Key("Description").String()) + assert.Equal(t, "true", cfg.Section("Component").Key("Enabled").String()) + _, err = os.Stat(filepath.Join(stagePath, "sysupdate.foo.d", "foo.component")) //nolint:forbidigo // test code require.ErrorIs(t, err, fs.ErrNotExist) - got, err := os.ReadFile(conflict) //nolint:forbidigo // test code + // Without a configured override URL, the source path is untouched. + out, err = os.ReadFile(filepath.Join(stagePath, "sysupdate.foo.d", "foo.transfer")) //nolint:forbidigo // test code + require.NoError(t, err) + cfg, err = loadINI(out) require.NoError(t, err) - assert.Equal(t, []byte("sysadmin"), got) + assert.Equal(t, "https://example.com/", cfg.Section("Source").Key("Path").String()) - liveTarget, err := os.Readlink(filepath.Join(env.storeDir, liveLink)) //nolint:forbidigo // test code + // A failed run keeps the staging directory (Close only), which the next + // run's Init prunes. + require.NoError(t, staged.Close()) + require.NoError(t, ext.Close()) + _, err = os.Stat(stagePath) require.NoError(t, err) - assert.Equal(t, fixedNewDirSubpath, liveTarget) + initExt(ctx, t) + _, err = os.Stat(stagePath) + assert.ErrorIs(t, err, fs.ErrNotExist) } -func TestBeforeUpdate_ReplacesOldQuarryLinks(t *testing.T) { +// A component whose component file neither enables it nor is suggested for +// the machine is not staged. +func TestStage_SkipsComponent(t *testing.T) { env, ctx := setupTestEnv(t) - - env.populateUpdateDir(t, priorSubpath, map[string][]byte{ - "sysupdate.d/old.transfer": []byte("old"), - }) - env.makeLive(t) - require.NoError(t, os.MkdirAll(filepath.Join(env.etcPath, "sysupdate.d"), 0o755)) //nolint:forbidigo // test code - oldLink := filepath.Join(env.etcPath, "sysupdate.d", "old.transfer") //nolint:forbidigo // test code - require.NoError(t, os.Symlink( //nolint:forbidigo // test code - filepath.Join(env.storeDir, priorSubpath, "sysupdate.d", "old.transfer"), //nolint:forbidigo // test code - oldLink, - )) - ext := initExt(ctx, t) - env.populateUpdateDir(t, fixedNewDirSubpath, map[string][]byte{ - "sysupdate.d/new.transfer": []byte("new"), - }) - require.NoError(t, ext.BeforeUpdate(ctx)) - - // `live` flipped to the new dir, `last` retained so Abort can roll back. - liveTarget, err := os.Readlink(filepath.Join(env.storeDir, liveLink)) //nolint:forbidigo // test code - require.NoError(t, err) - assert.Equal(t, fixedNewDirSubpath, liveTarget) - lastTarget, err := os.Readlink(filepath.Join(env.storeDir, lastLink)) //nolint:forbidigo // test code - require.NoError(t, err) - assert.Equal(t, priorSubpath, lastTarget) - - _, err = os.Lstat(oldLink) - require.ErrorIs(t, err, fs.ErrNotExist) - target, err := os.Readlink(filepath.Join(env.etcPath, "sysupdate.d", "new.transfer")) //nolint:forbidigo // test code + applyAll(ctx, t, ext, + inlineTarget(".quarry/sysupdate.foo.d/foo.transfer", []byte("[Transfer]\n")), + inlineTarget(".quarry/sysupdate.foo.d/foo.component", []byte("[Component]\nEnabled=false\nSuggestOnMachineTag=amutable.foo\n")), + inlineTarget(".quarry/sysupdate.bar.d/bar.transfer", []byte("[Transfer]\n")), + inlineTarget(".quarry/sysupdate.bar.d/bar.component", []byte("[Component]\nEnabled=no\nSuggestOnArchitecture=x86-64\n")), + ) + for _, component := range []string{"foo", "bar"} { + staged, err := ext.Stage(ctx, ext.Transfers()[component], StageOptions{MachineTags: []string{"amutable.other"}}) + require.NoError(t, err, component) + assert.Nil(t, staged, component) + } + entries, err := os.ReadDir(env.storeDir) require.NoError(t, err) - assert.Equal(t, filepath.Join(env.storeDir, fixedNewDirSubpath, "sysupdate.d", "new.transfer"), target) //nolint:forbidigo // test code + assert.Empty(t, entries) } -func TestAbort_FreshState(t *testing.T) { - env, ctx := setupTestEnv(t) - ext := initExt(ctx, t) - - require.NoError(t, ext.Abort(ctx, errors.New("test"))) - - _, err := os.Stat(filepath.Join(env.storeDir, fixedNewDirSubpath)) //nolint:forbidigo // test code - require.ErrorIs(t, err, fs.ErrNotExist) - - _, err = os.Lstat(filepath.Join(env.storeDir, liveLink)) //nolint:forbidigo // test code - require.ErrorIs(t, err, fs.ErrNotExist) - _, err = os.Lstat(filepath.Join(env.storeDir, lastLink)) //nolint:forbidigo // test code - require.ErrorIs(t, err, fs.ErrNotExist) - - assert.Nil(t, ext.storeDir) - assert.Nil(t, ext.newDir) - assert.Empty(t, ext.oldDirSubpath) - assert.Empty(t, ext.newDirSubpath) +// Staging rewrites transfer, feature and component files and keeps the rest +// of each file as it is. +func TestStage_Rewrites(t *testing.T) { + proxyURL := "http://proxy/" + for _, tc := range []struct { + name string + files map[string]string // in sysupdate.foo.d/ + override *string + tags []string + want map[string]string // in the staging directory + missing []string // in the staging directory + }{ + { + name: "transfer keeps repeated keys", + files: map[string]string{ + "foo.transfer": "[Transfer]\nProtectVersion=%A\n\n[Source]\nType=url-file\nPath=https://example.com/\nMatchPattern=foo_@v.raw\nMatchPattern=**/foo_@v.raw\n", + }, + override: &proxyURL, + want: map[string]string{ + "sysupdate.foo.d/foo.transfer": "[Transfer]\nProtectVersion=%A\n\n[Source]\nType=url-file\nMatchPattern=foo_@v.raw\nMatchPattern=**/foo_@v.raw\nPath=http://proxy/\n", + }, + // A component without a component file is staged without one, + // which sysupdate treats as enabled. + missing: []string{"sysupdate.foo.component"}, + }, + { + name: "transfer without source section", + files: map[string]string{"foo.transfer": "[Target]\nType=partition\n"}, + override: &proxyURL, + want: map[string]string{"sysupdate.foo.d/foo.transfer": "[Target]\nType=partition\n\n[Source]\nPath=http://proxy/\n"}, + }, + { + name: "features", + files: map[string]string{ + "foo.transfer": "[Transfer]\n", + "docker.feature": "[Feature]\nDescription=Docker\nEnabled=false\nSuggestOnMachineTag=amutable.ext.*\n", + "containerd.feature": "[Feature]\nEnabled=false\nSuggestOnMachineTag=amutable.ext.containerd\n", + "interactive.feature": "[Feature]\nEnabled=yes\nSuggestOnMachineTag=amutable.ext.none\n", + "arch.feature": "[Feature]\nSuggestOnArchitecture=x86-64\n", + }, + tags: []string{"amutable.ext.docker=x", "amutable.ext.interactive"}, + want: map[string]string{ + // Stamping moves Enabled= to the end of the section. + "sysupdate.foo.d/docker.feature": "[Feature]\nDescription=Docker\nSuggestOnMachineTag=amutable.ext.*\nEnabled=true\n", + // A feature that is not suggested is left as shipped. + "sysupdate.foo.d/containerd.feature": "[Feature]\nEnabled=false\nSuggestOnMachineTag=amutable.ext.containerd\n", + // An enabled feature keeps its own settings. + "sysupdate.foo.d/interactive.feature": "[Feature]\nEnabled=yes\nSuggestOnMachineTag=amutable.ext.none\n", + // quarry does not evaluate other conditions. + "sysupdate.foo.d/arch.feature": "[Feature]\nSuggestOnArchitecture=x86-64\n", + }, + }, + { + name: "suggested component", + files: map[string]string{ + "foo.transfer": "[Transfer]\n", + "foo.component": "[Component]\nDescription=Foo\nEnabled=false\nSuggestOnMachineTag=amutable.foo\n", + }, + tags: []string{"amutable.foo"}, + want: map[string]string{"sysupdate.foo.component": "[Component]\nDescription=Foo\nSuggestOnMachineTag=amutable.foo\nEnabled=true\n"}, + }, + { + name: "enabled component", + files: map[string]string{ + "foo.transfer": "[Transfer]\n", + "foo.component": "[Component]\nDescription=Foo\nEnabled=True\nSuggestOnMachineTag=amutable.foo\n", + }, + want: map[string]string{"sysupdate.foo.component": "[Component]\nDescription=Foo\nEnabled=True\nSuggestOnMachineTag=amutable.foo\n"}, + }, + { + name: "component enabled by default", + files: map[string]string{ + "foo.transfer": "[Transfer]\n", + "foo.component": "[Component]\nDescription=Foo\n", + }, + want: map[string]string{"sysupdate.foo.component": "[Component]\nDescription=Foo\n"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + _, ctx := setupTestEnv(t) + ext := &TransferFileExtension{OverrideSourcePathURL: tc.override} + _, err := ext.Init(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = ext.Close() }) + + for file, data := range tc.files { + applyAll(ctx, t, ext, inlineTarget(".quarry/sysupdate.foo.d/"+file, []byte(data))) + } + staged, err := ext.Stage(ctx, ext.Transfers()["foo"], StageOptions{MachineTags: tc.tags}) + require.NoError(t, err) + require.NotNil(t, staged) + t.Cleanup(func() { _ = staged.Remove() }) + + for file, want := range tc.want { + out, err := os.ReadFile(filepath.Join(staged.Path(), file)) //nolint:forbidigo // test code + require.NoError(t, err) + assert.Equal(t, want, string(out), file) + } + for _, file := range tc.missing { + _, err := os.Stat(filepath.Join(staged.Path(), file)) //nolint:forbidigo // test code + require.ErrorIs(t, err, fs.ErrNotExist, file) + } + }) + } } -// Covers the `else if ext.modifiedEtc` branch in Abort: no prior `live` to -// swap back, just remove the one BeforeUpdate created. -func TestAbort_FirstUpdateRemovesLive(t *testing.T) { +func TestBeforeUpdate_HostDefinitions(t *testing.T) { env, ctx := setupTestEnv(t) ext := initExt(ctx, t) - env.populateUpdateDir(t, fixedNewDirSubpath, map[string][]byte{ - "sysupdate.d/foo.transfer": []byte("foo"), - }) + // Nothing on the host is fine. require.NoError(t, ext.BeforeUpdate(ctx)) - _, err := os.Lstat(filepath.Join(env.storeDir, liveLink)) //nolint:forbidigo // test code + found, err := hostDefinitions() require.NoError(t, err) + assert.Empty(t, found) - require.NoError(t, ext.Abort(ctx, errors.New("test"))) - - _, err = os.Lstat(filepath.Join(env.storeDir, liveLink)) //nolint:forbidigo // test code - require.ErrorIs(t, err, fs.ErrNotExist) - _, err = os.Lstat(filepath.Join(env.storeDir, lastLink)) //nolint:forbidigo // test code - require.ErrorIs(t, err, fs.ErrNotExist) - _, err = os.Stat(filepath.Join(env.storeDir, fixedNewDirSubpath)) //nolint:forbidigo // test code - require.ErrorIs(t, err, fs.ErrNotExist) - _, err = os.Lstat(filepath.Join(env.etcPath, "sysupdate.d", "foo.transfer")) //nolint:forbidigo // test code - assert.ErrorIs(t, err, fs.ErrNotExist) + // Host definitions only produce a warning, but are found. + require.NoError(t, os.MkdirAll(filepath.Join(env.etcPath, "sysupdate.d"), 0o755)) //nolint:forbidigo // test code + require.NoError(t, os.WriteFile(filepath.Join(env.etcPath, "sysupdate.d", "x.transfer"), []byte("x"), 0o644)) //nolint:forbidigo // test code + require.NoError(t, os.MkdirAll(filepath.Join(env.etcPath, "sysupdate.foo.d"), 0o755)) //nolint:forbidigo // test code + require.NoError(t, os.WriteFile(filepath.Join(env.etcPath, "sysupdate.foo.component"), []byte("x"), 0o644)) //nolint:forbidigo // test code + require.NoError(t, ext.BeforeUpdate(ctx)) + found, err = hostDefinitions() + require.NoError(t, err) + assert.Equal(t, []string{ + filepath.Join(env.etcPath, "sysupdate.d", "x.transfer"), //nolint:forbidigo // test code + filepath.Join(env.etcPath, "sysupdate.foo.component"), //nolint:forbidigo // test code + filepath.Join(env.etcPath, "sysupdate.foo.d"), //nolint:forbidigo // test code + }, found) } -func TestAbort_AfterBeforeUpdate_RestoresPriorState(t *testing.T) { +func TestAbort_RemovesStaged(t *testing.T) { env, ctx := setupTestEnv(t) - - env.populateUpdateDir(t, priorSubpath, map[string][]byte{ - "sysupdate.d/prior.transfer": []byte("prior"), - }) - env.makeLive(t) - ext := initExt(ctx, t) - env.populateUpdateDir(t, fixedNewDirSubpath, map[string][]byte{ - "sysupdate.d/new.transfer": []byte("new"), - }) - require.NoError(t, ext.BeforeUpdate(ctx)) - - require.NoError(t, ext.Abort(ctx, errors.New("test"))) - target, err := os.Readlink(filepath.Join(env.storeDir, liveLink)) //nolint:forbidigo // test code + applyAll(ctx, t, ext, inlineTarget(".quarry/sysupdate.d/x.transfer", []byte("[Transfer]\n"))) + staged, err := ext.Stage(ctx, ext.Transfers()[""], StageOptions{}) require.NoError(t, err) - assert.Equal(t, priorSubpath, target) + require.NoError(t, staged.Close()) - // `last` is consumed by Rename(last, live). - _, err = os.Lstat(filepath.Join(env.storeDir, lastLink)) //nolint:forbidigo // test code - require.ErrorIs(t, err, fs.ErrNotExist) + require.NoError(t, ext.Abort(ctx, errors.New("test"))) - _, err = os.Lstat(filepath.Join(env.etcPath, "sysupdate.d", "new.transfer")) //nolint:forbidigo // test code + _, err = os.Stat(filepath.Join(env.storeDir, "sysupdate.d")) //nolint:forbidigo // test code require.ErrorIs(t, err, fs.ErrNotExist) - priorTarget, err := os.Readlink(filepath.Join(env.etcPath, "sysupdate.d", "prior.transfer")) //nolint:forbidigo // test code - require.NoError(t, err) - assert.Equal(t, filepath.Join(env.storeDir, priorSubpath, "sysupdate.d", "prior.transfer"), priorTarget) //nolint:forbidigo // test code - - _, err = os.Stat(filepath.Join(env.storeDir, fixedNewDirSubpath)) //nolint:forbidigo // test code - assert.ErrorIs(t, err, fs.ErrNotExist) + assert.Nil(t, ext.storeDir) } func TestAbort_Idempotent(t *testing.T) { @@ -530,124 +522,6 @@ func TestAbort_Idempotent(t *testing.T) { require.NoError(t, ext.Abort(ctx, errors.New("second"))) } -// Init renames live->last and then fails before constructing newDir. Abort -// must restore live from last using the still-open storeDir. -func TestAbort_RecoversAfterInitFailure(t *testing.T) { - env, ctx := setupTestEnv(t) - env.populateUpdateDir(t, priorSubpath, nil) - env.makeLive(t) - - // Planting a regular file at newDirSubpath makes Init's MkdirAll fail - // after the live->last rename has already succeeded. - require.NoError(t, os.WriteFile(filepath.Join(env.storeDir, fixedNewDirSubpath), nil, 0o644)) //nolint:forbidigo // test code - - ext := &TransferFileExtension{} - _, err := ext.Init(ctx) - require.Error(t, err) - - _, err = os.Lstat(filepath.Join(env.storeDir, liveLink)) //nolint:forbidigo // test code - require.ErrorIs(t, err, fs.ErrNotExist) - lastTarget, err := os.Readlink(filepath.Join(env.storeDir, lastLink)) //nolint:forbidigo // test code - require.NoError(t, err) - require.Equal(t, priorSubpath, lastTarget) - - require.NoError(t, ext.Abort(ctx, errors.New("init failed"))) - - liveTarget, err := os.Readlink(filepath.Join(env.storeDir, liveLink)) //nolint:forbidigo // test code - require.NoError(t, err) - assert.Equal(t, priorSubpath, liveTarget) - - _, err = os.Lstat(filepath.Join(env.storeDir, lastLink)) //nolint:forbidigo // test code - require.ErrorIs(t, err, fs.ErrNotExist) - - // The blocking file is also gone -- RemoveAll(newDirSubpath) cleared it. - _, err = os.Lstat(filepath.Join(env.storeDir, fixedNewDirSubpath)) //nolint:forbidigo // test code - assert.ErrorIs(t, err, fs.ErrNotExist) -} - -func TestApplyTarget_NotForUs(t *testing.T) { - _, ctx := setupTestEnv(t) - ext := initExt(ctx, t) - - info := &tufclient.TargetInfo{ - TargetFiles: &tufmetadata.TargetFiles{Path: "regular/target.bin"}, - } - applied, err := ext.ApplyTarget(ctx, info) - require.NoError(t, err) - assert.False(t, applied) -} - -func TestApplyTarget_FullFlow(t *testing.T) { - env, ctx := setupTestEnv(t) - - body := []byte(strings.TrimSpace(` -[Source] -Type=url-file -Path=https://upstream.example.com/orig/ -MatchPattern=foo_@v.raw - -[Target] -Type=partition -MatchPattern=foo_@v -`) + "\n") - - const targetPath = ".quarry/sysupdate.d/foo.transfer" - srv := testrepo.New(t) - target := srv.WriteTarget(t, targetPath, bytes.NewReader(body)) - - proxyURL := "http://localhost:9999/" - ext := &TransferFileExtension{OverrideSourcePathURL: &proxyURL} - _, err := ext.Init(ctx) - require.NoError(t, err) - t.Cleanup(func() { _ = ext.Close() }) - - info := &tufclient.TargetInfo{ - TargetFiles: target, - Repo: makeRepo(t, srv, "testrepo"), - } - applied, err := ext.ApplyTarget(ctx, info) - require.NoError(t, err) - assert.True(t, applied) - - out, err := os.ReadFile(filepath.Join(env.storeDir, fixedNewDirSubpath, "sysupdate.d", "foo.transfer")) //nolint:forbidigo // test code - require.NoError(t, err) - - cfg, err := ini.Load(out) - require.NoError(t, err) - assert.Equal(t, proxyURL, cfg.Section("Source").Key("Path").String()) - assert.Equal(t, "url-file", cfg.Section("Source").Key("Type").String()) - assert.Equal(t, "partition", cfg.Section("Target").Key("Type").String()) -} - -// Uses a non-systemd-feature path shape (sysupdate.d/foo.transfer.d/x.conf) -// to exercise multi-level MkdirAll in ApplyTarget for future extensions. -func TestApplyTarget_NestedPath(t *testing.T) { - env, ctx := setupTestEnv(t) - - body := []byte("dummy=value\n") - - const targetPath = ".quarry/sysupdate.d/foo.transfer.d/x.conf" - srv := testrepo.New(t) - target := srv.WriteTarget(t, targetPath, bytes.NewReader(body)) - - ext := &TransferFileExtension{} - _, err := ext.Init(ctx) - require.NoError(t, err) - t.Cleanup(func() { _ = ext.Close() }) - - info := &tufclient.TargetInfo{ - TargetFiles: target, - Repo: makeRepo(t, srv, "testrepo"), - } - applied, err := ext.ApplyTarget(ctx, info) - require.NoError(t, err) - assert.True(t, applied) - - info2, err := os.Stat(filepath.Join(env.storeDir, fixedNewDirSubpath, "sysupdate.d", "foo.transfer.d", "x.conf")) //nolint:forbidigo // test code - require.NoError(t, err) - assert.True(t, info2.Mode().IsRegular()) -} - func TestClose_Idempotent(t *testing.T) { _, ctx := setupTestEnv(t) ext := initExt(ctx, t)