diff --git a/internal/devbox/packages.go b/internal/devbox/packages.go index 25d9ddefdfc..9d999cf3bdd 100644 --- a/internal/devbox/packages.go +++ b/internal/devbox/packages.go @@ -349,6 +349,15 @@ func (d *Devbox) ensureStateIsUpToDate(ctx context.Context, mode installMode) er if err := d.installPackages(ctx, mode); err != nil { return err } + // Installing resolves package definitions that had no lockfile + // entry yet, for example after their version was changed in + // devbox.json. Their env, init hooks and created files depend on + // the entry, so reload includes to apply them in this run. + if lo.ContainsBy(d.InstallablePackages(), (*devpkg.Package).IsPkgDef) { + if err := d.cfg.LoadRecursive(d.lockfile); err != nil { + return err + } + } } recomputeState := mode == ensure || d.IsEnvEnabled() diff --git a/internal/devbox/pkgdef.go b/internal/devbox/pkgdef.go index 30503a6f4e1..f4ffff14bbd 100644 --- a/internal/devbox/pkgdef.go +++ b/internal/devbox/pkgdef.go @@ -5,6 +5,8 @@ package devbox import ( "context" + "maps" + "slices" "strings" "github.com/samber/lo" @@ -70,8 +72,19 @@ func (d *Devbox) installPkgDef(ctx context.Context, pkg *devpkg.Package, recordD } artifact := pkgdef.Artifact{} - if info := locked.Systems[system]; info != nil { - artifact = pkgdef.Artifact{URL: info.URL, Sha256: info.Sha256} + info := locked.Systems[system] + if info == nil && len(locked.Systems) > 0 { + // The lockfile lists the systems the vendor publishes this version + // for, and this is not one of them. + return nil, usererr.New( + "Package %s resolved to %s@%s, which %s does not publish for %s (it is available for %s). "+ + "Pick another version, or run `devbox add %s --exclude-platform %s` to skip it on this machine.", + pkg.Raw, def.Name, locked.Version, def.Name, system, + strings.Join(slices.Sorted(maps.Keys(locked.Systems)), ", "), pkg.Raw, system, + ) + } + if info != nil { + artifact = pkgdef.Artifact{URL: info.URL, Sha256: info.Sha256, Sha512: info.Sha512} } if !client.IsInstalled(def, locked.Version, system) { @@ -79,7 +92,7 @@ func (d *Devbox) installPkgDef(ctx context.Context, pkg *devpkg.Package, recordD if def.UsesScripts() { ux.Fwarningf(d.stderr, "Package definition %s runs shell commands while installing.\n", ref.Source) } - if artifact.Sha256 == "" { + if !artifact.HasDigest() { ux.Fwarningf(d.stderr, "Package definition %s has no checksum for %s; the digest of this download will be recorded in devbox.lock.\n", ref.Source, system) } } @@ -91,7 +104,7 @@ func (d *Devbox) installPkgDef(ctx context.Context, pkg *devpkg.Package, recordD // Trust on first use: record the digest we observed so every other // machine verifies against it. - if recordDigest && artifact.Sha256 == "" && installed.Sha256 != "" { + if recordDigest && !artifact.HasDigest() && installed.Sha256 != "" { if err := d.lockfile.SetArtifactForPackage(pkg.Raw, system, installed.URL, installed.Sha256); err != nil { return nil, err } diff --git a/internal/lock/package.go b/internal/lock/package.go index e6c0acc2642..39006355c8e 100644 --- a/internal/lock/package.go +++ b/internal/lock/package.go @@ -33,11 +33,12 @@ type Package struct { type SystemInfo struct { Outputs []Output `json:"outputs,omitempty"` - // URL and Sha256 locate and verify the vendor artifact for packages - // with Source "pkg-def". Sha256 may be empty until the first install - // records it. + // URL and Sha256 or Sha512 locate and verify the vendor artifact for + // packages with Source "pkg-def". Both digests may be empty until the + // first install records a sha256. URL string `json:"url,omitempty"` Sha256 string `json:"sha256,omitempty"` + Sha512 string `json:"sha512,omitempty"` // Legacy Format StorePath string `json:"store_path,omitempty"` @@ -125,7 +126,8 @@ func (i *SystemInfo) Equals(other *SystemInfo) bool { return slices.Equal(i.Outputs, other.Outputs) && i.URL == other.URL && - i.Sha256 == other.Sha256 + i.Sha256 == other.Sha256 && + i.Sha512 == other.Sha512 } // If we have a StorePath and no Outputs, we need to convert to the new format. diff --git a/internal/lock/pkgdef.go b/internal/lock/pkgdef.go index 1dba97ef424..60a6a554e47 100644 --- a/internal/lock/pkgdef.go +++ b/internal/lock/pkgdef.go @@ -51,13 +51,15 @@ func (f *File) resolvePkgDef(ctx context.Context, raw string, refresh bool) (*Pa DefinitionHash: loaded.Hash, Systems: make(map[string]*SystemInfo, len(resolved.Artifacts)), } + // Systems the vendor publishes no artifact for are absent from + // Systems; installing on one of them is an error. for system, artifact := range resolved.Artifacts { - info := &SystemInfo{URL: artifact.URL, Sha256: artifact.Sha256} + info := &SystemInfo{URL: artifact.URL, Sha256: artifact.Sha256, Sha512: artifact.Sha512} // A definition without a checksum source gets its digest recorded // by the first install (trust on first use). Keep that digest when // re-resolving to the same artifact so `devbox update` doesn't // throw it away. - if info.Sha256 == "" && existing != nil && existing.Version == resolved.Version { + if !artifact.HasDigest() && existing != nil && existing.Version == resolved.Version { if old := existing.Systems[system]; old != nil && old.URL == artifact.URL { info.Sha256 = old.Sha256 } diff --git a/internal/pkgdef/artifact_test.go b/internal/pkgdef/artifact_test.go new file mode 100644 index 00000000000..9bfb60aebc5 --- /dev/null +++ b/internal/pkgdef/artifact_test.go @@ -0,0 +1,443 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package pkgdef + +import ( + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// vendorServer is an httptest server with per-path hit counts. +type vendorServer struct { + *httptest.Server + mux *http.ServeMux + mu sync.Mutex + hits map[string]int +} + +func newVendorServer(t *testing.T) *vendorServer { + t.Helper() + srv := &vendorServer{mux: http.NewServeMux(), hits: map[string]int{}} + srv.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + srv.mu.Lock() + srv.hits[r.URL.RequestURI()]++ + srv.mu.Unlock() + srv.mux.ServeHTTP(w, r) + })) + t.Cleanup(srv.Close) + return srv +} + +func (srv *vendorServer) hitCount(uri string) int { + srv.mu.Lock() + defer srv.mu.Unlock() + return srv.hits[uri] +} + +func (srv *vendorServer) client(t *testing.T) *Client { + t.Helper() + cli := NewClient() + cli.HTTPClient = srv.Client() + cli.CacheDir = filepath.Join(t.TempDir(), "pkgs") + cli.GitHubAPIURL = srv.URL + cli.GitHubToken = "" + return cli +} + +func parseDef(t *testing.T, format string, args ...any) *Definition { + t.Helper() + def, err := Parse([]byte(fmt.Sprintf(format, args...))) + require.NoError(t, err) + return def +} + +func TestParseDigest(t *testing.T) { + hex256 := strings.Repeat("ab", 32) + hex512 := strings.Repeat("cd", 64) + raw512, _ := hex.DecodeString(hex512) + sri512 := "sha512-" + base64.StdEncoding.EncodeToString(raw512) + + tests := []struct { + in, want string + wantOK bool + wantAlgo string + }{ + {hex256, "", true, algoSHA256}, + {strings.ToUpper(hex256), "", true, algoSHA256}, + {"sha256:" + hex256, "", true, algoSHA256}, + {hex512, "", true, algoSHA512}, + {"sha512:" + hex512, "", true, algoSHA512}, + {sri512, "", true, algoSHA512}, + {sri512 + " sha1-abc", "", true, algoSHA512}, // several SRI hashes + {hex256, algoSHA512, false, ""}, + {"sha1-" + base64.StdEncoding.EncodeToString([]byte("0123456789abcdefghij")), "", false, ""}, + {strings.Repeat("ab", 20), "", false, ""}, // sha1 length + {strings.Repeat("zz", 32), "", false, ""}, + {"sha512-not base64!", "", false, ""}, + } + for _, tt := range tests { + t.Run(tt.in, func(t *testing.T) { + got, err := parseDigest(tt.in, "src", tt.want) + if !tt.wantOK { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantAlgo, got.algo) + if tt.wantAlgo == algoSHA512 { + assert.Equal(t, hex512, got.hex) + } else { + assert.Equal(t, hex256, got.hex) + } + }) + } +} + +// TestLockSkipsSystemsWithoutArtifacts locks a version that the vendor +// ships for one platform only: the other platform is left out of the +// lock, and a version with no artifact anywhere is an error. +func TestLockSkipsSystemsWithoutArtifacts(t *testing.T) { + srv := newVendorServer(t) + sha := strings.Repeat("ab", 32) + srv.mux.HandleFunc("/dl/1.0.0/SHASUMS256.txt", func(w http.ResponseWriter, _ *http.Request) { + fmt.Fprintf(w, "%s x-1.0.0-linux-amd64.tar.gz\n", sha) + }) + def := parseDef(t, `{ + "schema": 1, "name": "x", "platforms": ["x86_64-linux", "aarch64-darwin"], + "versions": {"type": "static", "versions": ["1.0.0", "2.0.0"]}, + "install": { + "type": "archive", + "url": "%[1]s/dl/{{.Version}}/x-{{.Version}}-{{.OS}}-{{.Arch}}.tar.gz", + "checksum": {"type": "sha256_url", "url": "%[1]s/dl/{{.Version}}/SHASUMS256.txt"} + } + }`, srv.URL) + cli := srv.client(t) + + artifacts, err := cli.Lock(t.Context(), def, "1.0.0") + require.NoError(t, err) + assert.Equal(t, map[string]Artifact{ + "x86_64-linux": {URL: srv.URL + "/dl/1.0.0/x-1.0.0-linux-amd64.tar.gz", Sha256: sha}, + }, artifacts) + assert.Equal(t, 1, srv.hitCount("/dl/1.0.0/SHASUMS256.txt"), "the checksum file is fetched once for both systems") + + // 2.0.0 has no SHASUMS256.txt at all. + _, err = cli.Lock(t.Context(), def, "2.0.0") + require.ErrorIs(t, err, ErrNoArtifact) + assert.Contains(t, err.Error(), "aarch64-darwin") + assert.Contains(t, err.Error(), "x86_64-linux") +} + +// TestLookup resolves artifacts through a zig-style index that maps +// version and platform to a URL and checksum. +func TestLookup(t *testing.T) { + srv := newVendorServer(t) + tarball := makeTarGz(t, map[string]string{"x-1.0.0/x": "#!/bin/sh\necho x 1.0.0\n"}) + sha := sha256Hex(tarball) + srv.mux.HandleFunc("/index.json", func(w http.ResponseWriter, _ *http.Request) { + fmt.Fprintf(w, `{ + "1.0.0": {"x86_64-linux": {"tarball": "%[1]s/dl/x-1.0.0.tar.gz", "shasum": %[2]q}}, + "0.9.0": {"x86_64-linux": {"tarball": "%[1]s/dl/x-0.9.0.tar.gz"}}, + "0.8.0": {"x86_64-linux": {"tarball": null}} + }`, srv.URL, sha) + }) + srv.mux.HandleFunc("/dl/x-1.0.0.tar.gz", func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(tarball) + }) + def := parseDef(t, `{ + "schema": 1, "name": "x", "platforms": ["x86_64-linux", "aarch64-darwin"], + "os_map": {"linux": "linux"}, "arch_map": {"x86_64": "x86_64", "aarch64": "aarch64"}, + "versions": {"type": "http_json", "url": "%[1]s/index.json", "select": "keys[]"}, + "install": { + "type": "archive", + "lookup": {"url": "%[1]s/index.json", "select": ".[\"{{.Version}}\"][\"{{.System}}\"] // empty | {url: .tarball, checksum: .shasum}"}, + "strip_components": 1 + }, + "bin": ["."], + "binaries": ["x"] + }`, srv.URL) + cli := srv.client(t) + + resolved, err := cli.Resolve(t.Context(), def, "1") + require.NoError(t, err) + assert.Equal(t, map[string]Artifact{ + "x86_64-linux": {URL: srv.URL + "/dl/x-1.0.0.tar.gz", Sha256: sha}, + }, resolved.Artifacts, "aarch64-darwin has no entry and is skipped") + + // An entry without a checksum is locked without one. + artifacts, err := cli.Lock(t.Context(), def, "0.9.0") + require.NoError(t, err) + assert.Equal(t, Artifact{URL: srv.URL + "/dl/x-0.9.0.tar.gz"}, artifacts["x86_64-linux"]) + + _, err = cli.Lock(t.Context(), def, "0.8.0") + require.ErrorContains(t, err, `produced an object without "url"`) + + // Install without a locked artifact looks it up too, and verifies it. + installed, err := cli.Install(t.Context(), def, "1.0.0", "x86_64-linux", Artifact{}) + require.NoError(t, err) + assert.Equal(t, sha, installed.Sha256) + assert.FileExists(t, filepath.Join(installed.Dir, "x")) + + assert.Equal(t, 1, srv.hitCount("/index.json"), "the index is fetched once per client") +} + +func TestLookupIndexNotFound(t *testing.T) { + srv := newVendorServer(t) + def := parseDef(t, `{ + "schema": 1, "name": "x", "platforms": ["x86_64-linux"], + "versions": {"type": "static", "versions": ["8.0.1"]}, + "install": { + "type": "archive", + "lookup": {"url": "%s/feature/{{ index (splitList \".\" .Version) 0 }}.json", "select": ".[0]"} + } + }`, srv.URL) + _, err := srv.client(t).Lock(t.Context(), def, "8.0.1") + require.ErrorIs(t, err, ErrNoArtifact) + assert.Equal(t, 1, srv.hitCount("/feature/8.json")) +} + +func TestChecksumGitHubRelease(t *testing.T) { + srv := newVendorServer(t) + sha := strings.Repeat("ef", 32) + srv.mux.HandleFunc("/repos/acme/x/releases/tags/v1.0.0", func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "Bearer secret", r.Header.Get("Authorization")) + fmt.Fprintf(w, `{"assets": [ + {"name": "x-linux-amd64.tar.gz", "digest": "sha256:%s"}, + {"name": "x-darwin-arm64.tar.gz", "digest": null} + ]}`, sha) + }) + def := parseDef(t, `{ + "schema": 1, "name": "x", "platforms": ["x86_64-linux", "aarch64-darwin", "aarch64-linux"], + "versions": {"type": "static", "versions": ["1.0.0", "2.0.0"]}, + "install": { + "type": "archive", + "url": "https://github.com/acme/x/releases/download/v{{.Version}}/x-{{.OS}}-{{.Arch}}.tar.gz", + "checksum": {"type": "github_release", "repo": "acme/x"} + } + }`) + cli := srv.client(t) + cli.GitHubToken = "secret" + + artifacts, err := cli.Lock(t.Context(), def, "1.0.0") + require.NoError(t, err) + assert.Equal(t, sha, artifacts["x86_64-linux"].Sha256) + assert.False(t, artifacts["aarch64-darwin"].HasDigest(), "an asset without a digest is locked without one") + assert.NotContains(t, artifacts, "aarch64-linux", "a platform without an asset is skipped") + assert.Equal(t, 1, srv.hitCount("/repos/acme/x/releases/tags/v1.0.0"), "the release is fetched once") + + _, err = cli.Lock(t.Context(), def, "2.0.0") + require.ErrorIs(t, err, ErrNoArtifact, "a missing release means no artifacts") +} + +func TestInstallVerifiesSHA512(t *testing.T) { + srv := newVendorServer(t) + tarball := makeTarGz(t, map[string]string{"package/bin/x": "#!/bin/sh\necho x\n"}) + sum := sha512.Sum512(tarball) + integrity := "sha512-" + base64.StdEncoding.EncodeToString(sum[:]) + srv.mux.HandleFunc("/x/1.0.0", func(w http.ResponseWriter, _ *http.Request) { + fmt.Fprintf(w, `{"dist": {"integrity": %q}}`, integrity) + }) + srv.mux.HandleFunc("/x/-/x-1.0.0.tgz", func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(tarball) + }) + def := parseDef(t, `{ + "schema": 1, "name": "x", "platforms": ["x86_64-linux"], + "versions": {"type": "static", "versions": ["1.0.0"]}, + "install": { + "type": "archive", + "url": "%[1]s/x/-/x-{{.Version}}.tgz", + "checksum": {"type": "http_json", "url": "%[1]s/x/{{.Version}}", "select": ".dist.integrity"}, + "strip_components": 1 + }, + "binaries": ["x"] + }`, srv.URL) + cli := srv.client(t) + + artifacts, err := cli.Lock(t.Context(), def, "1.0.0") + require.NoError(t, err) + artifact := artifacts["x86_64-linux"] + assert.Equal(t, hex.EncodeToString(sum[:]), artifact.Sha512) + assert.Empty(t, artifact.Sha256) + + bad := artifact + bad.Sha512 = strings.Repeat("00", 64) + _, err = cli.Install(t.Context(), def, "1.0.0", "x86_64-linux", bad) + require.ErrorIs(t, err, ErrChecksumMismatch) + assert.Contains(t, err.Error(), "sha512") + + installed, err := cli.Install(t.Context(), def, "1.0.0", "x86_64-linux", artifact) + require.NoError(t, err) + assert.FileExists(t, filepath.Join(installed.Dir, "bin", "x")) + + // The marker remembers the sha512, so a lockfile that disagrees with + // what is installed is caught without downloading again. + _, err = cli.Install(t.Context(), def, "1.0.0", "x86_64-linux", bad) + require.ErrorIs(t, err, ErrChecksumMismatch) +} + +// TestBinaries checks that listed binaries are made executable (npm +// tarballs drop the bit) and that a definition whose binaries are missing +// fails to install. +func TestBinaries(t *testing.T) { + srv := newVendorServer(t) + tarball := makeTarGz(t, map[string]string{"bin/x": "#!/bin/sh\necho x\n"}) + srv.mux.HandleFunc("/x.tar.gz", func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(tarball) + }) + const tmpl = `{ + "schema": 1, "name": "x", "platforms": ["x86_64-linux"], + "versions": {"type": "static", "versions": ["1.0.0"]}, + "install": {"type": "archive", "url": "%s/x.tar.gz"}, + "binaries": %s + }` + + cli := srv.client(t) + def := parseDef(t, tmpl, srv.URL, `["x"]`) + installed, err := cli.Install(t.Context(), def, "1.0.0", "x86_64-linux", Artifact{}) + require.NoError(t, err) + info, err := os.Stat(filepath.Join(installed.Dir, "bin", "x")) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o755), info.Mode().Perm()) + + def = parseDef(t, tmpl, srv.URL, `["x", "y"]`) + _, err = srv.client(t).Install(t.Context(), def, "1.0.0", "x86_64-linux", Artifact{}) + require.ErrorContains(t, err, "y not found") +} + +// TestPostInstallWithoutPATH runs a hook while PATH is empty, as it is +// when devbox runs under testscript. +func TestPostInstallWithoutPATH(t *testing.T) { + srv := newVendorServer(t) + srv.mux.HandleFunc("/x", func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("#!/bin/sh\n")) + }) + def := parseDef(t, `{ + "schema": 1, "name": "x", "platforms": ["x86_64-linux"], + "versions": {"type": "static", "versions": ["1.0.0"]}, + "install": {"type": "binary", "url": "%s/x"}, + "hooks": {"post_install": ["echo ok > hook.txt && ln -s hook.txt link.txt"]} + }`, srv.URL) + t.Setenv("PATH", "") + installed, err := srv.client(t).Install(t.Context(), def, "1.0.0", "x86_64-linux", Artifact{}) + require.NoError(t, err) + assert.FileExists(t, filepath.Join(installed.Dir, "link.txt"), "system utilities are on the hook's PATH") +} + +// TestVersionSourceUnion resolves against two sources, like yarn 1 and +// yarn 2+ living in different npm packages. +func TestVersionSourceUnion(t *testing.T) { + srv := newVendorServer(t) + srv.mux.HandleFunc("/berry", func(w http.ResponseWriter, _ *http.Request) { + fmt.Fprint(w, `{"versions": {"4.5.0": {}, "3.8.7": {}, "4.6.0-rc.1": {}}}`) + }) + srv.mux.HandleFunc("/classic", func(w http.ResponseWriter, _ *http.Request) { + fmt.Fprint(w, `{"versions": {"1.22.22": {}, "1.22.19": {}}}`) + }) + def := parseDef(t, `{ + "schema": 1, "name": "x", "platforms": ["x86_64-linux"], + "versions": [ + {"type": "http_json", "url": "%[1]s/berry", "select": ".versions | keys[]"}, + {"type": "http_json", "url": "%[1]s/classic", "select": ".versions | keys[]"} + ], + "install": {"type": "binary", "url": "https://example.com/x"} + }`, srv.URL) + cli := srv.client(t) + + for requested, want := range map[string]string{"latest": "4.5.0", "1": "1.22.22", "3": "3.8.7", "4.6.0-rc.1": "4.6.0-rc.1"} { + got, err := cli.ResolveVersion(t.Context(), def, requested) + require.NoError(t, err, requested) + assert.Equal(t, want, got, requested) + } +} + +// TestHTTPJSONPagination pages through an Adoptium-style API that returns +// 404 past the last page. +func TestHTTPJSONPagination(t *testing.T) { + srv := newVendorServer(t) + pages := [][]string{{"27.0.1+0", "21.0.5+11"}, {"17.0.13+11", "11.0.25+9"}, {"8.0.432+6"}} + srv.mux.HandleFunc("/versions", func(w http.ResponseWriter, r *http.Request) { + page, _ := strconv.Atoi(r.URL.Query().Get("page")) + if page >= len(pages) { + http.NotFound(w, r) + return + } + fmt.Fprintf(w, `{"versions": [%s]}`, `"`+strings.Join(pages[page], `", "`)+`"`) + }) + def := parseDef(t, `{ + "schema": 1, "name": "x", "platforms": ["x86_64-linux"], + "versions": {"type": "http_json", "url": "%s/versions?page={{.Page}}", "select": ".versions[]"}, + "install": {"type": "binary", "url": "https://example.com/x"} + }`, srv.URL) + + // Resolution stops at the first page with a match. + cli := srv.client(t) + got, err := cli.ResolveVersion(t.Context(), def, "latest") + require.NoError(t, err) + assert.Equal(t, "27.0.1+0", got) + assert.Equal(t, 0, srv.hitCount("/versions?page=1")) + + got, err = cli.ResolveVersion(t.Context(), def, "17") + require.NoError(t, err) + assert.Equal(t, "17.0.13+11", got) + assert.Equal(t, 0, srv.hitCount("/versions?page=2")) + + // Listing reads every page until the 404. + versions, err := cli.ListVersions(t.Context(), def) + require.NoError(t, err) + assert.Len(t, versions, 5) + assert.Equal(t, 1, srv.hitCount("/versions?page=3")) + assert.Equal(t, 0, srv.hitCount("/versions?page=4")) +} + +// TestGitHubReleasesPagination reads more pages of releases only when the +// requested version is not on the first one. +func TestGitHubReleasesPagination(t *testing.T) { + srv := newVendorServer(t) + srv.mux.HandleFunc("/repos/acme/x/releases", func(w http.ResponseWriter, r *http.Request) { + page, _ := strconv.Atoi(r.URL.Query().Get("page")) + var tags []string + switch page { + case 0, 1: // a full page of 3.x + for i := range githubPerPage { + tags = append(tags, fmt.Sprintf(`{"tag_name": "v3.%d.0"}`, githubPerPage-i)) + } + case 2: + tags = []string{`{"tag_name": "v2.1.0"}`, `{"tag_name": "v1.0.0"}`} + } + fmt.Fprintf(w, "[%s]", strings.Join(tags, ",")) + }) + def := parseDef(t, `{ + "schema": 1, "name": "x", "platforms": ["x86_64-linux"], + "versions": {"type": "github_releases", "repo": "acme/x"}, + "install": {"type": "binary", "url": "https://example.com/x"} + }`) + cli := srv.client(t) + + got, err := cli.ResolveVersion(t.Context(), def, "latest") + require.NoError(t, err) + assert.Equal(t, "3.100.0", got) + assert.Equal(t, 0, srv.hitCount("/repos/acme/x/releases?per_page=100&page=2")) + + got, err = cli.ResolveVersion(t.Context(), def, "2") + require.NoError(t, err) + assert.Equal(t, "2.1.0", got) + assert.Equal(t, 1, srv.hitCount("/repos/acme/x/releases?per_page=100&page=2")) + + // A short page is the last one. + _, err = cli.ResolveVersion(t.Context(), def, "9") + require.ErrorIs(t, err, ErrVersionNotFound) + assert.Equal(t, 0, srv.hitCount("/repos/acme/x/releases?per_page=100&page=3")) +} diff --git a/internal/pkgdef/checksum.go b/internal/pkgdef/checksum.go index 0a592817dcb..6bb4fe8a5ba 100644 --- a/internal/pkgdef/checksum.go +++ b/internal/pkgdef/checksum.go @@ -5,71 +5,146 @@ package pkgdef import ( "context" + "encoding/base64" "encoding/hex" "encoding/json" + "errors" "fmt" + "net/url" "strings" ) -// fetchChecksum returns the lowercase hex sha256 for the artifact described -// by vars, or "" when the checksum type is "none". -func (c *Client) fetchChecksum(ctx context.Context, def *Definition, src *Checksum, vars Vars) (string, error) { +// Digest algorithms. +const ( + algoSHA256 = "sha256" + algoSHA512 = "sha512" +) + +// digest is a parsed checksum: an algorithm and the lowercase hex digest. +// The zero value means "no checksum". +type digest struct { + algo string + hex string +} + +// apply records d on the artifact. +func (d digest) apply(a *Artifact) { + switch d.algo { + case algoSHA256: + a.Sha256 = d.hex + case algoSHA512: + a.Sha512 = d.hex + } +} + +// fetchChecksum returns the digest for the artifact described by vars, or +// the zero digest when the checksum type is "none" or the vendor publishes +// no digest for this artifact. It returns ErrNoArtifact when the checksum +// source shows that the artifact does not exist. +func (c *Client) fetchChecksum(ctx context.Context, def *Definition, src *Checksum, vars Vars) (digest, error) { if src == nil || src.Type == ChecksumNone { - return "", nil + return digest{}, nil + } + if src.Type == ChecksumGitHubRelease { + return c.fetchGitHubReleaseDigest(ctx, src, vars) } rawURL, err := Render(src.URL, vars) if err != nil { - return "", err + return digest{}, err } rawURL = def.ResolveURL(rawURL) switch src.Type { case ChecksumHTTPJSON: return c.fetchChecksumHTTPJSON(ctx, src, rawURL, vars) - case ChecksumSHA256URL: - body, err := c.get(ctx, rawURL) + case ChecksumSHA256URL, ChecksumSHA512URL: + body, err := c.getCached(ctx, rawURL) + if errors.Is(err, ErrNotFound) { + return digest{}, fmt.Errorf("checksum: %w: %w", ErrNoArtifact, err) + } if err != nil { - return "", fmt.Errorf("checksum: %w", err) + return digest{}, fmt.Errorf("checksum: %w", err) + } + algo := algoSHA256 + if src.Type == ChecksumSHA512URL { + algo = algoSHA512 } - return parseSHA256Body(string(body), vars.Filename, rawURL) + return parseChecksumFile(string(body), vars.Filename, rawURL, algo) default: - return "", fmt.Errorf("checksum.type %q is not supported", src.Type) + return digest{}, fmt.Errorf("checksum.type %q is not supported", src.Type) } } -func (c *Client) fetchChecksumHTTPJSON(ctx context.Context, src *Checksum, rawURL string, vars Vars) (string, error) { +func (c *Client) fetchChecksumHTTPJSON(ctx context.Context, src *Checksum, rawURL string, vars Vars) (digest, error) { expr, err := Render(src.Select, vars) if err != nil { - return "", err + return digest{}, err } - code, err := compileJQ(expr) + results, err := c.queryJSON(ctx, rawURL, expr) if err != nil { - return "", err + return digest{}, fmt.Errorf("checksum: %w", err) + } + for _, r := range results { + if s, ok := r.(string); ok && s != "" { + return parseDigest(s, rawURL, "") + } } - body, err := c.get(ctx, rawURL) + return digest{}, fmt.Errorf("checksum: %w: %s: jq expression %q matched nothing for %s", + ErrNoArtifact, rawURL, expr, vars.Filename) +} + +// githubRelease is the subset of the GitHub release API response used for +// asset digests. +type githubReleaseAssets struct { + Assets []struct { + Name string `json:"name"` + Digest *string `json:"digest"` + } `json:"assets"` +} + +// fetchGitHubReleaseDigest reads the digest GitHub computed for a release +// asset. Releases published before GitHub started computing digests have +// none, which is reported as the zero digest (trust on first use). +func (c *Client) fetchGitHubReleaseDigest(ctx context.Context, src *Checksum, vars Vars) (digest, error) { + tagTemplate := src.Tag + if tagTemplate == "" { + tagTemplate = "v{{.Version}}" + } + tag, err := Render(tagTemplate, vars) if err != nil { - return "", fmt.Errorf("checksum: %w", err) + return digest{}, err } - var input any - if err := json.Unmarshal(body, &input); err != nil { - return "", fmt.Errorf("checksum: %s: response is not JSON: %w", rawURL, err) + body, err := c.getGitHubAPI(ctx, "repos/"+src.Repo+"/releases/tags/"+url.PathEscape(tag)) + if errors.Is(err, ErrNotFound) { + return digest{}, fmt.Errorf("checksum: %w: %w", ErrNoArtifact, err) } - results, err := runJQ(code, expr, input) if err != nil { - return "", fmt.Errorf("checksum: %w", err) + return digest{}, fmt.Errorf("checksum: github release %s %s: %w", src.Repo, tag, err) } - if len(results) == 0 { - return "", fmt.Errorf("checksum: %s: jq expression %q matched nothing for %s", rawURL, expr, vars.Filename) + var release githubReleaseAssets + if err := json.Unmarshal(body, &release); err != nil { + return digest{}, fmt.Errorf("checksum: github release %s %s: response is not JSON: %w", src.Repo, tag, err) } - return normalizeSHA256(results[0], rawURL) + for _, asset := range release.Assets { + if asset.Name != vars.Filename { + continue + } + if asset.Digest == nil || *asset.Digest == "" { + return digest{}, nil + } + return parseDigest(*asset.Digest, src.Repo+" "+tag, "") + } + return digest{}, fmt.Errorf("checksum: %w: github release %s %s has no asset %s", + ErrNoArtifact, src.Repo, tag, vars.Filename) } -// parseSHA256Body extracts the digest for filename from a checksum file. +// parseChecksumFile extracts the digest for filename from a checksum file. // The body is either a bare digest or a sha256sum-style list of -// " " lines (a leading "*" marks binary mode). -func parseSHA256Body(body, filename, source string) (string, error) { +// " " lines (a leading "*" marks binary mode). A list +// without an entry for filename means the artifact does not exist. +func parseChecksumFile(body, filename, source, algo string) (digest, error) { fields := strings.Fields(body) if len(fields) == 1 { - return normalizeSHA256(fields[0], source) + return parseDigest(fields[0], source, algo) } for line := range strings.SplitSeq(body, "\n") { parts := strings.Fields(line) @@ -78,19 +153,53 @@ func parseSHA256Body(body, filename, source string) (string, error) { } name := strings.TrimPrefix(parts[1], "*") if name == filename || strings.HasSuffix(name, "/"+filename) { - return normalizeSHA256(parts[0], source) + return parseDigest(parts[0], source, algo) } } - return "", fmt.Errorf("checksum: %s has no entry for %s", source, filename) + return digest{}, fmt.Errorf("checksum: %w: %s has no entry for %s", ErrNoArtifact, source, filename) } -func normalizeSHA256(digest, source string) (string, error) { - digest = strings.ToLower(strings.TrimSpace(strings.TrimPrefix(digest, "sha256:"))) - if len(digest) != 64 { - return "", fmt.Errorf("checksum: %s: %q is not a hex sha256", source, digest) +// parseDigest parses a digest in one of the forms vendors publish: bare hex +// (the algorithm follows from the length), "sha256:", or an SRI string +// such as "sha512-". When want is not empty the digest must use +// that algorithm. +func parseDigest(value, source, want string) (digest, error) { + value = strings.TrimSpace(value) + invalid := fmt.Errorf("checksum: %s: %q is not a sha256 or sha512 digest", source, value) + + var parsed digest + switch { + case strings.HasPrefix(value, "sha256:"), strings.HasPrefix(value, "sha512:"): + algo, hexDigest, _ := strings.Cut(value, ":") + parsed = digest{algo: algo, hex: strings.ToLower(hexDigest)} + case strings.HasPrefix(value, "sha256-"), strings.HasPrefix(value, "sha512-"): + // SRI allows several space-separated hashes; the first is enough. + first, _, _ := strings.Cut(value, " ") + algo, b64, _ := strings.Cut(first, "-") + raw, err := base64.StdEncoding.DecodeString(b64) + if err != nil { + return digest{}, invalid + } + parsed = digest{algo: algo, hex: hex.EncodeToString(raw)} + default: + parsed = digest{hex: strings.ToLower(value)} + switch len(value) { + case 64: + parsed.algo = algoSHA256 + case 128: + parsed.algo = algoSHA512 + } + } + + wantLen := map[string]int{algoSHA256: 64, algoSHA512: 128}[parsed.algo] + if wantLen == 0 || len(parsed.hex) != wantLen { + return digest{}, invalid + } + if _, err := hex.DecodeString(parsed.hex); err != nil { + return digest{}, invalid } - if _, err := hex.DecodeString(digest); err != nil { - return "", fmt.Errorf("checksum: %s: %q is not a hex sha256", source, digest) + if want != "" && parsed.algo != want { + return digest{}, fmt.Errorf("checksum: %s: expected a %s digest, got %s", source, want, parsed.algo) } - return digest, nil + return parsed, nil } diff --git a/internal/pkgdef/client.go b/internal/pkgdef/client.go index f33f722a6dd..f99f4c6a52d 100644 --- a/internal/pkgdef/client.go +++ b/internal/pkgdef/client.go @@ -6,6 +6,7 @@ package pkgdef import ( "bytes" "context" + "encoding/json" "errors" "fmt" "io" @@ -15,6 +16,7 @@ import ( "runtime" "strconv" "strings" + "sync" "time" "github.com/itchyny/gojq" @@ -51,6 +53,12 @@ type Client struct { // Stderr receives the output of post_install hooks. Defaults to // io.Discard. Stderr io.Writer + + // responses memoizes successful GETs (indexes, checksum files, API + // responses) for the lifetime of the client, so that locking every + // system of a version fetches a shared index once. + responsesMu sync.Mutex + responses map[string][]byte } // NewClient returns a client that caches under $XDG_CACHE_HOME/devbox/pkgs @@ -87,21 +95,27 @@ func (c *Client) userAgent() string { // tests and local mirrors work without a server. No credentials are ever // attached: definitions are untrusted input and may point anywhere. func (c *Client) get(ctx context.Context, rawURL string) ([]byte, error) { - return c.getWithAuth(ctx, rawURL, "") + return c.getWithAuth(ctx, rawURL, "", false) +} + +// getCached is get for vendor metadata (version indexes, checksum files): +// a successful response is reused for the lifetime of the client. +func (c *Client) getCached(ctx context.Context, rawURL string) ([]byte, error) { + return c.getWithAuth(ctx, rawURL, "", true) } // getGitHubAPI fetches a GitHub REST API path (relative to GitHubAPIURL) -// with the configured token, if any. +// with the configured token, if any. Responses are cached like getCached. func (c *Client) getGitHubAPI(ctx context.Context, apiPath string) ([]byte, error) { endpoint := strings.TrimSuffix(c.GitHubAPIURL, "/") + "/" + strings.TrimPrefix(apiPath, "/") auth := "" if c.GitHubToken != "" { auth = "Bearer " + c.GitHubToken } - return c.getWithAuth(ctx, endpoint, auth) + return c.getWithAuth(ctx, endpoint, auth, true) } -func (c *Client) getWithAuth(ctx context.Context, rawURL, authorization string) ([]byte, error) { +func (c *Client) getWithAuth(ctx context.Context, rawURL, authorization string, cache bool) ([]byte, error) { if p, ok := strings.CutPrefix(rawURL, "file://"); ok { b, err := os.ReadFile(p) if errors.Is(err, os.ErrNotExist) { @@ -110,12 +124,52 @@ func (c *Client) getWithAuth(ctx context.Context, rawURL, authorization string) return b, err } + if cache { + c.responsesMu.Lock() + cached, ok := c.responses[rawURL] + c.responsesMu.Unlock() + if ok { + return cached, nil + } + } + body, err := c.open(ctx, rawURL, authorization) if err != nil { return nil, err } defer body.Close() - return io.ReadAll(body) + data, err := io.ReadAll(body) + if err != nil { + return nil, err + } + + if cache { + c.responsesMu.Lock() + if c.responses == nil { + c.responses = map[string][]byte{} + } + c.responses[rawURL] = data + c.responsesMu.Unlock() + } + return data, nil +} + +// queryJSON fetches rawURL and evaluates the jq expression against it, +// returning every value it emits. +func (c *Client) queryJSON(ctx context.Context, rawURL, expr string) ([]any, error) { + code, err := compileJQ(expr) + if err != nil { + return nil, err + } + body, err := c.getCached(ctx, rawURL) + if err != nil { + return nil, err + } + var input any + if err := json.Unmarshal(body, &input); err != nil { + return nil, fmt.Errorf("%s: response is not JSON: %w", rawURL, err) + } + return runJQValues(code, expr, input) } // open starts a GET request and returns the response body. The caller must @@ -170,11 +224,10 @@ func compileJQ(expr string) (*gojq.Code, error) { return code, nil } -// runJQ evaluates a compiled jq expression against input and returns every -// string it emits. Numbers are formatted as strings; arrays are flattened -// one level so that ".[].version" and "[.[].version]" both work. -func runJQ(code *gojq.Code, expr string, input any) ([]string, error) { - var out []string +// runJQValues evaluates a compiled jq expression against input and returns +// every value it emits. +func runJQValues(code *gojq.Code, expr string, input any) ([]any, error) { + var out []any iter := code.Run(input) for { value, ok := iter.Next() @@ -184,6 +237,21 @@ func runJQ(code *gojq.Code, expr string, input any) ([]string, error) { if err, isErr := value.(error); isErr { return nil, fmt.Errorf("jq expression %q: %w", expr, err) } + out = append(out, value) + } + return out, nil +} + +// runJQ evaluates a compiled jq expression against input and returns every +// string it emits. Numbers are formatted as strings; arrays are flattened +// one level so that ".[].version" and "[.[].version]" both work. +func runJQ(code *gojq.Code, expr string, input any) ([]string, error) { + values, err := runJQValues(code, expr, input) + if err != nil { + return nil, err + } + var out []string + for _, value := range values { if arr, isArr := value.([]any); isArr { for _, elem := range arr { out = appendJQValue(out, elem) diff --git a/internal/pkgdef/client_test.go b/internal/pkgdef/client_test.go index 12c4cedc3d8..279a148a0fb 100644 --- a/internal/pkgdef/client_test.go +++ b/internal/pkgdef/client_test.go @@ -291,32 +291,37 @@ func TestChecksumSHA256URL(t *testing.T) { require.NoError(t, err) assert.Equal(t, vendor.binSHA, artifacts[testSystem].Sha256) - // Missing checksum file is an error. + // A missing checksum file means there is no artifact for any system. def = vendor.binaryDef(t, fmt.Sprintf(`"checksum": {"type": "sha256_url", "url": "%s/dl/missing.txt"}`, vendor.srv.URL)) _, err = cli.Lock(t.Context(), def, "1.2.0") - require.ErrorIs(t, err, ErrNotFound) + require.ErrorIs(t, err, ErrNoArtifact) } -func TestParseSHA256Body(t *testing.T) { - digest := strings.Repeat("0a", 32) - got, err := parseSHA256Body(digest+"\n", "x", "src") +func TestParseChecksumFile(t *testing.T) { + hexDigest := strings.Repeat("0a", 32) + want := digest{algo: algoSHA256, hex: hexDigest} + got, err := parseChecksumFile(hexDigest+"\n", "x", "src", algoSHA256) require.NoError(t, err) - assert.Equal(t, digest, got) + assert.Equal(t, want, got) - got, err = parseSHA256Body("sha256:"+strings.ToUpper(digest), "x", "src") + got, err = parseChecksumFile("sha256:"+strings.ToUpper(hexDigest), "x", "src", algoSHA256) require.NoError(t, err) - assert.Equal(t, digest, got) + assert.Equal(t, want, got) - body := strings.Repeat("11", 32) + " other.tar.gz\n" + digest + " dist/x.tar.gz\n" - got, err = parseSHA256Body(body, "x.tar.gz", "src") + body := strings.Repeat("11", 32) + " other.tar.gz\n" + hexDigest + " dist/x.tar.gz\n" + got, err = parseChecksumFile(body, "x.tar.gz", "src", algoSHA256) require.NoError(t, err) - assert.Equal(t, digest, got) + assert.Equal(t, want, got) - _, err = parseSHA256Body(body, "nope.tar.gz", "src") - require.Error(t, err) + _, err = parseChecksumFile(body, "nope.tar.gz", "src", algoSHA256) + require.ErrorIs(t, err, ErrNoArtifact) - _, err = parseSHA256Body("nothex", "x", "src") + _, err = parseChecksumFile("nothex", "x", "src", algoSHA256) require.Error(t, err) + + // A sha512 file must hold a sha512 digest. + _, err = parseChecksumFile(hexDigest, "x", "src", algoSHA512) + require.ErrorContains(t, err, "expected a sha512 digest") } func TestListVersionsGitHub(t *testing.T) { @@ -339,7 +344,7 @@ func TestListVersionsGitHub(t *testing.T) { // An empty strip_prefix keeps every tag. empty := "" - def.Versions.StripPrefix = &empty + def.Versions[0].StripPrefix = &empty versions, err = cli.ListVersions(t.Context(), def) require.NoError(t, err) assert.Equal(t, []string{"v1.4.2", "v1.4.1", "nightly"}, versions) @@ -496,7 +501,7 @@ func TestGitHubTokenOnlyForGitHubAPI(t *testing.T) { _, err = cli.ListVersions(t.Context(), def) require.NoError(t, err) - def.Versions = VersionSource{Type: VersionsGitHubReleases, Repo: "acme/x"} + def.Versions = VersionSources{{Type: VersionsGitHubReleases, Repo: "acme/x"}} _, err = cli.ListVersions(t.Context(), def) require.NoError(t, err) @@ -613,7 +618,7 @@ func TestInstallRedoneWhenRecipeChanges(t *testing.T) { // Activation-only edits do not invalidate the install. def.Env = map[string]string{"X": "y"} assert.True(t, cli.IsInstalled(def, "1.2.0", testSystem)) - assert.Equal(t, def.InstallFingerprint(testSystem), def.InstallFingerprint(testSystem)) + assert.Equal(t, def.InstallFingerprint(testSystem, "1.2.0"), def.InstallFingerprint(testSystem, "1.2.0")) // Changing the install recipe does: the hook now writes a different file. def.Hooks.PostInstall = []string{"echo redone > $DEVBOX_PKG_INSTALL_DIR/redone.txt"} diff --git a/internal/pkgdef/definition.go b/internal/pkgdef/definition.go index 04dda5de38c..725e79fbd80 100644 --- a/internal/pkgdef/definition.go +++ b/internal/pkgdef/definition.go @@ -11,6 +11,7 @@ package pkgdef import ( + "bytes" "crypto/sha256" "encoding/hex" "encoding/json" @@ -56,9 +57,11 @@ const ( // Checksum source types. const ( - ChecksumHTTPJSON = "http_json" - ChecksumSHA256URL = "sha256_url" - ChecksumNone = "none" + ChecksumHTTPJSON = "http_json" + ChecksumSHA256URL = "sha256_url" + ChecksumSHA512URL = "sha512_url" + ChecksumGitHubRelease = "github_release" + ChecksumNone = "none" ) // Definition is a parsed package definition file. @@ -73,8 +76,8 @@ type Definition struct { OSMap map[string]string `json:"os_map,omitempty"` ArchMap map[string]string `json:"arch_map,omitempty"` - Versions VersionSource `json:"versions"` - Install Install `json:"install"` + Versions VersionSources `json:"versions"` + Install Install `json:"install"` Bin []string `json:"bin,omitempty"` Binaries []string `json:"binaries,omitempty"` @@ -104,6 +107,40 @@ func (d *Definition) ResolveURL(rawURL string) string { return "file://" + filepath.Join(d.baseDir, p) } +// VersionSources is one version source or a list of them. In JSON it is +// either an object or an array of objects. A list is the union of its +// sources, for packages whose versions are published in more than one place +// (yarn 1.x and yarn 2+ live in different npm packages). +type VersionSources []VersionSource + +// UnmarshalJSON accepts a single source object or an array of them. +func (v *VersionSources) UnmarshalJSON(data []byte) error { + data = bytes.TrimSpace(data) + if len(data) > 0 && data[0] == '[' { + var list []VersionSource + if err := json.Unmarshal(data, &list); err != nil { + return err + } + *v = list + return nil + } + var single VersionSource + if err := json.Unmarshal(data, &single); err != nil { + return err + } + *v = VersionSources{single} + return nil +} + +// MarshalJSON writes a single source as an object so that a definition +// round-trips to the shape it was written in. +func (v VersionSources) MarshalJSON() ([]byte, error) { + if len(v) == 1 { + return json.Marshal(v[0]) + } + return json.Marshal([]VersionSource(v)) +} + // VersionSource describes how to list the available versions of a package. type VersionSource struct { Type string `json:"type"` @@ -111,7 +148,8 @@ type VersionSource struct { // static Versions []string `json:"versions,omitempty"` - // http_json + // http_json. When URL contains {{.Page}}, pages 0, 1, ... are fetched + // until one returns 404 or no versions. URL string `json:"url,omitempty"` Select string `json:"select,omitempty"` @@ -128,9 +166,10 @@ type VersionSource struct { type Install struct { Type string `json:"type"` - // archive and binary + // archive and binary: either url (and checksum) or lookup. URL string `json:"url,omitempty"` Checksum *Checksum `json:"checksum,omitempty"` + Lookup *Lookup `json:"lookup,omitempty"` // archive StripComponents *int `json:"strip_components,omitempty"` @@ -151,13 +190,42 @@ type Install struct { // fields replace the defaults for that system. Overrides may not // themselves contain overrides. Overrides map[string]*Install `json:"overrides,omitempty"` + + // VersionOverrides are partial Installs for the versions matching + // their constraint, for vendors that changed how they ship artifacts. + // The first matching entry applies. An entry may carry its own + // per-system overrides but no further version overrides. + VersionOverrides []*VersionOverride `json:"version_overrides,omitempty"` +} + +// VersionOverride is an Install that applies to the versions matching +// Versions, a constraint such as "<11" or ">=2 <4". See MatchesConstraint. +type VersionOverride struct { + Versions string `json:"versions"` + Install } -// Checksum describes where to find the sha256 of an artifact. +// Lookup finds an artifact in a JSON index: the jq expression in Select is +// evaluated against the document at URL and must produce an object with +// "url" and "checksum" for the requested version and system. A lookup that +// produces nothing means the version is not available for that system. +type Lookup struct { + URL string `json:"url"` + Select string `json:"select"` +} + +// Checksum describes where to find the digest of an artifact. Digests may +// be sha256 or sha512, as hex, "sha256:" or an SRI string +// ("sha512-"). type Checksum struct { Type string `json:"type"` URL string `json:"url,omitempty"` Select string `json:"select,omitempty"` + + // github_release: the asset digest GitHub computes for every release + // asset. Tag defaults to "v{{.Version}}". + Repo string `json:"repo,omitempty"` + Tag string `json:"tag,omitempty"` } // Hooks are shell commands run at specific points of a package's lifecycle. @@ -227,21 +295,34 @@ func (d *Definition) Validate() error { } } - errs = append(errs, d.Versions.validate()...) - errs = append(errs, d.Install.validate("install")...) - for sys, override := range d.Install.Overrides { - if !slices.Contains(d.Platforms, sys) { - fail("install.overrides: %q is not in platforms", sys) + if len(d.Versions) == 0 { + fail("versions is required") + } + for i, src := range d.Versions { + path := "versions" + if len(d.Versions) > 1 { + path = fmt.Sprintf("versions[%d]", i) } + errs = append(errs, src.validate(path)...) + } + + errs = append(errs, d.Install.validate("install")...) + errs = append(errs, d.validateSystemOverrides("install", d.Install, d.Install.Overrides)...) + for i, override := range d.Install.VersionOverrides { + path := fmt.Sprintf("install.version_overrides[%d]", i) if override == nil { - fail("install.overrides.%s must be an object", sys) + fail("%s must be an object", path) continue } - if len(override.Overrides) > 0 { - fail("install.overrides.%s may not contain overrides", sys) + if err := validateConstraint(override.Versions); err != nil { + fail("%s.versions: %v", path, err) + } + if len(override.VersionOverrides) > 0 { + fail("%s may not contain version_overrides", path) } - merged := d.Install.merge(override) - errs = append(errs, merged.validate("install.overrides."+sys)...) + merged := d.Install.merge(&override.Install) + errs = append(errs, merged.validate(path)...) + errs = append(errs, d.validateSystemOverrides(path, merged, override.Overrides)...) } if len(errs) == 0 { @@ -250,45 +331,95 @@ func (d *Definition) Validate() error { return fmt.Errorf("invalid package definition %q:\n - %s", d.Name, strings.Join(errs, "\n - ")) } -func (v *VersionSource) validate() []string { +// validateSystemOverrides validates per-system overrides applied on top of +// base, the install they override. +func (d *Definition) validateSystemOverrides(path string, base Install, overrides map[string]*Install) []string { + var errs []string + for sys, override := range overrides { + if !slices.Contains(d.Platforms, sys) { + errs = append(errs, fmt.Sprintf("%s.overrides: %q is not in platforms", path, sys)) + } + if override == nil { + errs = append(errs, fmt.Sprintf("%s.overrides.%s must be an object", path, sys)) + continue + } + if len(override.Overrides) > 0 || len(override.VersionOverrides) > 0 { + errs = append(errs, fmt.Sprintf("%s.overrides.%s may not contain overrides", path, sys)) + } + merged := base.merge(override) + errs = append(errs, merged.validate(path+".overrides."+sys)...) + } + return errs +} + +func (v *VersionSource) validate(path string) []string { var errs []string switch v.Type { case VersionsStatic: if len(v.Versions) == 0 { - errs = append(errs, "versions.versions must list at least one version") + errs = append(errs, path+".versions must list at least one version") } case VersionsHTTPJSON: if v.URL == "" { - errs = append(errs, "versions.url is required for type http_json") + errs = append(errs, path+".url is required for type http_json") } if v.Select == "" { - errs = append(errs, "versions.select is required for type http_json") - } else if _, err := compileJQ(v.Select); err != nil { - errs = append(errs, fmt.Sprintf("versions.select: %v", err)) + errs = append(errs, path+".select is required for type http_json") + } else if err := validateJQ(v.Select); err != nil { + errs = append(errs, fmt.Sprintf("%s.select: %v", path, err)) } case VersionsGitHubReleases: - if _, repo, _ := strings.Cut(v.Repo, "/"); !githubRepoRegexp.MatchString(v.Repo) || repo == "." || repo == ".." { - errs = append(errs, "versions.repo must be of the form owner/name") + if !isGitHubRepo(v.Repo) { + errs = append(errs, path+".repo must be of the form owner/name") } case VersionsScript: if v.Run == "" { - errs = append(errs, "versions.run is required for type script") + errs = append(errs, path+".run is required for type script") } case "": - errs = append(errs, "versions.type is required") + errs = append(errs, path+".type is required") default: - errs = append(errs, fmt.Sprintf("versions.type %q is not one of %s, %s, %s, %s", - v.Type, VersionsStatic, VersionsHTTPJSON, VersionsGitHubReleases, VersionsScript)) + errs = append(errs, fmt.Sprintf("%s.type %q is not one of %s, %s, %s, %s", + path, v.Type, VersionsStatic, VersionsHTTPJSON, VersionsGitHubReleases, VersionsScript)) } return errs } +func isGitHubRepo(repo string) bool { + _, name, _ := strings.Cut(repo, "/") + return githubRepoRegexp.MatchString(repo) && name != "." && name != ".." +} + +// validateJQ compiles a jq expression. Expressions that contain template +// actions are only checked once rendered, because an action may expand to +// anything. +func validateJQ(expr string) error { + if strings.Contains(expr, "{{") { + return nil + } + _, err := compileJQ(expr) + return err +} + func (in *Install) validate(path string) []string { var errs []string switch in.Type { case InstallArchive, InstallBinary: - if in.URL == "" { - errs = append(errs, path+".url is required for type "+in.Type) + switch { + case in.URL == "" && in.Lookup == nil: + errs = append(errs, path+".url or .lookup is required for type "+in.Type) + case in.URL != "" && in.Lookup != nil: + errs = append(errs, path+".url and .lookup are mutually exclusive") + } + if in.Lookup != nil { + if in.Lookup.URL == "" || in.Lookup.Select == "" { + errs = append(errs, path+".lookup.url and .lookup.select are required") + } else if err := validateJQ(in.Lookup.Select); err != nil { + errs = append(errs, fmt.Sprintf("%s.lookup.select: %v", path, err)) + } + if in.Checksum != nil { + errs = append(errs, path+".checksum is not used with .lookup, which provides the checksum") + } } if in.StripComponents != nil && *in.StripComponents < 0 { errs = append(errs, path+".strip_components must be >= 0") @@ -319,19 +450,23 @@ func (c *Checksum) validate(path string) []string { case ChecksumHTTPJSON: if c.URL == "" || c.Select == "" { errs = append(errs, path+".url and .select are required for type http_json") - } else if _, err := compileJQ(c.Select); err != nil { + } else if err := validateJQ(c.Select); err != nil { errs = append(errs, fmt.Sprintf("%s.select: %v", path, err)) } - case ChecksumSHA256URL: + case ChecksumSHA256URL, ChecksumSHA512URL: if c.URL == "" { - errs = append(errs, path+".url is required for type sha256_url") + errs = append(errs, path+".url is required for type "+c.Type) + } + case ChecksumGitHubRelease: + if !isGitHubRepo(c.Repo) { + errs = append(errs, path+".repo must be of the form owner/name") } case ChecksumNone: case "": errs = append(errs, path+".type is required") default: - errs = append(errs, fmt.Sprintf("%s.type %q is not one of %s, %s, %s", - path, c.Type, ChecksumHTTPJSON, ChecksumSHA256URL, ChecksumNone)) + errs = append(errs, fmt.Sprintf("%s.type %q is not one of %s, %s, %s, %s, %s", + path, c.Type, ChecksumHTTPJSON, ChecksumSHA256URL, ChecksumSHA512URL, ChecksumGitHubRelease, ChecksumNone)) } return errs } @@ -350,27 +485,51 @@ func (d *Definition) BinDirs() []string { return d.Bin } -// InstallFor returns the install strategy for system with any override for -// that system applied. -func (d *Definition) InstallFor(system string) Install { - override := d.Install.Overrides[system] - if override == nil { - merged := d.Install - merged.Overrides = nil - return merged +// InstallFor returns the install strategy for version on system. Overrides +// apply in order: the first version override matching version, the +// definition's override for system, and the version override's own +// override for system. +func (d *Definition) InstallFor(system, version string) Install { + merged := d.Install.merge(&Install{}) + var versionOverride *VersionOverride + for _, vo := range d.Install.VersionOverrides { + if vo != nil && MatchesConstraint(version, vo.Versions) { + versionOverride = vo + break + } } - return d.Install.merge(override) + if versionOverride != nil { + merged = merged.merge(&versionOverride.Install) + } + if override := d.Install.Overrides[system]; override != nil { + merged = merged.merge(override) + } + if versionOverride != nil { + if override := versionOverride.Overrides[system]; override != nil { + merged = merged.merge(override) + } + } + return merged } // merge returns a copy of in with every non-zero field of override applied. +// The result has no overrides of its own. url and lookup are alternatives, +// so an override that sets one clears the other. func (in *Install) merge(override *Install) Install { merged := *in merged.Overrides = nil + merged.VersionOverrides = nil if override.Type != "" { merged.Type = override.Type } if override.URL != "" { merged.URL = override.URL + merged.Lookup = nil + } + if override.Lookup != nil { + merged.Lookup = override.Lookup + merged.URL = "" + merged.Checksum = nil } if override.Checksum != nil { merged.Checksum = override.Checksum @@ -409,19 +568,20 @@ func (in *Install) ChecksumType() string { } // InstallFingerprint identifies everything about the definition that -// affects what an install of a version on system contains: the install +// affects what an install of version on system contains: the install // strategy, the bin layout, hooks and platform maps. Activation fields // (env, shell, create_files) are excluded because they are applied at // shell time. A changed fingerprint makes Install redo an existing install. -func (d *Definition) InstallFingerprint(system string) string { - install := d.InstallFor(system) +func (d *Definition) InstallFingerprint(system, version string) string { + install := d.InstallFor(system, version) payload, err := json.Marshal(struct { - Install Install `json:"install"` - Bin []string `json:"bin"` - Hooks Hooks `json:"hooks"` - OSMap map[string]string `json:"os_map"` - ArchMap map[string]string `json:"arch_map"` - }{install, d.Bin, d.Hooks, d.OSMap, d.ArchMap}) + Install Install `json:"install"` + Bin []string `json:"bin"` + Binaries []string `json:"binaries"` + Hooks Hooks `json:"hooks"` + OSMap map[string]string `json:"os_map"` + ArchMap map[string]string `json:"arch_map"` + }{install, d.Bin, d.Binaries, d.Hooks, d.OSMap, d.ArchMap}) if err != nil { return "" } @@ -433,7 +593,12 @@ func (d *Definition) InstallFingerprint(system string) string { // should be told about: script-type sources or strategies, or post-install // hooks. func (d *Definition) UsesScripts() bool { - if d.Versions.Type == VersionsScript || d.Install.Type == InstallScript { + for _, src := range d.Versions { + if src.Type == VersionsScript { + return true + } + } + if d.Install.Type == InstallScript { return true } for _, o := range d.Install.Overrides { @@ -441,5 +606,18 @@ func (d *Definition) UsesScripts() bool { return true } } + for _, vo := range d.Install.VersionOverrides { + if vo == nil { + continue + } + if vo.Type == InstallScript { + return true + } + for _, o := range vo.Overrides { + if o != nil && o.Type == InstallScript { + return true + } + } + } return len(d.Hooks.PostInstall) > 0 } diff --git a/internal/pkgdef/definition_test.go b/internal/pkgdef/definition_test.go index 41bc218db05..0fc39d72af1 100644 --- a/internal/pkgdef/definition_test.go +++ b/internal/pkgdef/definition_test.go @@ -4,6 +4,7 @@ package pkgdef import ( + "encoding/json" "os" "path/filepath" "strings" @@ -28,7 +29,7 @@ func TestParseGoExample(t *testing.T) { def, err := Parse(content) require.NoError(t, err) assert.Equal(t, "go", def.Name) - assert.Equal(t, VersionsHTTPJSON, def.Versions.Type) + assert.Equal(t, VersionsHTTPJSON, def.Versions[0].Type) assert.Equal(t, InstallArchive, def.Install.Type) assert.Equal(t, ChecksumHTTPJSON, def.Install.ChecksumType()) assert.Equal(t, []string{"bin"}, def.BinDirs()) @@ -106,7 +107,7 @@ func TestParseErrors(t *testing.T) { { name: "install url missing", json: `{"schema": 1, "name": "x", "platforms": ["x86_64-linux"], "versions": {"type": "static", "versions": ["1"]}, "install": {"type": "binary"}}`, - want: "install.url is required for type binary", + want: "install.url or .lookup is required for type binary", }, { name: "unknown install type", @@ -165,13 +166,13 @@ func TestInstallFor(t *testing.T) { }`)) require.NoError(t, err) - linux := def.InstallFor("x86_64-linux") + linux := def.InstallFor("x86_64-linux", "1.0.0") assert.Equal(t, InstallArchive, linux.Type) assert.Equal(t, "default", linux.URL) assert.Equal(t, ChecksumSHA256URL, linux.ChecksumType()) assert.Nil(t, linux.Overrides) - mac := def.InstallFor("aarch64-darwin") + mac := def.InstallFor("aarch64-darwin", "1.0.0") assert.Equal(t, InstallBinary, mac.Type) assert.Equal(t, "mac", mac.URL) assert.Equal(t, ChecksumNone, mac.ChecksumType()) @@ -190,3 +191,167 @@ func TestUsesScripts(t *testing.T) { def.Hooks.PostInstall = []string{"true"} assert.True(t, def.UsesScripts()) } + +func TestVersionSourcesJSON(t *testing.T) { + def, err := Parse([]byte(minimalDef)) + require.NoError(t, err) + require.Len(t, def.Versions, 1) + + // A list of sources is their union. + def, err = Parse([]byte(`{ + "schema": 1, "name": "x", "platforms": ["x86_64-linux"], + "versions": [ + {"type": "static", "versions": ["2.0.0"]}, + {"type": "static", "versions": ["1.0.0"]} + ], + "install": {"type": "binary", "url": "https://example.com/x"} + }`)) + require.NoError(t, err) + require.Len(t, def.Versions, 2) + + // A single source round-trips as an object, a list as an array. + single, err := json.Marshal(VersionSources{{Type: VersionsStatic, Versions: []string{"1"}}}) + require.NoError(t, err) + assert.JSONEq(t, `{"type": "static", "versions": ["1"]}`, string(single)) + list, err := json.Marshal(def.Versions) + require.NoError(t, err) + assert.JSONEq(t, `[{"type": "static", "versions": ["2.0.0"]}, {"type": "static", "versions": ["1.0.0"]}]`, string(list)) + + // Each source in a list is validated, with its index in the error. + _, err = Parse([]byte(`{ + "schema": 1, "name": "x", "platforms": ["x86_64-linux"], + "versions": [{"type": "static", "versions": ["1"]}, {"type": "github_releases"}], + "install": {"type": "binary", "url": "https://example.com/x"} + }`)) + require.ErrorContains(t, err, "versions[1].repo must be of the form owner/name") + + _, err = Parse([]byte(`{ + "schema": 1, "name": "x", "platforms": ["x86_64-linux"], + "versions": [], + "install": {"type": "binary", "url": "https://example.com/x"} + }`)) + require.ErrorContains(t, err, "versions is required") +} + +func TestInstallForVersionOverrides(t *testing.T) { + def, err := Parse([]byte(`{ + "schema": 1, "name": "x", "platforms": ["x86_64-linux", "aarch64-darwin"], + "versions": {"type": "static", "versions": ["1"]}, + "install": { + "type": "archive", "url": "new", "strip_components": 1, + "checksum": {"type": "github_release", "repo": "acme/x"}, + "overrides": {"aarch64-darwin": {"url": "new-mac"}}, + "version_overrides": [ + {"versions": "<2", "type": "binary", "url": "old", "checksum": {"type": "none"}, + "overrides": {"aarch64-darwin": {"url": "old-mac"}}}, + {"versions": "<3", "lookup": {"url": "https://example.com/index.json", "select": ".x"}} + ] + } + }`)) + require.NoError(t, err) + + tests := []struct { + system, version string + wantType string + wantURL string + wantLookup bool + wantChecksum string + }{ + {"x86_64-linux", "3.0.0", InstallArchive, "new", false, ChecksumGitHubRelease}, + {"aarch64-darwin", "3.0.0", InstallArchive, "new-mac", false, ChecksumGitHubRelease}, + // The first matching version override wins. + {"x86_64-linux", "1.9.0", InstallBinary, "old", false, ChecksumNone}, + // The version override's own system override applies last. + {"aarch64-darwin", "1.9.0", InstallBinary, "old-mac", false, ChecksumNone}, + // A lookup replaces url and checksum. + {"x86_64-linux", "2.5.0", InstallArchive, "", true, ChecksumNone}, + // The definition's system override sets url, which replaces the lookup. + {"aarch64-darwin", "2.5.0", InstallArchive, "new-mac", false, ChecksumNone}, + } + for _, tt := range tests { + t.Run(tt.system+"@"+tt.version, func(t *testing.T) { + got := def.InstallFor(tt.system, tt.version) + assert.Equal(t, tt.wantType, got.Type) + assert.Equal(t, tt.wantURL, got.URL) + assert.Equal(t, tt.wantLookup, got.Lookup != nil) + assert.Equal(t, tt.wantChecksum, got.ChecksumType()) + assert.Nil(t, got.Overrides) + assert.Nil(t, got.VersionOverrides) + }) + } + + assert.NotEqual(t, def.InstallFingerprint("x86_64-linux", "1.9.0"), def.InstallFingerprint("x86_64-linux", "3.0.0"), + "versions installed by different recipes have different fingerprints") + assert.Equal(t, def.InstallFingerprint("x86_64-linux", "3.0.0"), def.InstallFingerprint("x86_64-linux", "3.1.0")) +} + +func TestParseInstallErrors(t *testing.T) { + tests := []struct { + name string + install string + wantErr string + }{ + { + name: "url and lookup", + install: `{"type": "archive", "url": "u", "lookup": {"url": "i", "select": ".x"}}`, + wantErr: "install.url and .lookup are mutually exclusive", + }, + { + name: "lookup without select", + install: `{"type": "archive", "lookup": {"url": "i"}}`, + wantErr: "install.lookup.url and .lookup.select are required", + }, + { + name: "lookup with checksum", + install: `{"type": "archive", "lookup": {"url": "i", "select": ".x"}, "checksum": {"type": "none"}}`, + wantErr: "install.checksum is not used with .lookup", + }, + { + name: "lookup select does not compile", + install: `{"type": "archive", "lookup": {"url": "i", "select": ".[[["}}`, + wantErr: "install.lookup.select: invalid jq expression", + }, + { + name: "github_release checksum without repo", + install: `{"type": "archive", "url": "u", "checksum": {"type": "github_release"}}`, + wantErr: "install.checksum.repo must be of the form owner/name", + }, + { + name: "bad version constraint", + install: `{"type": "archive", "url": "u", "version_overrides": [{"versions": "<<1", "url": "v"}]}`, + wantErr: "install.version_overrides[0].versions: invalid term", + }, + { + name: "nested version overrides", + install: `{"type": "archive", "url": "u", "version_overrides": [{"versions": "<1", "version_overrides": [{"versions": "<1"}]}]}`, + wantErr: "install.version_overrides[0] may not contain version_overrides", + }, + { + name: "version override is validated merged", + install: `{"type": "archive", "url": "u", "version_overrides": [{"versions": "<1", "type": "script"}]}`, + wantErr: "install.version_overrides[0].run is required for type script", + }, + { + name: "version override system override for unknown platform", + install: `{"type": "archive", "url": "u", "version_overrides": [{"versions": "<1", "overrides": {"aarch64-darwin": {"url": "m"}}}]}`, + wantErr: `install.version_overrides[0].overrides: "aarch64-darwin" is not in platforms`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := Parse([]byte(`{ + "schema": 1, "name": "x", "platforms": ["x86_64-linux"], + "versions": {"type": "static", "versions": ["1"]}, + "install": ` + tt.install + ` + }`)) + require.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func TestUsesScriptsInVersionOverrides(t *testing.T) { + def, err := Parse([]byte(minimalDef)) + require.NoError(t, err) + def.Install.VersionOverrides = []*VersionOverride{{Versions: "<1", Install: Install{Type: InstallScript, Run: "true"}}} + assert.True(t, def.UsesScripts()) +} diff --git a/internal/pkgdef/install.go b/internal/pkgdef/install.go index d648335927d..5459e40e046 100644 --- a/internal/pkgdef/install.go +++ b/internal/pkgdef/install.go @@ -7,6 +7,7 @@ import ( "bytes" "context" "crypto/sha256" + "crypto/sha512" "encoding/hex" "encoding/json" "errors" @@ -39,6 +40,7 @@ type marker struct { System string `json:"system"` URL string `json:"url,omitempty"` Sha256 string `json:"sha256,omitempty"` + Sha512 string `json:"sha512,omitempty"` // InstallHash is Definition.InstallFingerprint at install time. When // the definition's install recipe changes, the install is redone. @@ -70,8 +72,9 @@ type Installed struct { // with a file lock. // // artifact.URL may be empty, in which case the URL is rendered from the -// definition. Relative file:// URLs are resolved against the definition's -// directory. When artifact.Sha256 is set the download must match it. +// definition (or looked up in its index). Relative file:// URLs are +// resolved against the definition's directory. When the artifact has a +// digest the download must match it. func (c *Client) Install(ctx context.Context, def *Definition, version, system string, artifact Artifact) (*Installed, error) { if !def.SupportsSystem(system) { return nil, fmt.Errorf("%s@%s: %w (%s; supported: %s)", @@ -79,7 +82,7 @@ func (c *Client) Install(ctx context.Context, def *Definition, version, system s } dir := c.InstallDir(def.Name, version, system) - fingerprint := def.InstallFingerprint(system) + fingerprint := def.InstallFingerprint(system, version) // Fast path: no lock needed to read a finished install. if m, ok := readMarker(dir); ok && m.InstallHash == fingerprint { @@ -100,15 +103,20 @@ func (c *Client) Install(ctx context.Context, def *Definition, version, system s } func (c *Client) installedFromMarker(def *Definition, version, dir string, m marker, artifact Artifact) (*Installed, error) { - if artifact.Sha256 != "" && m.Sha256 != "" && !strings.EqualFold(artifact.Sha256, m.Sha256) { - return nil, fmt.Errorf("%s@%s: %w: %s was installed from an artifact with sha256 %s but the lockfile expects %s; remove the directory to reinstall", - def.Name, version, ErrChecksumMismatch, dir, m.Sha256, artifact.Sha256) + for _, d := range []struct{ algo, want, got string }{ + {algoSHA256, artifact.Sha256, m.Sha256}, + {algoSHA512, artifact.Sha512, m.Sha512}, + } { + if d.want != "" && d.got != "" && !strings.EqualFold(d.want, d.got) { + return nil, fmt.Errorf("%s@%s: %w: %s was installed from an artifact with %s %s but the lockfile expects %s; remove the directory to reinstall", + def.Name, version, ErrChecksumMismatch, dir, d.algo, d.got, d.want) + } } return c.installed(def, dir, m.URL, m.Sha256), nil } func (c *Client) install(ctx context.Context, def *Definition, version, system, dir, fingerprint string, artifact Artifact) (*Installed, error) { - install := def.InstallFor(system) + install := def.InstallFor(system, version) vars, err := def.TemplateVars(system, version) if err != nil { return nil, err @@ -124,7 +132,18 @@ func (c *Client) install(ctx context.Context, def *Definition, version, system, return nil, fmt.Errorf("%s: install.type %q is not supported", def.Name, install.Type) } - if artifact.URL == "" { + switch { + case artifact.URL != "": + case install.Lookup != nil: + found, err := c.lookupArtifact(ctx, def, install.Lookup, vars) + if err != nil { + return nil, fmt.Errorf("%s@%s: %w", def.Name, version, err) + } + artifact.URL = found.URL + if !artifact.HasDigest() { + artifact.Sha256, artifact.Sha512 = found.Sha256, found.Sha512 + } + default: artifact.URL, err = Render(install.URL, vars) if err != nil { return nil, err @@ -137,14 +156,13 @@ func (c *Client) install(ctx context.Context, def *Definition, version, system, // one filesystem. downloadDir := filepath.Join(c.CacheDir, ".downloads", def.Name, version, system) archivePath := filepath.Join(downloadDir, vars.Filename) - sha, err := c.download(ctx, fetchURL, archivePath) + sums, err := c.download(ctx, fetchURL, archivePath) if err != nil { return nil, fmt.Errorf("%s@%s: %w", def.Name, version, err) } - if artifact.Sha256 != "" && !strings.EqualFold(sha, artifact.Sha256) { + if err := sums.verify(artifact); err != nil { _ = os.Remove(archivePath) - return nil, fmt.Errorf("%s@%s: %w: %s has sha256 %s, lockfile expects %s", - def.Name, version, ErrChecksumMismatch, fetchURL, sha, artifact.Sha256) + return nil, fmt.Errorf("%s@%s: %w: %s %w", def.Name, version, ErrChecksumMismatch, fetchURL, err) } // Unpack into a staging directory and rename it into place so that a @@ -180,6 +198,9 @@ func (c *Client) install(ctx context.Context, def *Definition, version, system, return nil, fmt.Errorf("%s@%s: unpacking %s: %w", def.Name, version, vars.Filename, err) } _ = os.Remove(archivePath) + if err := makeBinariesExecutable(staging, def.BinDirs(), def.Binaries); err != nil { + return nil, err + } if err := os.Rename(staging, dir); err != nil { return nil, err @@ -190,19 +211,64 @@ func (c *Client) install(ctx context.Context, def *Definition, version, system, // so the next install retries. return nil, fmt.Errorf("%s@%s: %w", def.Name, version, err) } + if missing := missingBinaries(dir, def.BinDirs(), def.Binaries); len(missing) > 0 { + return nil, fmt.Errorf("%s@%s: installed, but %s not found in %s (bin: %s)", + def.Name, version, strings.Join(missing, ", "), dir, strings.Join(def.BinDirs(), ", ")) + } - m := marker{Name: def.Name, Version: version, System: system, URL: artifact.URL, Sha256: sha, InstallHash: fingerprint} + m := marker{ + Name: def.Name, Version: version, System: system, URL: artifact.URL, + Sha256: sums.sha256, Sha512: sums.sha512, InstallHash: fingerprint, + } if err := writeMarker(dir, m); err != nil { return nil, err } - return c.installed(def, dir, artifact.URL, sha), nil + return c.installed(def, dir, artifact.URL, sums.sha256), nil +} + +// makeBinariesExecutable sets the executable bits on every file named in +// binaries that exists in one of binDirs. Archives built by tools that do +// not preserve modes (npm tarballs, for example) ship scripts without them. +func makeBinariesExecutable(root string, binDirs, binaries []string) error { + for _, dir := range binDirs { + for _, name := range binaries { + p := filepath.Join(root, dir, name) + info, err := os.Stat(p) + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0o111 { + continue + } + if err := os.Chmod(p, info.Mode().Perm()|0o111); err != nil { + return err + } + } + } + return nil +} + +// missingBinaries returns the names in binaries that are not present in +// any of binDirs. +func missingBinaries(root string, binDirs, binaries []string) []string { + var missing []string + for _, name := range binaries { + found := false + for _, dir := range binDirs { + if _, err := os.Stat(filepath.Join(root, dir, name)); err == nil { + found = true + break + } + } + if !found { + missing = append(missing, name) + } + } + return missing } // IsInstalled reports whether version of def is fully installed on system // from the definition's current install recipe. func (c *Client) IsInstalled(def *Definition, version, system string) bool { m, ok := readMarker(c.InstallDir(def.Name, version, system)) - return ok && m.InstallHash == def.InstallFingerprint(system) + return ok && m.InstallHash == def.InstallFingerprint(system, version) } func (c *Client) installed(def *Definition, dir, url, sha string) *Installed { @@ -233,53 +299,71 @@ func lockDir(dir string) (func(), error) { }, nil } +// fileDigests are the digests of a downloaded file in every algorithm a +// lockfile may record. +type fileDigests struct { + sha256 string + sha512 string +} + +// verify checks the digests against those recorded for the artifact. +func (f fileDigests) verify(artifact Artifact) error { + if artifact.Sha256 != "" && !strings.EqualFold(f.sha256, artifact.Sha256) { + return fmt.Errorf("has sha256 %s, lockfile expects %s", f.sha256, artifact.Sha256) + } + if artifact.Sha512 != "" && !strings.EqualFold(f.sha512, artifact.Sha512) { + return fmt.Errorf("has sha512 %s, lockfile expects %s", f.sha512, artifact.Sha512) + } + return nil +} + // download fetches rawURL to dest (unless dest already exists) and returns -// the hex sha256 of the file. -func (c *Client) download(ctx context.Context, rawURL, dest string) (string, error) { - if sha, err := fileSHA256(dest); err == nil { - return sha, nil +// the digests of the file. +func (c *Client) download(ctx context.Context, rawURL, dest string) (fileDigests, error) { + if sums, err := digestFile(dest); err == nil { + return sums, nil } if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { - return "", err + return fileDigests{}, err } body, err := c.open(ctx, rawURL, "") if err != nil { - return "", err + return fileDigests{}, err } defer body.Close() tmp, err := os.CreateTemp(filepath.Dir(dest), filepath.Base(dest)+".part-") if err != nil { - return "", err + return fileDigests{}, err } defer os.Remove(tmp.Name()) - hasher := sha256.New() - if _, err := io.Copy(io.MultiWriter(tmp, hasher), body); err != nil { + h256, h512 := sha256.New(), sha512.New() + if _, err := io.Copy(io.MultiWriter(tmp, h256, h512), body); err != nil { tmp.Close() - return "", fmt.Errorf("downloading %s: %w", rawURL, err) + return fileDigests{}, fmt.Errorf("downloading %s: %w", rawURL, err) } if err := tmp.Close(); err != nil { - return "", err + return fileDigests{}, err } if err := os.Rename(tmp.Name(), dest); err != nil { - return "", err + return fileDigests{}, err } - return hex.EncodeToString(hasher.Sum(nil)), nil + return fileDigests{hex.EncodeToString(h256.Sum(nil)), hex.EncodeToString(h512.Sum(nil))}, nil } -func fileSHA256(path string) (string, error) { +func digestFile(path string) (fileDigests, error) { f, err := os.Open(path) if err != nil { - return "", err + return fileDigests{}, err } defer f.Close() - hasher := sha256.New() - if _, err := io.Copy(hasher, f); err != nil { - return "", err + h256, h512 := sha256.New(), sha512.New() + if _, err := io.Copy(io.MultiWriter(h256, h512), f); err != nil { + return fileDigests{}, err } - return hex.EncodeToString(hasher.Sum(nil)), nil + return fileDigests{hex.EncodeToString(h256.Sum(nil)), hex.EncodeToString(h512.Sum(nil))}, nil } func placeBinary(src, dest string) error { @@ -316,7 +400,9 @@ func (c *Client) runPostInstall(ctx context.Context, def *Definition, dir string tail := &tailBuffer{limit: hookOutputLimit} output := io.MultiWriter(c.Stderr, tail) - cmd := exec.CommandContext(ctx, "sh", "-c", script) + // /bin/sh rather than a PATH lookup: the hook must run even when + // devbox itself was started with a minimal PATH. + cmd := exec.CommandContext(ctx, "/bin/sh", "-c", script) cmd.Dir = dir cmd.Stdout = output cmd.Stderr = output @@ -324,6 +410,9 @@ func (c *Client) runPostInstall(ctx context.Context, def *Definition, dir string "DEVBOX_PKG_INSTALL_DIR="+dir, "DEVBOX_PKG_VERSION="+vars.Version, "DEVBOX_PKG_SYSTEM="+vars.System, + // Hooks may use basic utilities (ln, chmod, ...) even when + // devbox was started with a minimal PATH. + "PATH="+hookPath(os.Getenv("PATH")), ) if err := cmd.Run(); err != nil { if out := strings.TrimSpace(tail.String()); out != "" { @@ -335,6 +424,14 @@ func (c *Client) runPostInstall(ctx context.Context, def *Definition, dir string return nil } +// hookPath returns path with the standard system directories appended. +func hookPath(path string) string { + if path == "" { + return "/usr/bin:/bin" + } + return path + ":/usr/bin:/bin" +} + // tailBuffer keeps the last limit bytes written to it. type tailBuffer struct { buf bytes.Buffer diff --git a/internal/pkgdef/lock.go b/internal/pkgdef/lock.go index aacc09de926..14c6b57a8e0 100644 --- a/internal/pkgdef/lock.go +++ b/internal/pkgdef/lock.go @@ -5,28 +5,42 @@ package pkgdef import ( "context" + "errors" "fmt" "slices" + "strings" ) +// ErrNoArtifact is returned when the vendor publishes no artifact of a +// version for a system, for example an old release that predates a +// platform. +var ErrNoArtifact = errors.New("no artifact for this system") + // Artifact is what gets recorded per system in devbox.lock: the exact URL -// to download and its sha256. Sha256 is empty when the definition has no -// checksum source; it is filled in by the first install (trust on first -// use). +// to download and its digest. Vendors publish sha256 or sha512 digests, so +// at most one of them is usually set. Both are empty when the definition +// has no checksum source; the first install fills in Sha256 (trust on +// first use). type Artifact struct { URL string `json:"url,omitempty"` Sha256 string `json:"sha256,omitempty"` + Sha512 string `json:"sha512,omitempty"` +} + +// HasDigest reports whether the artifact has a digest to verify against. +func (a Artifact) HasDigest() bool { + return a.Sha256 != "" || a.Sha512 != "" } // Resolved is the result of resolving a version constraint: the concrete -// version and the artifact for every system the definition supports. +// version and the artifact for every supported system the vendor publishes +// it for. type Resolved struct { Version string Artifacts map[string]Artifact } -// Resolve resolves requested to a concrete version and locks the artifacts -// for every supported system. +// Resolve resolves requested to a concrete version and locks its artifacts. func (c *Client) Resolve(ctx context.Context, def *Definition, requested string) (*Resolved, error) { version, err := c.ResolveVersion(ctx, def, requested) if err != nil { @@ -40,23 +54,34 @@ func (c *Client) Resolve(ctx context.Context, def *Definition, requested string) } // Lock renders the artifact URL and fetches the checksum of version for -// every system in the definition's platforms. +// every system in the definition's platforms. Systems the vendor publishes +// no artifact for are left out; it is an error only when no system has +// one. func (c *Client) Lock(ctx context.Context, def *Definition, version string) (map[string]Artifact, error) { artifacts := make(map[string]Artifact, len(def.Platforms)) systems := slices.Clone(def.Platforms) slices.Sort(systems) + var missing []string for _, system := range systems { artifact, err := c.lockSystem(ctx, def, version, system) + if errors.Is(err, ErrNoArtifact) { + missing = append(missing, fmt.Sprintf("%s: %v", system, err)) + continue + } if err != nil { return nil, fmt.Errorf("%s@%s (%s): %w", def.Name, version, system, err) } artifacts[system] = artifact } + if len(artifacts) == 0 { + return nil, fmt.Errorf("%s@%s: %w on any supported system:\n %s", + def.Name, version, ErrNoArtifact, strings.Join(missing, "\n ")) + } return artifacts, nil } func (c *Client) lockSystem(ctx context.Context, def *Definition, version, system string) (Artifact, error) { - install := def.InstallFor(system) + install := def.InstallFor(system, version) switch install.Type { case InstallArchive, InstallBinary: case InstallManager, InstallScript: @@ -69,6 +94,10 @@ func (c *Client) lockSystem(ctx context.Context, def *Definition, version, syste if err != nil { return Artifact{}, err } + if install.Lookup != nil { + return c.lookupArtifact(ctx, def, install.Lookup, vars) + } + // The URL is recorded as rendered (a relative file:// URL stays // relative) so that devbox.lock is portable; it is resolved against the // definition's directory when fetching. @@ -78,9 +107,55 @@ func (c *Client) lockSystem(ctx context.Context, def *Definition, version, syste } vars = vars.withURL(rawURL) - sha, err := c.fetchChecksum(ctx, def, install.Checksum, vars) + d, err := c.fetchChecksum(ctx, def, install.Checksum, vars) + if err != nil { + return Artifact{}, err + } + artifact := Artifact{URL: rawURL} + d.apply(&artifact) + return artifact, nil +} + +// lookupArtifact evaluates an install lookup: it renders the index URL and +// jq expression for version and system and reads "url" and "checksum" from +// the first object the expression produces. An index that does not exist +// or has no entry means the vendor has no artifact for this system. +func (c *Client) lookupArtifact(ctx context.Context, def *Definition, lookup *Lookup, vars Vars) (Artifact, error) { + rawURL, err := Render(lookup.URL, vars) if err != nil { return Artifact{}, err } - return Artifact{URL: rawURL, Sha256: sha}, nil + rawURL = def.ResolveURL(rawURL) + expr, err := Render(lookup.Select, vars) + if err != nil { + return Artifact{}, err + } + results, err := c.queryJSON(ctx, rawURL, expr) + if errors.Is(err, ErrNotFound) { + return Artifact{}, fmt.Errorf("lookup: %w: %w", ErrNoArtifact, err) + } + if err != nil { + return Artifact{}, fmt.Errorf("lookup: %w", err) + } + if len(results) == 0 { + return Artifact{}, fmt.Errorf("lookup: %w: %s has no entry for %s on %s", ErrNoArtifact, rawURL, vars.Version, vars.System) + } + obj, ok := results[0].(map[string]any) + if !ok { + return Artifact{}, fmt.Errorf("lookup: %s: jq expression %q must produce objects with \"url\" and \"checksum\", got %T", + rawURL, expr, results[0]) + } + artifactURL, _ := obj["url"].(string) + if artifactURL == "" { + return Artifact{}, fmt.Errorf("lookup: %s: jq expression %q produced an object without \"url\"", rawURL, expr) + } + artifact := Artifact{URL: artifactURL} + if checksum, _ := obj["checksum"].(string); checksum != "" { + d, err := parseDigest(checksum, rawURL, "") + if err != nil { + return Artifact{}, err + } + d.apply(&artifact) + } + return artifact, nil } diff --git a/internal/pkgdef/template.go b/internal/pkgdef/template.go index 06c72217cca..225c9907240 100644 --- a/internal/pkgdef/template.go +++ b/internal/pkgdef/template.go @@ -24,6 +24,10 @@ type Vars struct { InstallDir string CacheDir string + + // Page is the 0-based page number when listing versions from a + // paginated http_json source. + Page int } var defaultOSMap = map[string]string{ @@ -37,13 +41,15 @@ var defaultArchMap = map[string]string{ } // templateFuncs follow the sprig argument order (the string comes last) so -// that pipelines read naturally: {{ .Version | trimPrefix "v" }}. +// that pipelines read naturally: {{ .Version | trimPrefix "v" }}. The +// text/template builtins (eq, index, urlquery, ...) are available too. var templateFuncs = template.FuncMap{ "trimPrefix": func(prefix, s string) string { return strings.TrimPrefix(s, prefix) }, "trimSuffix": func(suffix, s string) string { return strings.TrimSuffix(s, suffix) }, "replace": func(old, replacement, s string) string { return strings.ReplaceAll(s, old, replacement) }, "lower": strings.ToLower, "upper": strings.ToUpper, + "splitList": func(sep, s string) []string { return strings.Split(s, sep) }, } // TemplateVars returns the template variables for installing version on diff --git a/internal/pkgdef/version.go b/internal/pkgdef/version.go index 92fb21b5dcc..adbb407e829 100644 --- a/internal/pkgdef/version.go +++ b/internal/pkgdef/version.go @@ -26,10 +26,11 @@ var ErrVersionNotFound = errors.New("version not found") // - "latest" picks the highest non-pre-release version. // - An exact match wins when present. // - Otherwise requested is a prefix: "1.22" matches the highest -// "1.22.x" that is not a pre-release. +// "1.22.x" that is not a pre-release, and "21.0.5" matches +// "21.0.5+11" (build metadata). // -// A version is a pre-release when it contains a letter or a hyphen, for -// example "1.23.0-rc1" or "1.21rc2". +// A version is a pre-release when the part before any "+build" suffix +// contains a letter or a hyphen, for example "1.23.0-rc1" or "1.21rc2". func MatchVersion(available []string, requested string) (string, error) { if requested == "" { requested = Latest @@ -43,7 +44,7 @@ func MatchVersion(available []string, requested string) (string, error) { if v == "" || IsPrerelease(v) { continue } - if requested != Latest && !strings.HasPrefix(v, requested+".") { + if requested != Latest && !hasVersionPrefix(v, requested) { continue } if best == "" || CompareVersions(v, best) > 0 { @@ -57,12 +58,20 @@ func MatchVersion(available []string, requested string) (string, error) { return best, nil } -// IsPrerelease reports whether v looks like a pre-release version. +// hasVersionPrefix reports whether v is a more specific version of prefix: +// "1.22.3" and "1.22+4" have the prefix "1.22", "1.2.3" does not have "1.22". +func hasVersionPrefix(v, prefix string) bool { + return strings.HasPrefix(v, prefix+".") || strings.HasPrefix(v, prefix+"+") +} + +// IsPrerelease reports whether v looks like a pre-release version. Build +// metadata after "+" does not count. func IsPrerelease(v string) bool { - if strings.Contains(v, "-") { + core, _, _ := strings.Cut(v, "+") + if strings.Contains(core, "-") { return true } - for _, r := range v { + for _, r := range core { if unicode.IsLetter(r) { return true } @@ -73,7 +82,24 @@ func IsPrerelease(v string) bool { // CompareVersions orders two version strings. Dot-separated segments are // compared numerically when both are numbers and lexically otherwise. A // version that is a prefix of another sorts first ("1.22" < "1.22.1"). +// Build metadata after "+" breaks ties between equal versions, so a +// vendor's respin ("21.0.12+101") sorts after the original ("21.0.12+8"). func CompareVersions(a, b string) int { + aCore, aBuild, _ := strings.Cut(a, "+") + bCore, bBuild, _ := strings.Cut(b, "+") + if c := compareDotted(aCore, bCore); c != 0 { + return c + } + return compareDotted(aBuild, bBuild) +} + +func compareDotted(a, b string) int { + if a == b { + return 0 + } + if a == "" || b == "" { + return len(a) - len(b) + } as := strings.Split(a, ".") bs := strings.Split(b, ".") for i := range min(len(as), len(bs)) { @@ -84,6 +110,78 @@ func CompareVersions(a, b string) int { return len(as) - len(bs) } +// MatchesConstraint reports whether version satisfies constraint, a list +// of comparisons separated by spaces or commas that must all hold, such as +// "<11" or ">=2, <4". Operators are <, <=, >, >=, = and !=. A bare version +// matches itself and every more specific version ("1.22" matches +// "1.22.3"). Comparison uses CompareVersions, so "<11" matches "10.9.9" +// but not "11.0.0". An invalid constraint matches nothing. +func MatchesConstraint(version, constraint string) bool { + terms, err := parseConstraint(constraint) + if err != nil { + return false + } + for _, t := range terms { + if !t.matches(version) { + return false + } + } + return true +} + +func validateConstraint(constraint string) error { + _, err := parseConstraint(constraint) + return err +} + +type constraintTerm struct { + op string + version string +} + +func (t constraintTerm) matches(version string) bool { + cmp := CompareVersions(version, t.version) + switch t.op { + case "<": + return cmp < 0 + case "<=": + return cmp <= 0 + case ">": + return cmp > 0 + case ">=": + return cmp >= 0 + case "!=": + return cmp != 0 + case "=": + return cmp == 0 + default: // bare version + return version == t.version || hasVersionPrefix(version, t.version) + } +} + +func parseConstraint(constraint string) ([]constraintTerm, error) { + fields := strings.FieldsFunc(constraint, func(r rune) bool { return r == ' ' || r == ',' }) + if len(fields) == 0 { + return nil, errors.New("constraint is empty") + } + terms := make([]constraintTerm, 0, len(fields)) + for _, field := range fields { + term := constraintTerm{} + for _, op := range []string{"<=", ">=", "!=", "<", ">", "="} { + if rest, ok := strings.CutPrefix(field, op); ok { + term.op, field = op, rest + break + } + } + if field == "" || strings.ContainsAny(field, "<>=!") { + return nil, fmt.Errorf("invalid term in version constraint %q", constraint) + } + term.version = field + terms = append(terms, term) + } + return terms, nil +} + func compareSegment(a, b string) int { an, aerr := strconv.Atoi(a) bn, berr := strconv.Atoi(b) diff --git a/internal/pkgdef/version_test.go b/internal/pkgdef/version_test.go index c5a56fa7545..7adb4eaf8dd 100644 --- a/internal/pkgdef/version_test.go +++ b/internal/pkgdef/version_test.go @@ -79,3 +79,65 @@ func TestSummarize(t *testing.T) { assert.Equal(t, "2.0, 1.0", summarize([]string{"1.0", "2.0"})) assert.Equal(t, "7, 6, 5, 4, 3, and 2 more", summarize([]string{"1", "2", "3", "4", "5", "6", "7"})) } + +func TestBuildMetadata(t *testing.T) { + // OpenJDK-style versions carry a build number after "+". A respin + // ("+101") sorts after the original build of the same version. + available := []string{"21.0.12+8", "21.0.12+101", "21.0.9+10", "17.0.21+9", "21.0.13+9.0.LTS"} + + got, err := MatchVersion(available, "21") + require.NoError(t, err) + assert.Equal(t, "21.0.13+9.0.LTS", got, "build metadata with letters is not a pre-release") + + got, err = MatchVersion(available, "21.0.12") + require.NoError(t, err) + assert.Equal(t, "21.0.12+101", got, "a version matches its builds") + + got, err = MatchVersion(available, "21.0.12+8") + require.NoError(t, err) + assert.Equal(t, "21.0.12+8", got) + + assert.Negative(t, CompareVersions("21.0.12+8", "21.0.12+101")) + assert.Negative(t, CompareVersions("21.0.12", "21.0.12+8")) + assert.Negative(t, CompareVersions("21.0.9+10", "21.0.12+1")) + assert.False(t, IsPrerelease("21.0.13+9.0.LTS")) + assert.True(t, IsPrerelease("3.14.0a1+20250101")) +} + +func TestMatchesConstraint(t *testing.T) { + tests := []struct { + constraint string + version string + want bool + }{ + {"<11", "10.34.9", true}, + {"<11", "11.0.0", false}, + {"<11", "11", false}, + {"<2", "1.22.22", true}, + {"<2", "2.4.1", false}, + {">=2 <4", "3.8.7", true}, + {">=2, <4", "4.0.0", false}, + {">2", "2.0.1", true}, + {"<=1.5", "1.5", true}, + {"<=1.5", "1.5.1", false}, + {"=1.5", "1.5", true}, + {"!=1.5", "1.5", false}, + {"<2.0.1", "2.0.0", true}, + {"<2.0.1", "2.0.1", false}, + {"1.22", "1.22.3", true}, // a bare version matches itself and below + {"1.22", "1.2", false}, + {"", "1.0", false}, + {"<", "1.0", false}, + } + for _, tt := range tests { + t.Run(tt.constraint+" "+tt.version, func(t *testing.T) { + assert.Equal(t, tt.want, MatchesConstraint(tt.version, tt.constraint)) + }) + } + + require.NoError(t, validateConstraint(">=1.2, <3")) + require.Error(t, validateConstraint("")) + require.Error(t, validateConstraint("<")) + require.Error(t, validateConstraint("<<1")) + require.Error(t, validateConstraint("1<2")) +} diff --git a/internal/pkgdef/versions.go b/internal/pkgdef/versions.go index f320491dc60..224560666e6 100644 --- a/internal/pkgdef/versions.go +++ b/internal/pkgdef/versions.go @@ -6,22 +6,64 @@ package pkgdef import ( "context" "encoding/json" + "errors" "fmt" "slices" + "strconv" "strings" ) -// ListVersions returns every version the definition's version source knows -// about, unsorted and including pre-releases. +// maxPages bounds how many pages of a paginated version source are read. +const maxPages = 20 + +// githubPerPage is the page size for the GitHub releases API (its maximum). +const githubPerPage = 100 + +// ListVersions returns every version the definition's version sources +// know about, unsorted and including pre-releases. Paginated sources are +// read up to maxPages. func (c *Client) ListVersions(ctx context.Context, def *Definition) ([]string, error) { - src := def.Versions + return c.listVersions(ctx, def, nil) +} + +// listVersions lists the versions of every source and returns their union. +// When enough is not nil, a paginated source stops being read as soon as +// enough reports true for the versions read from it so far. +func (c *Client) listVersions(ctx context.Context, def *Definition, enough func([]string) bool) ([]string, error) { + var all []string + for _, src := range def.Versions { + versions, err := c.listSource(ctx, def, src, enough) + if err != nil { + return nil, err + } + all = append(all, versions...) + } + return all, nil +} + +func (c *Client) listSource(ctx context.Context, def *Definition, src VersionSource, enough func([]string) bool) ([]string, error) { switch src.Type { case VersionsStatic: return slices.Clone(src.Versions), nil case VersionsHTTPJSON: - return c.listVersionsHTTPJSON(ctx, def.ResolveURL(src.URL), src.Select) + if !strings.Contains(src.URL, ".Page") { + return c.listVersionsHTTPJSON(ctx, def.ResolveURL(src.URL), src.Select) + } + return readPages(enough, 0, func(page int) ([]string, bool, error) { + rawURL, err := Render(src.URL, Vars{Page: page}) + if err != nil { + return nil, false, err + } + versions, err := c.listVersionsHTTPJSON(ctx, def.ResolveURL(rawURL), src.Select) + if errors.Is(err, ErrNotFound) && page > 0 { + return nil, false, nil + } + return versions, len(versions) > 0, err + }) case VersionsGitHubReleases: - return c.listVersionsGitHub(ctx, src) + return readPages(enough, 1, func(page int) ([]string, bool, error) { + return c.listVersionsGitHub(ctx, src, page) + }) case VersionsScript: return nil, fmt.Errorf("versions.type %q: %w", src.Type, ErrNotImplemented) default: @@ -29,10 +71,33 @@ func (c *Client) ListVersions(ctx context.Context, def *Definition) ([]string, e } } +// readPages reads pages starting at first until fetch reports there are no +// more, maxPages have been read, or enough is satisfied. +func readPages(enough func([]string) bool, first int, fetch func(page int) ([]string, bool, error)) ([]string, error) { + var all []string + for page := first; page < first+maxPages; page++ { + versions, more, err := fetch(page) + if err != nil { + return nil, err + } + all = append(all, versions...) + if !more || (enough != nil && enough(all)) { + break + } + } + return all, nil +} + // ResolveVersion resolves a version constraint ("latest", "1.22", "1.22.3") -// to a concrete version using the definition's version source. +// to a concrete version using the definition's version sources. Paginated +// sources are read until a page yields a match, so they should list newest +// versions first. func (c *Client) ResolveVersion(ctx context.Context, def *Definition, requested string) (string, error) { - available, err := c.ListVersions(ctx, def) + enough := func(versions []string) bool { + _, err := MatchVersion(versions, requested) + return err == nil + } + available, err := c.listVersions(ctx, def, enough) if err != nil { return "", fmt.Errorf("%s: listing versions: %w", def.Name, err) } @@ -48,7 +113,7 @@ func (c *Client) listVersionsHTTPJSON(ctx context.Context, rawURL, selectExpr st if err != nil { return nil, err } - body, err := c.get(ctx, rawURL) + body, err := c.getCached(ctx, rawURL) if err != nil { return nil, err } @@ -66,22 +131,26 @@ type githubRelease struct { Prerelease bool `json:"prerelease"` } -func (c *Client) listVersionsGitHub(ctx context.Context, src VersionSource) ([]string, error) { +// listVersionsGitHub lists one page of a repository's releases, newest +// first, and reports whether there may be more pages. +func (c *Client) listVersionsGitHub(ctx context.Context, src VersionSource, page int) ([]string, bool, error) { owner, repo, _ := strings.Cut(src.Repo, "/") stripPrefix := "v" if src.StripPrefix != nil { stripPrefix = *src.StripPrefix } - // The first page of 100 releases is enough for version resolution in - // practice; pagination can be added when a definition needs it. - body, err := c.getGitHubAPI(ctx, "repos/"+owner+"/"+repo+"/releases?per_page=100") + apiPath := "repos/" + owner + "/" + repo + "/releases?per_page=" + strconv.Itoa(githubPerPage) + if page > 1 { + apiPath += "&page=" + strconv.Itoa(page) + } + body, err := c.getGitHubAPI(ctx, apiPath) if err != nil { - return nil, fmt.Errorf("github releases for %s: %w", src.Repo, err) + return nil, false, fmt.Errorf("github releases for %s: %w", src.Repo, err) } var releases []githubRelease if err := json.Unmarshal(body, &releases); err != nil { - return nil, fmt.Errorf("github releases for %s: response is not JSON: %w", src.Repo, err) + return nil, false, fmt.Errorf("github releases for %s: response is not JSON: %w", src.Repo, err) } versions := make([]string, 0, len(releases)) @@ -100,5 +169,5 @@ func (c *Client) listVersionsGitHub(ctx context.Context, src VersionSource) ([]s versions = append(versions, tag) } } - return versions, nil + return versions, len(releases) == githubPerPage, nil } diff --git a/testscripts/pkgdef/pkgdef.test.txt b/testscripts/pkgdef/pkgdef.test.txt index 3971835bf39..ed6d48607b0 100644 --- a/testscripts/pkgdef/pkgdef.test.txt +++ b/testscripts/pkgdef/pkgdef.test.txt @@ -50,11 +50,23 @@ exec devbox rm pkg:./hello.pkg.json@2 ! devboxjson.packages.contains devbox.json 'pkg:./hello.pkg.json@2' ! devboxlock.packages.contains devbox.lock 'pkg:./hello.pkg.json@2' +# Editing devbox.json to a version that is not in devbox.lock yet applies the +# definition's env and init hook in the same run that resolves it. +cp devbox.v1.json devbox.json +exec devbox run -- sh -c 'echo "HELLO_VERSION=$HELLO_VERSION"; cat "$HELLO_INIT_FILE"' +stdout 'HELLO_VERSION=1.2.0' +stdout 'init hook ran for 1.2.0' + -- devbox.json -- { "packages": [] } +-- devbox.v1.json -- +{ + "packages": ["pkg:./hello.pkg.json@1"] +} + -- hello.pkg.json -- // A tiny package definition used by the testscript. {