Fix deploy.jq arch filter precedence and harden registry reading helpers

.test/deploy-arch-mismatch/in.jq

@@ -0,0 +1,59 @@
+# fully synthetic input for "deploy.jq" arch filtering (so nothing here depends on real images, registries, or "builds.json" content)
+
+# obviously fake digests: "a" => sha256:aaaa...
+def digest($c): "sha256:" + ($c * 64);
+
+# $annotation is the value of the bashbrew arch annotation (null means no annotation at all)
+def manifest($c; $arch; $annotation):
+	{
+		mediaType: "application/vnd.oci.image.manifest.v1+json",
+		digest: digest($c),
+		size: 1,
+		annotations: (
+			if $annotation == null then {} else {
+				"com.docker.official-images.bashbrew.arch": $annotation,
+			} end
+		),
+		platform: { os: "linux", architecture: $arch },
+	}
+;
+
+# both builds share the "synthetic:latest" tag (like "library/" tags are shared by every architecture) and have their own arch-specific tag
+def build($arch; $manifests):
+	{
+		buildId: "synthetic-\($arch)",
+		build: {
+			$arch,
+			resolved: {
+				schemaVersion: 2,
+				manifests: $manifests,
+			},
+		},
+		source: {
+			arches: {
+				($arch): {
+					tags: [ "synthetic:latest" ],
+					archTags: [ "synthetic:\($arch)" ],
+				},
+			},
+		},
+	}
+;
+
+[
+	build("amd64"; [
+		manifest("a"; "amd64"; "amd64"), # correct arch: KEEP
+		manifest("b"; "arm64"; "arm64v8"), # annotated for a different arch (cross-architecture "bleeding"): DROP
+		manifest("c"; "amd64"; ""), # empty (but present) annotation, which is truthy in jq: DROP
+		manifest("d"; "amd64"; null), # no annotation at all: DROP
+		empty # trailing comma
+	]),
+	build("arm64v8"; [
+		manifest("e"; "arm64"; "arm64v8"), # correct arch: KEEP
+		manifest("f"; "amd64"; "amd64"), # the same bleeding in the other direction: DROP
+		empty # trailing comma
+	]),
+	empty # trailing comma
+]
+| map({ (.buildId): . })
+| add

.test/deploy-arch-mismatch/in.json

@@ -0,0 +1,118 @@
+{
+	"synthetic-amd64": {
+		"buildId": "synthetic-amd64",
+		"build": {
+			"arch": "amd64",
+			"resolved": {
+				"schemaVersion": 2,
+				"manifests": [
+					{
+						"mediaType": "application/vnd.oci.image.manifest.v1+json",
+						"digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+						"size": 1,
+						"annotations": {
+							"com.docker.official-images.bashbrew.arch": "amd64"
+						},
+						"platform": {
+							"os": "linux",
+							"architecture": "amd64"
+						}
+					},
+					{
+						"mediaType": "application/vnd.oci.image.manifest.v1+json",
+						"digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+						"size": 1,
+						"annotations": {
+							"com.docker.official-images.bashbrew.arch": "arm64v8"
+						},
+						"platform": {
+							"os": "linux",
+							"architecture": "arm64"
+						}
+					},
+					{
+						"mediaType": "application/vnd.oci.image.manifest.v1+json",
+						"digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
+						"size": 1,
+						"annotations": {
+							"com.docker.official-images.bashbrew.arch": ""
+						},
+						"platform": {
+							"os": "linux",
+							"architecture": "amd64"
+						}
+					},
+					{
+						"mediaType": "application/vnd.oci.image.manifest.v1+json",
+						"digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
+						"size": 1,
+						"annotations": {},
+						"platform": {
+							"os": "linux",
+							"architecture": "amd64"
+						}
+					}
+				]
+			}
+		},
+		"source": {
+			"arches": {
+				"amd64": {
+					"tags": [
+						"synthetic:latest"
+					],
+					"archTags": [
+						"synthetic:amd64"
+					]
+				}
+			}
+		}
+	},
+	"synthetic-arm64v8": {
+		"buildId": "synthetic-arm64v8",
+		"build": {
+			"arch": "arm64v8",
+			"resolved": {
+				"schemaVersion": 2,
+				"manifests": [
+					{
+						"mediaType": "application/vnd.oci.image.manifest.v1+json",
+						"digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
+						"size": 1,
+						"annotations": {
+							"com.docker.official-images.bashbrew.arch": "arm64v8"
+						},
+						"platform": {
+							"os": "linux",
+							"architecture": "arm64"
+						}
+					},
+					{
+						"mediaType": "application/vnd.oci.image.manifest.v1+json",
+						"digest": "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+						"size": 1,
+						"annotations": {
+							"com.docker.official-images.bashbrew.arch": "amd64"
+						},
+						"platform": {
+							"os": "linux",
+							"architecture": "amd64"
+						}
+					}
+				]
+			}
+		},
+		"source": {
+			"arches": {
+				"arm64v8": {
+					"tags": [
+						"synthetic:latest"
+					],
+					"archTags": [
+						"synthetic:arm64v8"
+					]
+				}
+			}
+		}
+	}
+}

.test/deploy-arch-mismatch/out.json

@@ -0,0 +1,72 @@
+{
+	"synthetic:latest": [
+		{
+			"mediaType": "application/vnd.oci.image.manifest.v1+json",
+			"digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+			"size": 1,
+			"annotations": {
+				"com.docker.official-images.bashbrew.arch": "amd64"
+			},
+			"platform": {
+				"os": "linux",
+				"architecture": "amd64"
+			}
+		},
+		{
+			"mediaType": "application/vnd.oci.image.manifest.v1+json",
+			"digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
+			"size": 1,
+			"annotations": {
+				"com.docker.official-images.bashbrew.arch": "arm64v8"
+			},
+			"platform": {
+				"os": "linux",
+				"architecture": "arm64"
+			}
+		}
+	],
+	"synthetic:amd64": [
+		{
+			"mediaType": "application/vnd.oci.image.manifest.v1+json",
+			"digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+			"size": 1,
+			"annotations": {
+				"com.docker.official-images.bashbrew.arch": "amd64"
+			},
+			"platform": {
+				"os": "linux",
+				"architecture": "amd64"
+			}
+		}
+	],
+	"synthetic:arm64v8": [
+		{
+			"mediaType": "application/vnd.oci.image.manifest.v1+json",
+			"digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
+			"size": 1,
+			"annotations": {
+				"com.docker.official-images.bashbrew.arch": "arm64v8"
+			},
+			"platform": {
+				"os": "linux",
+				"architecture": "arm64"
+			}
+		}
+	]
+}
+{
+	"synthetic:amd64": [
+		{
+			"mediaType": "application/vnd.oci.image.manifest.v1+json",
+			"digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+			"size": 1,
+			"annotations": {
+				"com.docker.official-images.bashbrew.arch": "amd64"
+			},
+			"platform": {
+				"os": "linux",
+				"architecture": "amd64"
+			}
+		}
+	]
+}

.test/deploy-arch-mismatch/test.jq

@@ -0,0 +1,7 @@
+include "deploy";
+
+# every ref ("library/" and arch-specific) for every build: only "a" and "e" should survive (each under its own build's tags, and both under the shared tag)
+tagged_manifests(true; .source.arches[.build.arch].tags, .source.arches[.build.arch].archTags),
+
+# just amd64: only "a"
+arch_tagged_manifests("amd64")

.test/test.sh

@@ -19,6 +19,8 @@ export BASHBREW_STAGING_TEMPLATE='oisupport/staging-ARCH:BUILD'
 dir="$(dirname "$BASH_SOURCE")"
 dir="$(readlink -ve "$dir")"
 export BASHBREW_LIBRARY="$dir/library"
+BASHBREW_META_SCRIPTS="$(dirname "$dir")"
+export BASHBREW_META_SCRIPTS
 
 doDeploy=
 if [ "${1:-}" = '--deploy' ]; then
@@ -32,15 +34,15 @@ set -- docker:cli docker:dind docker:windowsservercore notary busybox:{latest,gl
 time bashbrew fetch "$@"
 
 # generate sources, but remove the first item so we can test cache with some missing
-time "$dir/../sources.sh" "$@" | jq --tab 'del(first(.[]))' > "$dir/sources-cache.json"
+time "$BASHBREW_META_SCRIPTS/sources.sh" "$@" | jq --tab 'del(first(.[]))' > "$dir/sources-cache.json"
 # again but with cache
-time "$dir/../sources.sh" --cache-file "$dir/sources-cache.json" "$@" > "$dir/sources-doi.json"
+time "$BASHBREW_META_SCRIPTS/sources.sh" --cache-file "$dir/sources-cache.json" "$@" > "$dir/sources-doi.json"
 
 # also fetch/include Tianon's more cursed "infosiftr/moby" example (a valid manifest with arch-specific non-archTags that end up mapping to the same sourceId)
 bashbrew fetch infosiftr-moby
-( BASHBREW_ARCH_NAMESPACES= "$dir/../sources.sh" infosiftr-moby > "$dir/sources-cache.json" )
+( BASHBREW_ARCH_NAMESPACES= "$BASHBREW_META_SCRIPTS/sources.sh" infosiftr-moby > "$dir/sources-cache.json" )
 # again but with cache
-( BASHBREW_ARCH_NAMESPACES= "$dir/../sources.sh" --cache-file="$dir/sources-cache.json" infosiftr-moby > "$dir/sources-moby.json" )
+( BASHBREW_ARCH_NAMESPACES= "$BASHBREW_META_SCRIPTS/sources.sh" --cache-file="$dir/sources-cache.json" infosiftr-moby > "$dir/sources-moby.json" )
 # technically, this *also* needs BASHBREW_STAGING_TEMPLATE='tianon/zz-staging:ARCH-BUILD', but that's a "builds.sh" flag and separating that would complicate including this even more, so Tianon has run the following one-liner to "inject" those builds as if they lived in 'oisupport/staging-ARCH:BUILD' instead:
 #   jq --raw-output '[ .[] | select(any(.source.arches[].tags[]; startswith("infosiftr-moby:"))) | "tianon/zz-staging:\(.build.arch)-\(.buildId)" as $tianon | @sh "../bin/lookup \($tianon) | jq --arg img \(.build.img) \("{ indexes: { ($img): . } }")" ] | "{ " + join(" && ") + @sh " && cat cache-builds.json; } | jq --slurp --tab \("reduce .[] as $i ({ indexes: { } }; .indexes += $i.indexes)") > cache-builds.json.new && mv cache-builds.json.new cache-builds.json"' builds.json | bash -Eeuo pipefail -x
 # (and then re-run the tests to canonicalize the file ordering)
@@ -77,7 +79,7 @@ time "$coverage/builds.sh" --cache "$dir/cache-builds.json" "$dir/sources.json"
 time "$coverage/builds.sh" --cache="$dir/cache-builds.json" "$dir/sources.json" > "$dir/builds.json"
 
 # test "lookup" code for more edge cases
-"$dir/../.go-env.sh" go build -coverpkg=./... -trimpath -o "$coverage/bin/lookup" ./cmd/lookup
+"$BASHBREW_META_SCRIPTS/.go-env.sh" go build -coverpkg=./... -trimpath -o "$coverage/bin/lookup" ./cmd/lookup
 lookup=(
 	# force a config blob lookup for platform object creation (and top-level Docker media type!)
 	'tianon/test@sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23'
@@ -135,7 +137,7 @@ lookup=(
 if [ -n "$doDeploy" ]; then
 	# also test "deploy" (optional, disabled by default, because it's a much heavier test)
 
-	"$dir/../.go-env.sh" go build -coverpkg=./... -trimpath -o "$coverage/bin/deploy" ./cmd/deploy
+	"$BASHBREW_META_SCRIPTS/.go-env.sh" go build -coverpkg=./... -trimpath -o "$coverage/bin/deploy" ./cmd/deploy
 
 	docker rm -vf meta-scripts-test-registry &> /dev/null || :
 	trap 'docker rm -vf meta-scripts-test-registry &> /dev/null || :' EXIT
@@ -270,12 +272,12 @@ if [ -n "$doDeploy" ]; then
 fi
 
 # Go tests
-"$dir/../.go-env.sh" go test -coverpkg=./... ./... -args -test.gocoverdir="$GOCOVERDIR"
+"$BASHBREW_META_SCRIPTS/.go-env.sh" go test -coverpkg=./... ./... -args -test.gocoverdir="$GOCOVERDIR"
 
 # combine the coverage data into the "legacy" coverage format (understood by "go tool cover") and pre-generate HTML for easier digestion of the data
-"$dir/../.go-env.sh" go tool covdata textfmt -i "$GOCOVERDIR" -o "$coverage/coverage.txt"
-"$dir/../.go-env.sh" go tool cover -html "$coverage/coverage.txt" -o "$coverage/coverage.html"
-"$dir/../.go-env.sh" go tool cover -func "$coverage/coverage.txt"
+"$BASHBREW_META_SCRIPTS/.go-env.sh" go tool covdata textfmt -i "$GOCOVERDIR" -o "$coverage/coverage.txt"
+"$BASHBREW_META_SCRIPTS/.go-env.sh" go tool cover -html "$coverage/coverage.txt" -o "$coverage/coverage.html"
+"$BASHBREW_META_SCRIPTS/.go-env.sh" go tool cover -func "$coverage/coverage.txt"
 
 # also run our "jq" tests (like generating example commands from the "builds.json" we just generated)
 "$dir/jq.sh"

Jenkinsfile.meta

@@ -66,6 +66,7 @@ node('doi-meta') { // use the dedicated meta instance
 	}
 
 	env.BASHBREW_LIBRARY = workspace + '/meta/.doi/library'
+	env.BASHBREW_META_SCRIPTS = workspace + '/meta/.scripts'
 
 	dir('meta') {
 		withCredentials([

cmd/builds/main.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/docker-library/meta-scripts/om"
 	"github.com/docker-library/meta-scripts/registry"
+	"github.com/docker-library/meta-scripts/sm"
 
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
@@ -50,7 +51,7 @@ type MetaBuild struct {
 
 var (
 	// keys are image/tag names, values are functions that return either *ocispec.Index or error
-	cacheResolve = sync.Map{}
+	cacheResolve = sm.Map[string, func() (*ocispec.Index, error)]{}
 	cacheFile    string
 )
 
@@ -71,7 +72,7 @@ func resolveIndex(ctx context.Context, img string, diskCacheForSure bool) (*ocis
 		return registry.SynthesizeIndex(ctx, ref)
 	}))
 
-	index, err := cacheFunc.(func() (*ocispec.Index, error))()
+	index, err := cacheFunc()
 	if err != nil {
 		return nil, err
 	}
@@ -174,7 +175,7 @@ func loadCacheFromFile() error {
 		fun, _ := cacheResolve.LoadOrStore(img.String(), sync.OnceValues(func() (*ocispec.Index, error) {
 			return index, nil
 		}))
-		index2, err := fun.(func() (*ocispec.Index, error))()
+		index2, err := fun()
 		if err != nil {
 			// this should never happen (hence panic vs return) 🙈
 			panic(err)