diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 0fb3ff8..b2fb207 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -21,7 +21,7 @@ "ghcr.io/devcontainers/features/github-cli:1": {} }, // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "echo \"export \\\"PATH=$(make -s tools-path):\\${PATH}\\\"\" >> ~/.bashrc && curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash && cargo check && make tools", + "postCreateCommand": "./scripts/init-devcontainer.sh", // Configure tool-specific properties. "customizations": { // Configure properties specific to VS Code. diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8ec1844..fc87e11 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,22 +1,22 @@ version: 2 updates: -- package-ecosystem: github-actions - directory: "/" - schedule: - interval: daily -- package-ecosystem: cargo - directory: "/" - schedule: - interval: daily -- package-ecosystem: cargo - directory: "/tools" - schedule: - interval: daily -- package-ecosystem: rust-toolchain - directory: "/" - schedule: - interval: daily -- package-ecosystem: devcontainers - directory: "/" - schedule: - interval: weekly + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: daily + - package-ecosystem: cargo + directory: "/" + schedule: + interval: daily + - package-ecosystem: cargo + directory: "/tools" + schedule: + interval: daily + - package-ecosystem: rust-toolchain + directory: "/" + schedule: + interval: daily + - package-ecosystem: devcontainers + directory: "/" + schedule: + interval: weekly diff --git a/.github/workflows/bump-version.yaml b/.github/workflows/bump-version.yaml index 7321cf4..e55203e 100644 --- a/.github/workflows/bump-version.yaml +++ b/.github/workflows/bump-version.yaml @@ -1,159 +1,203 @@ name: Bump version on: - workflow_dispatch: - inputs: - version: - description: The new version of the interface package and crates, e.g. 0.1.0 or 0.2.0-dev - required: true - type: string - default: "0.1.0-dev" # the current version, kept current by scripts/bump-version.sh + workflow_dispatch: + inputs: + version: + description: The new version of the interface package, e.g. 0.1.0 or 0.2.0-dev + required: true + type: string + default: "0.1.0-dev" # the current version, kept current by scripts/bump-version.sh jobs: - # bumps the version with read only access, the changes are handed to the pull-request job as a - # patch so the third party actions used to build never run with write access - bump: - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@v2 - - name: Install cargo binstall - uses: cargo-bins/cargo-binstall@main - - name: Install tools - run: | - make tools - make -s tools-path >> "${GITHUB_PATH}" - - name: Bump version - # also fetches the wit dependencies for the new version, and builds and tests the components - run: scripts/bump-version.sh "${VERSION}" - env: - VERSION: ${{ inputs.version }} - - name: Collect changes - run: | - git add --all - git diff --cached --binary > bump-version.patch - - name: Upload changes - uses: actions/upload-artifact@v7 - with: - name: bump-version.patch - path: bump-version.patch - if-no-files-found: error - retention-days: 1 + # bumps the version with read only access, the changes are handed to the push job as a patch so + # the third party actions used to build never run with write access + bump: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Check version + # a semver version without a leading `v`, the tag adds it. Build metadata is used to tag the + # debug builds of components, e.g. 0.1.0+debug, and isn't allowed in the version itself. + run: | + identifier='(0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)' + semver="^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-${identifier}(\.${identifier})*)?$" + if ! [[ "${VERSION}" =~ ${semver} ]] ; then + echo "::error::the version '${VERSION}' is not a valid semver version, e.g. 0.1.0 or 0.2.0-dev" + exit 1 + fi + if [[ "${VERSION}" == *+* ]] ; then + echo "::error::the version '${VERSION}' must not contain build metadata" + exit 1 + fi + env: + VERSION: ${{ inputs.version }} + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions-rust-lang/setup-rust-toolchain@v2 + - name: Install cargo binstall + uses: cargo-bins/cargo-binstall@main + - name: Install tools + run: | + make tools + make -s tools-path >> "${GITHUB_PATH}" + - name: Bump version + # also fetches the wit dependencies for the new version, and builds and tests the components + run: scripts/bump-version.sh "${VERSION}" + env: + VERSION: ${{ inputs.version }} + - name: Collect changes + run: | + git add --all + git diff --cached --binary > bump-version.patch + - name: Upload changes + uses: actions/upload-artifact@v7 + with: + name: bump-version.patch + path: bump-version.patch + if-no-files-found: error + retention-days: 1 - # opens the pull request using only first party actions and the gh cli - pull-request: - needs: - - bump - runs-on: ubuntu-latest - # the branch and pull request are created with a token for the custodian GitHub App rather than - # the GITHUB_TOKEN, which can't change workflow files and doesn't trigger the CI workflow - permissions: - contents: read + # pushes the bump to the branch the workflow was run on, using only first party actions and the gh + # cli + push: + needs: + - bump + # the bump is a new commit on the branch, a tag can't be moved forward + if: github.ref_type == 'branch' + runs-on: ubuntu-latest + # the commit is pushed with a token for the custodian GitHub App rather than the GITHUB_TOKEN, + # which can't change workflow files and doesn't trigger the CI workflow. The app must be allowed + # to bypass the branch's ruleset, the commit is pushed without the status checks it requires, + # the bump job built and tested the changes instead. + permissions: + contents: read + env: + VERSION: ${{ inputs.version }} + steps: + - name: Check custodian app credentials + run: | + if [ -z "${CLIENT_ID}" ] || [ -z "${PRIVATE_KEY}" ] ; then + echo "::error::the CUSTODIAN_CLIENT_ID and CUSTODIAN_PRIVATE_KEY secrets must be available to this repository, the private key of the custodian GitHub App is needed to create a token" + exit 1 + fi + env: + CLIENT_ID: ${{ secrets.CUSTODIAN_CLIENT_ID }} + PRIVATE_KEY: ${{ secrets.CUSTODIAN_PRIVATE_KEY }} + - name: Create custodian app token + id: app-token + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ secrets.CUSTODIAN_CLIENT_ID }} + private-key: ${{ secrets.CUSTODIAN_PRIVATE_KEY }} + # only this repository, with only the permissions the bump needs + repositories: ${{ github.event.repository.name }} + permission-contents: write + # the bump changes the default version in this workflow + permission-workflows: write + # a release runs this workflow again, to bump to the next dev version + permission-actions: write + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Download changes + uses: actions/download-artifact@v8 + with: + name: bump-version.patch + path: ${{ runner.temp }} + - name: Read current version + # the checkout is before the bump, the wit package's version is the current version + run: | + current=$( sed -n 's/^package componentized:sockets@\(.*\);$/\1/p' wit/worlds.wit ) + echo "CURRENT_VERSION=${current}" >> "${GITHUB_ENV}" + - name: Commit changes + # the commit is created with the REST API, as the app's token can't push. The patch is applied + # locally only to find the changed files and their modes. env: - VERSION: ${{ inputs.version }} - steps: - - name: Check custodian app credentials - run: | - if [ -z "${CLIENT_ID}" ] || [ -z "${PRIVATE_KEY}" ] ; then - echo "::error::the CUSTODIAN_CLIENT_ID and CUSTODIAN_PRIVATE_KEY secrets must be available to this repository, the private key of the custodian GitHub App is needed to create a token" - exit 1 - fi - env: - CLIENT_ID: ${{ secrets.CUSTODIAN_CLIENT_ID }} - PRIVATE_KEY: ${{ secrets.CUSTODIAN_PRIVATE_KEY }} - - name: Create custodian app token - id: app-token - uses: actions/create-github-app-token@v3 - with: - client-id: ${{ secrets.CUSTODIAN_CLIENT_ID }} - private-key: ${{ secrets.CUSTODIAN_PRIVATE_KEY }} - # only this repository, with only the permissions the bump needs - repositories: ${{ github.event.repository.name }} - permission-contents: write - permission-pull-requests: write - # the bump changes the default version in this workflow - permission-workflows: write - - uses: actions/checkout@v7 - with: - persist-credentials: false - - name: Download changes - uses: actions/download-artifact@v8 - with: - name: bump-version.patch - path: ${{ runner.temp }} - - name: Read current version - # the checkout is before the bump, the crates' workspace version is the current version - run: | - current=$( sed -n '/^\[workspace.package\]/,/^\[/s/^version = "\(.*\)"$/\1/p' Cargo.toml ) - echo "CURRENT_VERSION=${current}" >> "${GITHUB_ENV}" - - name: Commit changes - # the commit is created with the REST API, as the app's token can't push. The patch is applied - # locally only to find the changed files and their modes. - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - APP_SLUG: ${{ steps.app-token.outputs.app-slug }} - run: | - branch="bump-version/${VERSION}" - api="repos/${GITHUB_REPOSITORY}" - base=$( git rev-parse HEAD ) - git apply --index "${RUNNER_TEMP}/bump-version.patch" + GH_TOKEN: ${{ steps.app-token.outputs.token }} + APP_SLUG: ${{ steps.app-token.outputs.app-slug }} + run: | + api="repos/${GITHUB_REPOSITORY}" + base=$( git rev-parse HEAD ) + git apply --index "${RUNNER_TEMP}/bump-version.patch" - # a blob for each changed file, or a null sha for a deleted file - entries="${RUNNER_TEMP}/tree-entries.json" - echo '[]' > "${entries}" - git diff --cached --no-renames --name-status "${base}" | while IFS=$'\t' read -r status path ; do - if [ "${status}" = "D" ] ; then - entry=$( jq -n --arg path "${path}" '{path: $path, mode: "100644", type: "blob", sha: null}' ) - else - mode=$( git ls-files --stage -- "${path}" | cut -d' ' -f1 ) - sha=$( base64 < "${path}" | tr -d '\n' | jq -Rs '{encoding: "base64", content: .}' | gh api --method POST "${api}/git/blobs" --input - --jq .sha ) - entry=$( jq -n --arg path "${path}" --arg mode "${mode}" --arg sha "${sha}" '{path: $path, mode: $mode, type: "blob", sha: $sha}' ) - fi - jq --argjson entry "${entry}" '. + [$entry]' "${entries}" > "${entries}.tmp" && mv "${entries}.tmp" "${entries}" - echo "${status} ${path}" - done - tree=$( jq --arg base "$( git rev-parse "${base}^{tree}" )" '{base_tree: $base, tree: .}' "${entries}" | gh api --method POST "${api}/git/trees" --input - --jq .sha ) + # a blob for each changed file, or a null sha for a deleted file + entries="${RUNNER_TEMP}/tree-entries.json" + echo '[]' > "${entries}" + git diff --cached --no-renames --name-status "${base}" | while IFS=$'\t' read -r status path ; do + if [ "${status}" = "D" ] ; then + entry=$( jq -n --arg path "${path}" '{path: $path, mode: "100644", type: "blob", sha: null}' ) + else + mode=$( git ls-files --stage -- "${path}" | cut -d' ' -f1 ) + sha=$( base64 < "${path}" | tr -d '\n' | jq -Rs '{encoding: "base64", content: .}' | gh api --method POST "${api}/git/blobs" --input - --jq .sha ) + entry=$( jq -n --arg path "${path}" --arg mode "${mode}" --arg sha "${sha}" '{path: $path, mode: $mode, type: "blob", sha: $sha}' ) + fi + jq --argjson entry "${entry}" '. + [$entry]' "${entries}" > "${entries}.tmp" && mv "${entries}.tmp" "${entries}" + echo "${status} ${path}" + done + tree=$( jq --arg base "$( git rev-parse "${base}^{tree}" )" '{base_tree: $base, tree: .}' "${entries}" | gh api --method POST "${api}/git/trees" --input - --jq .sha ) - # authored and signed off (DCO) by the user who triggered the workflow, with their GitHub - # noreply email so the commit is attributed to them without exposing their email address. - # Committed by the custodian app's bot, which made the commit on their behalf. The commit is - # unsigned, GitHub only signs commits it attributes entirely to the app. - name=$( gh api "users/${GITHUB_ACTOR}" --jq '.name // .login' ) - name="${name:-${GITHUB_ACTOR}}" - email="${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com" - bot="${APP_SLUG}[bot]" - bot_email="$( gh api "users/${bot}" --jq .id )+${bot}@users.noreply.github.com" - commit=$( jq -n \ - --arg message "$( printf 'Bump version from %s to %s\n\nSigned-off-by: %s <%s>' "${CURRENT_VERSION}" "${VERSION}" "${name}" "${email}" )" \ - --arg tree "${tree}" --arg parent "${base}" --arg name "${name}" --arg email "${email}" \ - --arg bot "${bot}" --arg bot_email "${bot_email}" \ - '{message: $message, tree: $tree, parents: [$parent], author: {name: $name, email: $email}, committer: {name: $bot, email: $bot_email}}' \ - | gh api --method POST "${api}/git/commits" --input - --jq .sha ) - echo "created commit ${commit}" + # authored, committed and signed off (DCO) by the custodian app's bot. GitHub only signs + # commits it attributes entirely to the app, so the author and committer are left for GitHub + # to fill in. The user who triggered the workflow is credited as a co-author, with their + # GitHub noreply email so their email address isn't exposed. A run started by the app after a + # release has no one else to credit. + bot="${APP_SLUG}[bot]" + bot_email="$( gh api "users/${bot}" --jq .id )+${bot}@users.noreply.github.com" + message=$( printf 'Bump version from %s to %s\n\nSigned-off-by: %s <%s>' "${CURRENT_VERSION}" "${VERSION}" "${bot}" "${bot_email}" ) + if [ "${GITHUB_ACTOR}" != "${bot}" ] ; then + name=$( gh api "users/${GITHUB_ACTOR}" --jq '.name // .login' ) + name="${name:-${GITHUB_ACTOR}}" + email="${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com" + message=$( printf '%s\nCo-authored-by: %s <%s>' "${message}" "${name}" "${email}" ) + fi + read -r commit verified < <( jq -n \ + --arg message "${message}" \ + --arg tree "${tree}" --arg parent "${base}" \ + '{message: $message, tree: $tree, parents: [$parent]}' \ + | gh api --method POST "${api}/git/commits" --input - --jq '"\(.sha) \(.verification.verified)"' ) + echo "created commit ${commit}" + # the branch requires signed commits, fail before pushing rather than after + if [ "${verified}" != "true" ] ; then + echo "::error::the commit '${commit}' was not created, or was not signed by GitHub" + exit 1 + fi - # points the branch at the commit, replacing the branch left by an earlier run for the same version - if gh api "${api}/git/ref/heads/${branch}" --silent 2> /dev/null ; then - gh api --method PATCH "${api}/git/refs/heads/${branch}" -f sha="${commit}" -F force=true --silent - else - gh api --method POST "${api}/git/refs" -f ref="refs/heads/${branch}" -f sha="${commit}" --silent - fi - - name: Open pull request - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - run: | - branch="bump-version/${VERSION}" - if [ -n "$( gh pr list --head "${branch}" --state open --json number --jq '.[].number' )" ] ; then - echo "A pull request for ${branch} is already open, updated by the new commit" - exit 0 - fi - gh pr create \ - --base "${GITHUB_REF_NAME}" \ - --head "${branch}" \ - --title "Bump version from \`${CURRENT_VERSION}\` to \`${VERSION}\`" \ - --body "Bumps the wit package and crates from \`${CURRENT_VERSION}\` to \`${VERSION}\`. + # a release, rather than a pre-release, is tagged, e.g. v0.1.0. Tags are immutable, check the + # tag is free before pushing the branch, so a release isn't left without its tag + tag="" + if [[ "${VERSION}" != *-* ]] ; then + tag="v${VERSION}" + if gh api "${api}/git/ref/tags/${tag}" --silent 2> /dev/null ; then + echo "::error::the tag ${tag} already exists" + exit 1 + fi + fi - Triggered by @${GITHUB_ACTOR} from the [Bump version](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}) workflow." + # fast forwards the branch to the commit, failing rather than losing commits pushed to the + # branch since the workflow started + if ! gh api --method PATCH "${api}/git/refs/heads/${GITHUB_REF_NAME}" -f sha="${commit}" -F force=false --silent ; then + echo "::error::failed to push ${commit} to ${GITHUB_REF_NAME}, either the branch moved since ${base} and the workflow should be run again, or the custodian app is not allowed to bypass the branch's ruleset" + exit 1 + fi + echo "pushed ${commit} to ${GITHUB_REF_NAME}" + + # a lightweight tag, pushed with the app's token so the CI workflow runs for it and drafts + # the release + if [ -n "${tag}" ] ; then + gh api --method POST "${api}/git/refs" -f ref="refs/tags/${tag}" -f sha="${commit}" --silent + echo "tagged ${commit} as ${tag}" + fi + - name: Bump to the next dev version + # after a release, the branch moves on to a pre-release of the next patch version, e.g. 0.3.0 + # is followed by 0.3.1-dev. Run with the app's token, the GITHUB_TOKEN can't start workflows. + if: ${{ !contains(inputs.version, '-') }} + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + IFS=. read -r major minor patch <<< "${VERSION}" + next="${major}.${minor}.$(( patch + 1 ))-dev" + gh workflow run bump-version.yaml --repo "${GITHUB_REPOSITORY}" --ref "${GITHUB_REF_NAME}" -f version="${next}" + echo "started the ${GITHUB_WORKFLOW} workflow for ${next} on ${GITHUB_REF_NAME}" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cd547f3..64d6a83 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -100,16 +100,30 @@ jobs: if: startsWith(github.ref, 'refs/tags/') || (github.ref == 'refs/heads/main' && contains(steps.interface_version.outputs.VERSION, '-')) run: make publish env: + PUBLISH_LOG: "${{ runner.temp }}/published.txt" REPOSITORY: "ghcr.io/${{ github.repository }}" VERSION: "${{ case(github.ref == 'refs/heads/main', steps.interface_version.outputs.VERSION, steps.tag_version.outputs.VERSION) }}" + - name: Draft release notes + if: startsWith(github.ref, 'refs/tags/') || (github.ref == 'refs/heads/main' && contains(steps.interface_version.outputs.VERSION, '-')) + run: | + { + echo "## Published components" + echo + echo "| File | Size | Image |" + echo "| --- | --: | --- |" + while read -r file bytes image ; do + size=$( numfmt --to=iec-i --suffix=B --format=%.1f "${bytes}" ) + echo "| \`${file}\` | ${size} | \`${image}\` |" + done < "${RUNNER_TEMP}/published.txt" + } > "${RUNNER_TEMP}/release-notes.md" + cat "${RUNNER_TEMP}/release-notes.md" - name: Draft GitHub Release if: startsWith(github.ref, 'refs/tags/') uses: softprops/action-gh-release@v3 with: draft: true + body_path: ${{ runner.temp }}/release-notes.md files: | - target/components/*.wasm - target/components/*/*.wasm components.tar fail_on_unmatched_files: true token: ${{ secrets.GITHUB_TOKEN }} diff --git a/Cargo.toml b/Cargo.toml index 1fa378a..303e58b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,3 +47,18 @@ wac-graph = "0.12" wasmtime = { version = "49", features = ["component-model-async"] } wasmtime-wasi = { version = "49", features = ["p3"] } wit-bindgen = { version = "0.62.0", features = ["async-spawn"] } + +[profile.release] +opt-level = "s" +lto = true +codegen-units = 1 +strip = "debuginfo" + +[profile.component-debug] +inherits = "dev" +opt-level = 1 # most of the size win; locals mostly still visible +lto = true +codegen-units = 1 +incremental = false # no benefit with one codegen unit +debug = "full" # variables and types for lldb, file:line in backtraces +# kept from dev: debug-assertions, overflow-checks, no strip diff --git a/Makefile b/Makefile index bceba92..9e26af7 100644 --- a/Makefile +++ b/Makefile @@ -2,209 +2,30 @@ SHELL := /bin/bash export RUST_BACKTRACE ?= 1 export WASMTIME_BACKTRACE_DETAILS ?= 1 - -COMPONENTS_DIR := target/components -TOOLS_DIR := target/tools/$(shell rustc --print host-tuple) -# absolute, tools also run from other directories, e.g. `cd components && wkg fetch` -export PATH := $(abspath $(TOOLS_DIR))/bin:$(PATH) - -# cargo binstall downloads prebuilt binaries, without it the tools are built with cargo install -CARGO_INSTALL := $(if $(shell command -v cargo-binstall 2> /dev/null),cargo binstall --no-confirm --disable-telemetry,cargo install) - -COMPONENTS = $(sort $(foreach file,$(wildcard $(addprefix components/*/,wit/*.constants.wit *.properties *.wac *.wkg Cargo.toml)),$(word 2,$(subst /, ,$(file))))) -TOOLS := componentized-constants-cli componentized-static-config-cli wac-cli wasm-tools wasmtime-cli wkg - export WKG_CONFIG_FILE := $(abspath .config/wasm-pkg/config.toml) -# a path relative to the root of the repository, e.g. `wit` for `components/../wit` -relpath = $(if $(filter $(CURDIR),$(abspath $(1))),.,$(patsubst $(CURDIR)/%,%,$(abspath $(1)))) +# `make` without a goal runs all, otherwise the first target of an included file would run +.DEFAULT_GOAL := all + +# tools first, the others use the variables it defines for each tool when their rules are read +include make/tools.mk +include make/wit.mk +include make/components.mk +include make/publish.mk .PHONY: all -all: components +all: tools wit components test .PHONY: clean -clean: clean-wit - cargo clean +clean: clean-components clean-wit + @: -.PHONY: clean-components -clean-components: clean-wit - rm -rf ${COMPONENTS_DIR} +.PHONY: clean-all +clean-all: clean-components clean-tools clean-wit + cargo clean -.PHONY: clean-wit ## Remove the fetched wit dependencies, fetched again by `make wit` -clean-wit: - rm -rf $(WIT_DEPS) .PHONY: test test: components cargo test --workspace - - -tool_version = $(shell sed -n 's/^$(1) = "=\(.*\)"$$/\1/p' tools/Cargo.toml) -# a stamp naming the version of a tool installed in $(TOOLS_DIR)/bin, e.g. `wkg@0.16.1`, the binary -# does not say which version it is. Bumping the pinned version names a stamp that does not exist yet, -# so the tool is installed again. -tool = $(TOOLS_DIR)/.installed/$(1)@$(call tool_version,$(1)) - -.PHONY: tools ## Install the cli tools pinned in tools/Cargo.toml -tools: $(foreach name,$(TOOLS),$(call tool,$(name))) - -.PHONY: tools-path ## Print the directory of the installed tools for this platform, to add to the PATH -tools-path: - @echo $(abspath $(TOOLS_DIR))/bin - -define INSTALL_TOOL - -$(call tool,$1): - $(CARGO_INSTALL) --locked --root $(TOOLS_DIR) --version $(call tool_version,$1) $1 - @mkdir -p $$(@D) - @# only the installed version has a stamp, so going back to a previous version installs it again - @rm -f $$(@D)/$1@* - @touch $$@ - -endef - -$(foreach name,$(TOOLS),$(eval $(call INSTALL_TOOL,$(name)))) - -.PHONY: components -components: ${COMPONENTS_DIR}/interface.wasm $(foreach component,$(COMPONENTS),${COMPONENTS_DIR}/$(component)/$(component).wasm ${COMPONENTS_DIR}/$(component)/$(component).debug.wasm) - -define BUILD_COMPONENT - -.PHONY: components/$1 -components/$1: ${COMPONENTS_DIR}/$1/$1.wasm ${COMPONENTS_DIR}/$1/$1.debug.wasm - -ifneq ($(wildcard components/$1/wit/$1.constants.wit),) - -${COMPONENTS_DIR}/$1/$1.wasm: components/$1/wit/deps ${COMPONENTS_DIR}/$1/README.md | $(call tool,componentized-constants-cli) - constants --wit components/$1/wit -o ${COMPONENTS_DIR}/$1/$1.wasm - -${COMPONENTS_DIR}/$1/$1.debug.wasm: components/$1/wit/deps ${COMPONENTS_DIR}/$1/README.md | $(call tool,componentized-constants-cli) - constants --wit components/$1/wit -o ${COMPONENTS_DIR}/$1/$1.debug.wasm - -else ifneq ($(wildcard components/$1/$1.properties),) - -${COMPONENTS_DIR}/$1/$1.wasm: components/$1/$1.properties ${COMPONENTS_DIR}/$1/README.md | $(call tool,componentized-static-config-cli) - static-config -f components/$1/$1.properties -o ${COMPONENTS_DIR}/$1/$1.wasm - -${COMPONENTS_DIR}/$1/$1.debug.wasm: components/$1/$1.properties ${COMPONENTS_DIR}/$1/README.md | $(call tool,componentized-static-config-cli) - static-config -f components/$1/$1.properties -o ${COMPONENTS_DIR}/$1/$1.debug.wasm - -else ifneq ($(wildcard components/$1/$1.wac),) - -# the local packages the composition instantiates, e.g. `new local:latch-n2 { ... }` -WAC_DEPS_$1 := $$(shell grep -v '^\s*//' components/$1/$1.wac | grep -oE 'local:[a-z0-9-]+' | sed 's/^local://' | sort -u) - -${COMPONENTS_DIR}/$1/$1.wasm: components/$1/$1.wac $$(foreach component,$$(WAC_DEPS_$1),$${COMPONENTS_DIR}/$$(component)/$$(component).wasm) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wac-cli) - wac compose $$(foreach component,$$(WAC_DEPS_$1),-d local:$$(component)=$${COMPONENTS_DIR}/$$(component)/$$(component).wasm) -o ${COMPONENTS_DIR}/$1/$1.wasm components/$1/$1.wac - -${COMPONENTS_DIR}/$1/$1.debug.wasm: components/$1/$1.wac $$(foreach component,$$(WAC_DEPS_$1),$${COMPONENTS_DIR}/$$(component)/$$(component).debug.wasm) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wac-cli) - wac compose $$(foreach component,$$(WAC_DEPS_$1),-d local:$$(component)=$${COMPONENTS_DIR}/$$(component)/$$(component).debug.wasm) -o ${COMPONENTS_DIR}/$1/$1.debug.wasm components/$1/$1.wac - -else ifneq ($(wildcard components/$1/$1.wkg),) - -${COMPONENTS_DIR}/$1/$1.wasm: components/$1/$1.wkg ${COMPONENTS_DIR}/$1/README.md | $(call tool,wkg) - wkg oci pull $(shell cat components/$1/$1.wkg 2> /dev/null | head -1) -o ${COMPONENTS_DIR}/$1/$1.wasm - -${COMPONENTS_DIR}/$1/$1.debug.wasm: components/$1/$1.wkg ${COMPONENTS_DIR}/$1/README.md | $(call tool,wkg) - wkg oci pull $(shell cat components/$1/$1.wkg 2> /dev/null | tail -1 2> /dev/null) -o ${COMPONENTS_DIR}/$1/$1.debug.wasm - -# cargo is checked last, other strategies may have a Cargo.toml for tests of non-rust sources -else ifneq ($(wildcard components/$1/Cargo.toml),) - -${COMPONENTS_DIR}/$1/$1.wasm: Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f) $(shell find crates -type f 2> /dev/null) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wasm-tools) - cargo build -p $1 --target wasm32-unknown-unknown --release - wasm-tools component new target/wasm32-unknown-unknown/release/$(subst -,_,$1).wasm -o ${COMPONENTS_DIR}/$1/$1.wasm - -${COMPONENTS_DIR}/$1/$1.debug.wasm: Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f) $(shell find crates -type f 2> /dev/null) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wasm-tools) - cargo build --target wasm32-unknown-unknown -p $1 - wasm-tools component new target/wasm32-unknown-unknown/debug/$(subst -,_,$1).wasm -o ${COMPONENTS_DIR}/$1/$1.debug.wasm - -endif - -${COMPONENTS_DIR}/$1/README.md: components/$1/README.md - @mkdir -p ${COMPONENTS_DIR}/$1 - @cp components/$1/README.md ${COMPONENTS_DIR}/$1/README.md - -endef - -$(foreach component,$(COMPONENTS),$(eval $(call BUILD_COMPONENT,$(component)))) - -${COMPONENTS_DIR}/interface.wasm: wit/deps README.md | $(call tool,wkg) - @mkdir -p ${COMPONENTS_DIR} - wkg build -o ${COMPONENTS_DIR}/interface.wasm - @cp README.md ${COMPONENTS_DIR}/README.md - -# directories with a wkg.toml, each fetches the dependencies of its wit directory into wit/deps, -# e.g. `.` and `components` -WKG_DIRS := $(sort $(patsubst ./%,%,$(patsubst %/,%,$(dir $(shell find . -name wkg.toml -not -path './target/*' -not -path '*/deps/*'))))) - -# the wit/deps directory of a directory with a wkg.toml, e.g. `wit/deps` for `.` -wit_deps = $(patsubst ./%,%,$(1)/wit/deps) - -WIT_DEPS := $(foreach dir,$(WKG_DIRS),$(call wit_deps,$(dir))) - -.PHONY: wit -wit: $(WIT_DEPS) - -define FETCH_WIT - -# a package overridden with a local path, e.g. `{ path = "../wit" }`, has its dependencies fetched first -$(call wit_deps,$1): $1/wkg.toml $1/wkg.lock $(shell find $1/wit -type f -name "*.wit" -not -path "*/deps/*") $(foreach path,$(shell sed -n 's/.*path *= *"\(.*\)".*/\1/p' $1/wkg.toml),$(call relpath,$1/$(path))/deps) | $(call tool,wkg) - $(if $(filter .,$1),,cd $1 && )wkg fetch - -endef - -$(foreach dir,$(WKG_DIRS),$(eval $(call FETCH_WIT,$(dir)))) - -# sign published components with cosign, `SIGN=false` to push without signing, e.g. to a local registry -SIGN ?= true - -# the files that can be published, e.g. gate.wasm, published from target/components/gate/gate.wasm -PUBLISH_FILES := interface.wasm $(foreach component,$(filter-out dep-% test-%,$(COMPONENTS)),$(component).wasm $(component).debug.wasm) - -.PHONY: publish ## Publish each component in the target/components directory -publish: $(addprefix publish-,$(PUBLISH_FILES)) - -.PHONY: $(addprefix publish-,$(PUBLISH_FILES)) -$(addprefix publish-,$(PUBLISH_FILES)): publish-%: | $(call tool,wkg) -ifndef VERSION - $(error VERSION is undefined) -endif -ifndef REPOSITORY - $(error REPOSITORY is undefined) -endif - @$(eval FILE := $(@:publish-%=%)) - @$(eval COMPONENT := $(patsubst %.wasm,%,$(patsubst %.debug.wasm,%,$(FILE)))) -# components are in a directory of their own, the interface is not, e.g. gate/gate.wasm and interface.wasm - @$(eval COMPONENT_FILE := $(if $(filter interface.wasm,$(FILE)),$(FILE),$(COMPONENT)/$(FILE))) - @$(eval README := ${COMPONENTS_DIR}/$(dir $(COMPONENT_FILE))README.md) - @$(eval TITLE := $(subst /,:,$(GITHUB_REPOSITORY))$(if $(filter interface,$(COMPONENT)),,-$(COMPONENT))$(if $(filter %.debug.wasm,$(FILE)), (debug))) - @$(eval DESCRIPTION := $(shell head -n 3 "$(README)" | tail -n 1)) - @$(eval COMMIT := $(shell git rev-parse HEAD)) - @$(eval README_DIR := $(if $(wildcard components/$(COMPONENT)/README.md),/components/$(COMPONENT))) - @$(eval URL := https://github.com/${GITHUB_REPOSITORY}/tree/${COMMIT}${README_DIR}) - @$(eval REVISION := ${COMMIT}$(shell git diff --quiet HEAD || echo "+dirty")) - @$(eval COMPONENT_VERSION := $(if $(filter %.debug.wasm,$(FILE)),${VERSION}+debug,${VERSION})) - @$(eval TAG := $(patsubst v%,%,$(subst +,_,$(COMPONENT_VERSION)))) - @$(eval IMAGE := $(if $(filter interface.wasm,$(FILE)),${REPOSITORY}:${TAG},${REPOSITORY}/${COMPONENT}:${TAG})) - - @echo "::group::${FILE} -> ${IMAGE}" - @set -o pipefail ; \ - DIGEST=$$( \ - wkg oci push \ - --annotation "org.opencontainers.image.title=${TITLE}" \ - --annotation "org.opencontainers.image.description=${DESCRIPTION}" \ - --annotation "org.opencontainers.image.version=${COMPONENT_VERSION}" \ - --annotation "org.opencontainers.image.url=${URL}" \ - --annotation "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}.git" \ - --annotation "org.opencontainers.image.revision=${REVISION}" \ - --annotation "org.opencontainers.image.licenses=Apache-2.0" \ - "${IMAGE}" \ - "${COMPONENTS_DIR}/${COMPONENT_FILE}" \ - 2>&1 \ - | tee /dev/stderr \ - | grep -o 'sha256:[a-f0-9]\{64\}' \ - ) && \ - $(if $(filter true,$(SIGN)),cosign sign --yes "${IMAGE}@$${DIGEST}",echo "Not signing ${IMAGE}@$${DIGEST}, SIGN=${SIGN}") - @echo "::endgroup::" diff --git a/make/components.mk b/make/components.mk new file mode 100644 index 0000000..a82da73 --- /dev/null +++ b/make/components.mk @@ -0,0 +1,83 @@ +COMPONENTS_DIR := target/components + +# the WIT packages of components/wit/deps, encoded where `wac compose` looks for a package a +# composition names without instantiating it, e.g. `import stderr: wasi:cli/stderr@0.3.0;` +WAC_DEPS_DIR := target/wac-deps + +COMPONENTS = $(sort $(foreach file,$(wildcard $(addprefix components/*/,wit/*.constants.wit *.properties *.wac *.wkg Cargo.toml)),$(word 2,$(subst /, ,$(file))))) + +.PHONY: clean-components +clean-components: clean-wit + rm -rf ${COMPONENTS_DIR} ${WAC_DEPS_DIR} + +.PHONY: components +components: ${COMPONENTS_DIR}/interface.wasm $(foreach component,$(COMPONENTS),${COMPONENTS_DIR}/$(component)/$(component).wasm ${COMPONENTS_DIR}/$(component)/$(component).debug.wasm) + +define BUILD_COMPONENT + +.PHONY: components/$1 +components/$1: ${COMPONENTS_DIR}/$1/$1.wasm ${COMPONENTS_DIR}/$1/$1.debug.wasm + +ifneq ($(wildcard components/$1/wit/$1.constants.wit),) + +${COMPONENTS_DIR}/$1/$1.wasm: components/$1/wit/deps ${COMPONENTS_DIR}/$1/README.md | $(call tool,componentized-constants-cli) + $(CONSTANTS) --wit components/$1/wit -o ${COMPONENTS_DIR}/$1/$1.wasm + +${COMPONENTS_DIR}/$1/$1.debug.wasm: components/$1/wit/deps ${COMPONENTS_DIR}/$1/README.md | $(call tool,componentized-constants-cli) + $(CONSTANTS) --wit components/$1/wit -o ${COMPONENTS_DIR}/$1/$1.debug.wasm + +else ifneq ($(wildcard components/$1/$1.properties),) + +${COMPONENTS_DIR}/$1/$1.wasm: components/$1/$1.properties ${COMPONENTS_DIR}/$1/README.md | $(call tool,componentized-static-config-cli) + $(STATIC_CONFIG) -f components/$1/$1.properties -o ${COMPONENTS_DIR}/$1/$1.wasm + +${COMPONENTS_DIR}/$1/$1.debug.wasm: components/$1/$1.properties ${COMPONENTS_DIR}/$1/README.md | $(call tool,componentized-static-config-cli) + $(STATIC_CONFIG) -f components/$1/$1.properties -o ${COMPONENTS_DIR}/$1/$1.debug.wasm + +else ifneq ($(wildcard components/$1/$1.wac),) + +# the local packages the composition instantiates, e.g. `new local:latch-n2 { ... }` +WAC_DEPS_$1 := $$(shell grep -v '^\s*//' components/$1/$1.wac | grep -oE 'local:[a-z0-9-]+' | sed 's/^local://' | sort -u) + +${COMPONENTS_DIR}/$1/$1.wasm: components/$1/$1.wac $$(foreach component,$$(WAC_DEPS_$1),$${COMPONENTS_DIR}/$$(component)/$$(component).wasm) ${COMPONENTS_DIR}/$1/README.md ${WAC_DEPS_DIR} | $(call tool,wac-cli) + $(WAC) compose --deps-dir ${WAC_DEPS_DIR} $$(foreach component,$$(WAC_DEPS_$1),-d local:$$(component)=$${COMPONENTS_DIR}/$$(component)/$$(component).wasm) -o ${COMPONENTS_DIR}/$1/$1.wasm components/$1/$1.wac + +${COMPONENTS_DIR}/$1/$1.debug.wasm: components/$1/$1.wac $$(foreach component,$$(WAC_DEPS_$1),$${COMPONENTS_DIR}/$$(component)/$$(component).debug.wasm) ${COMPONENTS_DIR}/$1/README.md ${WAC_DEPS_DIR} | $(call tool,wac-cli) + $(WAC) compose --deps-dir ${WAC_DEPS_DIR} $$(foreach component,$$(WAC_DEPS_$1),-d local:$$(component)=$${COMPONENTS_DIR}/$$(component)/$$(component).debug.wasm) -o ${COMPONENTS_DIR}/$1/$1.debug.wasm components/$1/$1.wac + +else ifneq ($(wildcard components/$1/$1.wkg),) + +${COMPONENTS_DIR}/$1/$1.wasm: components/$1/$1.wkg ${COMPONENTS_DIR}/$1/README.md | $(call tool,wkg) + $(WKG) oci pull $(shell cat components/$1/$1.wkg 2> /dev/null | head -1) -o ${COMPONENTS_DIR}/$1/$1.wasm + +${COMPONENTS_DIR}/$1/$1.debug.wasm: components/$1/$1.wkg ${COMPONENTS_DIR}/$1/README.md | $(call tool,wkg) + $(WKG) oci pull $(shell cat components/$1/$1.wkg 2> /dev/null | tail -1 2> /dev/null) -o ${COMPONENTS_DIR}/$1/$1.debug.wasm + +# cargo is checked last, other strategies may have a Cargo.toml for tests of non-rust sources +else ifneq ($(wildcard components/$1/Cargo.toml),) + +${COMPONENTS_DIR}/$1/$1.wasm: Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f) $(shell find crates -type f 2> /dev/null) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wasm-tools) + cargo build -p $1 --target wasm32-unknown-unknown --release + $(WASM_TOOLS) component new target/wasm32-unknown-unknown/release/$(subst -,_,$1).wasm -o ${COMPONENTS_DIR}/$1/$1.wasm + +${COMPONENTS_DIR}/$1/$1.debug.wasm: Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f) $(shell find crates -type f 2> /dev/null) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wasm-tools) + cargo build -p $1 --target wasm32-unknown-unknown --profile component-debug + $(WASM_TOOLS) component new target/wasm32-unknown-unknown/component-debug/$(subst -,_,$1).wasm -o ${COMPONENTS_DIR}/$1/$1.debug.wasm + +endif + +${COMPONENTS_DIR}/$1/README.md: components/$1/README.md + @mkdir -p ${COMPONENTS_DIR}/$1 + @cp components/$1/README.md ${COMPONENTS_DIR}/$1/README.md + +endef + +$(foreach component,$(COMPONENTS),$(eval $(call BUILD_COMPONENT,$(component)))) + +${WAC_DEPS_DIR}: components/wit/deps scripts/wac-deps.sh | $(call tool,wasm-tools) + WASM_TOOLS=$(WASM_TOOLS) scripts/wac-deps.sh components/wit/deps ${WAC_DEPS_DIR} + +${COMPONENTS_DIR}/interface.wasm: wit/deps README.md | $(call tool,wkg) + @mkdir -p ${COMPONENTS_DIR} + $(WKG) build -o ${COMPONENTS_DIR}/interface.wasm + @cp README.md ${COMPONENTS_DIR}/README.md diff --git a/make/pinned.jq b/make/pinned.jq new file mode 100644 index 0000000..78fc559 --- /dev/null +++ b/make/pinned.jq @@ -0,0 +1,6 @@ +# generates target/tools/pinned.mk from `cargo metadata --no-deps` of tools/Cargo.toml, see make/tools.mk +.packages[0] as $tools +| ($tools.dependencies[].name | select($tools.metadata.bins[.] == null) | error("\(.) is missing from [package.metadata.bins] in tools/Cargo.toml")), + "TOOLS := \([$tools.dependencies[].name] | join(" "))", + "TOOLS_PINNED := \([$tools.dependencies[] | "\(.name)@\(.req | ltrimstr("="))"] | join(" "))", + ($tools.metadata.bins[][] | "\(ascii_upcase | gsub("-"; "_")) := $(TOOLS_BIN)/\(.)") diff --git a/make/publish.mk b/make/publish.mk new file mode 100644 index 0000000..2b3a472 --- /dev/null +++ b/make/publish.mk @@ -0,0 +1,15 @@ +# sign published components with cosign, `SIGN=false` to push without signing, e.g. to a local registry +SIGN ?= true +# append each published file and its image to this file, e.g. `gate.wasm ghcr.io/componentized/http/gate:0.1.0@sha256:...` +PUBLISH_LOG ?= + +# the files that can be published, e.g. gate.wasm, published from target/components/gate/gate.wasm +PUBLISH_FILES := interface.wasm $(foreach component,$(filter-out dep-% test-%,$(COMPONENTS)),$(component).wasm $(component).debug.wasm) + +.PHONY: publish ## Publish each component in the target/components directory +publish: $(addprefix publish-,$(PUBLISH_FILES)) + +.PHONY: $(addprefix publish-,$(PUBLISH_FILES)) +$(addprefix publish-,$(PUBLISH_FILES)): publish-%: | $(call tool,wkg) + @VERSION="$(VERSION)" REPOSITORY="$(REPOSITORY)" COMPONENTS_DIR="$(COMPONENTS_DIR)" SIGN="$(SIGN)" PUBLISH_LOG="$(PUBLISH_LOG)" \ + scripts/publish.sh $* diff --git a/make/tools.mk b/make/tools.mk new file mode 100644 index 0000000..f408f38 --- /dev/null +++ b/make/tools.mk @@ -0,0 +1,53 @@ +TOOLS_DIR := target/tools/$(shell rustc --print host-tuple) +# absolute, tools also run from other directories, e.g. `cd components && wkg fetch` +TOOLS_BIN := $(abspath $(TOOLS_DIR))/bin +# for the scripts and tests the recipes run, they look up the tools on the PATH +export PATH := $(TOOLS_BIN):$(PATH) + +# the tools pinned in tools/Cargo.toml, generated when it changes, make restarts after generating it. +# Defines TOOLS, TOOLS_PINNED with the version of each tool, e.g. `wkg@0.16.1`, and a variable for +# each binary a tool installs, e.g. `WKG := $(TOOLS_BIN)/wkg`. Recipes run the pinned tools by path, +# make 3.81 runs a simple command without a shell and looks it up on its own PATH, before the export +# above, which would find a tool already on the system. +TOOLS_MK := target/tools/pinned.mk +include $(TOOLS_MK) + +# cargo binstall downloads prebuilt binaries, without it the tools are built with cargo install +CARGO_INSTALL := $(if $(shell command -v cargo-binstall 2> /dev/null),cargo binstall --no-confirm --disable-telemetry,cargo install) + +.PHONY: clean-tools +clean-tools: + rm -rf ${TOOLS_DIR} + +# each tool is a dependency of the tools package, the binaries it installs are listed in +# [package.metadata.bins] +$(TOOLS_MK): tools/Cargo.toml make/pinned.jq + @mkdir -p $(@D) + @set -o pipefail; cargo metadata --manifest-path tools/Cargo.toml --no-deps --format-version 1 | jq -r -f make/pinned.jq > $@.tmp + @mv $@.tmp $@ + +tool_version = $(patsubst $(1)@%,%,$(filter $(1)@%,$(TOOLS_PINNED))) +# a stamp naming the version of a tool installed in $(TOOLS_DIR)/bin, e.g. `wkg@0.16.1`, the binary +# does not say which version it is. Bumping the pinned version names a stamp that does not exist yet, +# so the tool is installed again. +tool = $(TOOLS_DIR)/.installed/$(1)@$(call tool_version,$(1)) + +.PHONY: tools ## Install the cli tools pinned in tools/Cargo.toml +tools: $(foreach name,$(TOOLS),$(call tool,$(name))) + +.PHONY: tools-path ## Print the directory of the installed tools for this platform, to add to the PATH +tools-path: + @echo $(TOOLS_BIN) + +define INSTALL_TOOL + +$(call tool,$1): + $(CARGO_INSTALL) --locked --root $(TOOLS_DIR) --version $(call tool_version,$1) $1 + @mkdir -p $$(@D) + @# only the installed version has a stamp, so going back to a previous version installs it again + @rm -f $$(@D)/$1@* + @touch $$@ + +endef + +$(foreach name,$(TOOLS),$(eval $(call INSTALL_TOOL,$(name)))) diff --git a/make/wit.mk b/make/wit.mk new file mode 100644 index 0000000..ac4971c --- /dev/null +++ b/make/wit.mk @@ -0,0 +1,28 @@ +# a path relative to the root of the repository, e.g. `wit` for `components/../wit` +relpath = $(if $(filter $(CURDIR),$(abspath $(1))),.,$(patsubst $(CURDIR)/%,%,$(abspath $(1)))) + +# directories with a wkg.toml, each fetches the dependencies of its wit directory into wit/deps, +# e.g. `.` and `components` +WKG_DIRS := $(sort $(patsubst ./%,%,$(patsubst %/,%,$(dir $(shell find . -name wkg.toml -not -path './target/*' -not -path '*/deps/*'))))) + +# the wit/deps directory of a directory with a wkg.toml, e.g. `wit/deps` for `.` +wit_deps = $(patsubst ./%,%,$(1)/wit/deps) + +WIT_DEPS := $(foreach dir,$(WKG_DIRS),$(call wit_deps,$(dir))) + +.PHONY: clean-wit ## Remove the fetched wit dependencies, fetched again by `make wit` +clean-wit: + rm -rf $(WIT_DEPS) + +.PHONY: wit +wit: $(WIT_DEPS) + +define FETCH_WIT + +# a package overridden with a local path, e.g. `{ path = "../wit" }`, has its dependencies fetched first +$(call wit_deps,$1): $1/wkg.toml $1/wkg.lock $(shell find $1/wit -type f -name "*.wit" -not -path "*/deps/*") $(foreach path,$(shell sed -n 's/.*path *= *"\(.*\)".*/\1/p' $1/wkg.toml),$(call relpath,$1/$(path))/deps) | $(call tool,wkg) + $(if $(filter .,$1),,cd $1 && )$(WKG) fetch + +endef + +$(foreach dir,$(WKG_DIRS),$(eval $(call FETCH_WIT,$(dir)))) diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index a3a9666..6d40039 100755 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -1,13 +1,11 @@ #!/usr/bin/env bash -# Bump the version of the wit interface package, and of the crates. +# Bump the version of the wit interface package. # # scripts/bump-version.sh # # Updates the package declaration and every reference to the package in tracked files, then -# refreshes the generated wit dependencies. The crates share the interface's version: the -# workspace version the crates inherit, and the workspace's requirement on the library, move to the -# new version too. Items whose `@since` names an unreleased (prerelease) +# refreshes the generated wit dependencies. Items whose `@since` names an unreleased (prerelease) # version move to the new version, since they were never published under the old one. Items # released under the old version keep their `@since`. # @@ -19,8 +17,6 @@ set -euo pipefail cd "$(dirname "$0")/.." PACKAGE="${PACKAGE:-componentized:$(basename $(git rev-parse --show-toplevel))}" -# the library crate, the workspace's requirement on it moves to the new version -LIBRARY="${LIBRARY:-componentized-constants}" SEMVER='^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$' new="${1:-}" @@ -70,7 +66,7 @@ fi old_re="${old//./\\.}" # references to the package or one of its interfaces, an interface named for a keyword is escaped -# with `%`, e.g. `componentized:constants/%u8@0.1.0` +# with `%`, e.g. `componentized:sockets/latch@0.1.0-dev` ref_re="${PACKAGE}(/%?[a-z0-9-]+)?" # the fetched wit dependencies and the wkg.lock files are left to `make wit`, wkg replaces the # dependencies and updates the locks for the new version @@ -90,16 +86,10 @@ if [[ "$old" == *-* ]]; then done fi -# the version in the [workspace.package] section, inherited by the crates -perl -pi -e 'if (/^\[workspace\.package\]/ .. /^\[(?!workspace\.package\])/) { s/^version = "[^"]*"/version = "'"${new}"'"/ }' Cargo.toml -echo "updated the workspace version in Cargo.toml" -perl -pi -e 's/^(\Q'"${LIBRARY}"'\E = \{.*\bversion = ")[^"]*(")/${1}'"${new}"'${2}/' Cargo.toml -echo "updated the ${LIBRARY} requirement in Cargo.toml" - perl -pi -e 's{^(\s+)\Q'"${workflow_default}"'\E$}{${1}'"${workflow_default/\"${old}\"/\"${new}\"}"'}' "$workflow" echo "updated the default version in ${workflow}" # regenerate the wit dependencies for the new version make wit components test -echo "bumped ${PACKAGE} from ${old} to ${new}" \ No newline at end of file +echo "bumped ${PACKAGE} from ${old} to ${new}" diff --git a/scripts/init-devcontainer.sh b/scripts/init-devcontainer.sh new file mode 100755 index 0000000..59f0186 --- /dev/null +++ b/scripts/init-devcontainer.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +# Setup script run once for new devcontainers to init the environment. + +set -euo pipefail + +curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash + +cargo check + +echo "export \"PATH=$(make -s tools-path):${PATH}\"" >> ~/.bashrc +make tools diff --git a/scripts/publish.sh b/scripts/publish.sh new file mode 100755 index 0000000..669c112 --- /dev/null +++ b/scripts/publish.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +# Publish a built component to an OCI registry, and sign it with cosign. +# +# VERSION= REPOSITORY= scripts/publish.sh +# +# The file is one of the files built into the components directory, components are in a directory +# of their own, the interface is not, e.g. gate.wasm from target/components/gate/gate.wasm and +# interface.wasm from target/components/interface.wasm. The interface is published to the +# repository, each component to a repository of its own under it, a debug build is tagged with a +# `_debug` suffix. +# +# interface.wasm -> ${REPOSITORY}:0.1.0 +# gate.wasm -> ${REPOSITORY}/gate:0.1.0 +# gate.debug.wasm -> ${REPOSITORY}/gate:0.1.0_debug +# +# VERSION the version to publish, a leading `v` is dropped from the tag, e.g. v0.1.0 +# REPOSITORY the repository to publish to, e.g. ghcr.io/componentized/http +# GITHUB_REPOSITORY the GitHub repository the components are built from, e.g. componentized/http +# COMPONENTS_DIR the directory the components are built into, defaults to target/components +# SIGN sign the published components with cosign, `false` to push without signing, +# e.g. to a local registry +# PUBLISH_LOG append the published file and its image to this file, e.g. +# `gate.wasm ghcr.io/componentized/http/gate:0.1.0@sha256:...` + +set -euo pipefail + +cd "$(dirname "$0")/.." + +file="${1:-}" +if [[ -z "$file" ]]; then + echo "usage: $0 , e.g. interface.wasm or gate.wasm" >&2 + exit 1 +fi +: "${VERSION:?VERSION is undefined}" +: "${REPOSITORY:?REPOSITORY is undefined}" +GITHUB_REPOSITORY="${GITHUB_REPOSITORY:-componentized/$(basename "$(git rev-parse --show-toplevel)")}" +COMPONENTS_DIR="${COMPONENTS_DIR:-target/components}" +SIGN="${SIGN:-true}" +PUBLISH_LOG="${PUBLISH_LOG:-}" + +component="${file%.wasm}" +component="${component%.debug}" +debug="" +[[ "$file" == *.debug.wasm ]] && debug=true + +if [[ "$file" == interface.wasm ]]; then + component_file="$file" + title="${GITHUB_REPOSITORY/\//:}" +else + component_file="${component}/${file}" + title="${GITHUB_REPOSITORY/\//:}-${component}" +fi +[[ -n "$debug" ]] && title="${title} (debug)" + +# the description is the line following the title in the readme +readme="${COMPONENTS_DIR}/$(dirname "$component_file")/README.md" +description=$(sed -n 3p "$readme") + +commit=$(git rev-parse HEAD) +revision="$commit" +git diff --quiet HEAD || revision="${commit}+dirty" +url="https://github.com/${GITHUB_REPOSITORY}/tree/${commit}" +[[ -f "components/${component}/README.md" ]] && url="${url}/components/${component}" + +component_version="$VERSION" +[[ -n "$debug" ]] && component_version="${VERSION}+debug" +tag="${component_version#v}" +tag="${tag//+/_}" + +if [[ "$file" == interface.wasm ]]; then + image="${REPOSITORY}:${tag}" +else + image="${REPOSITORY}/${component}:${tag}" +fi + +echo "::group::${file} -> ${image}" + +digest=$( + wkg oci push \ + --annotation "org.opencontainers.image.title=${title}" \ + --annotation "org.opencontainers.image.description=${description}" \ + --annotation "org.opencontainers.image.version=${component_version}" \ + --annotation "org.opencontainers.image.url=${url}" \ + --annotation "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}.git" \ + --annotation "org.opencontainers.image.revision=${revision}" \ + --annotation "org.opencontainers.image.licenses=Apache-2.0" \ + "$image" \ + "${COMPONENTS_DIR}/${component_file}" \ + 2>&1 \ + | tee /dev/stderr \ + | grep -o 'sha256:[a-f0-9]\{64\}' +) + +if [[ -n "$PUBLISH_LOG" ]]; then + size=$(wc -c < "${COMPONENTS_DIR}/${component_file}" | tr -d ' ') + echo "${file} ${size} ${image}@${digest}" >> "$PUBLISH_LOG" +fi + +if [[ "$SIGN" == true ]]; then + cosign sign --yes "${image}@${digest}" +else + echo "Not signing ${image}@${digest}, SIGN=${SIGN}" +fi + +echo "::endgroup::" diff --git a/scripts/wac-deps.sh b/scripts/wac-deps.sh new file mode 100755 index 0000000..7e10af8 --- /dev/null +++ b/scripts/wac-deps.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +# Encode each WIT package of a wit/deps directory where `wac compose --deps-dir` looks for it, so a +# composition can name the interfaces of a package it does not instantiate, e.g. +# `import stderr: wasi:cli/stderr@0.3.0;`. +# +# WASM_TOOLS= scripts/wac-deps.sh +# +# wac looks for a versioned package at `//.wasm`, e.g. +# +# wit/deps/wasi-cli-0.3.0 -> /wasi/cli/0.3.0.wasm +# +# Each package is encoded with the other packages as its dependencies. + +set -euo pipefail + +deps=$1 +out=$2 +wasm_tools=${WASM_TOOLS:-wasm-tools} + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +rm -rf "$out" +mkdir -p "$out" + +for package in "$deps"/*/; do + package=${package%/} + rm -rf "$work/package" + mkdir -p "$work/package/deps" + cp "$package"/*.wit "$work/package/" + for dep in "$deps"/*/; do + dep=${dep%/} + [[ "$dep" == "$package" ]] || cp -R "$dep" "$work/package/deps/" + done + "$wasm_tools" component wit "$work/package" --wasm -o "$work/package.wasm" + + # e.g. `package wasi:cli@0.3.0;` + id=$("$wasm_tools" component wit "$work/package.wasm" | sed -n 's/^package \(.*\);$/\1/p' | head -n 1) + namespace=${id%%:*} + name=${id#*:} + name=${name%%@*} + version=${id#*@} + mkdir -p "$out/$namespace/$name" + cp "$work/package.wasm" "$out/$namespace/$name/$version.wasm" +done diff --git a/tools/Cargo.toml b/tools/Cargo.toml index fe3ce73..7d6bdd2 100644 --- a/tools/Cargo.toml +++ b/tools/Cargo.toml @@ -12,6 +12,16 @@ publish = false [lib] path = "lib.rs" +# the binaries each tool installs, the Makefile runs each by path from a variable named after it, +# e.g. `$(STATIC_CONFIG)` for `static-config` +[package.metadata.bins] +componentized-constants-cli = ["constants"] +componentized-static-config-cli = ["static-config"] +wac-cli = ["wac"] +wasm-tools = ["wasm-tools"] +wasmtime-cli = ["wasmtime"] +wkg = ["wkg"] + [dependencies] componentized-constants-cli = "=0.1.1" componentized-static-config-cli = "=0.3.0"