diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39533206..e88328e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: run: make lint # --------------------------------------------------------------------------- - # TIDY (module files + BUILD files in sync) + # TIDY (module files + BUILD files + generated proto in sync) # --------------------------------------------------------------------------- tidy: name: Tidy @@ -52,6 +52,9 @@ jobs: - name: Check BUILD files are up to date run: make check-gazelle + - name: Check generated proto files are up to date + run: make check-proto + # --------------------------------------------------------------------------- # BUILD AND UNIT TESTS # --------------------------------------------------------------------------- diff --git a/.protocversion b/.protocversion new file mode 100644 index 00000000..08a68b85 --- /dev/null +++ b/.protocversion @@ -0,0 +1 @@ +29.3 diff --git a/MODULE.bazel b/MODULE.bazel index d44b8457..9492e814 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -49,6 +49,7 @@ use_repo( "com_github_uber_go_tally_v4", "in_gopkg_yaml_v3", "org_golang_google_grpc", + "org_golang_google_grpc_cmd_protoc_gen_go_grpc", "org_golang_google_protobuf", "org_golang_x_oauth2", "org_uber_go_fx", diff --git a/Makefile b/Makefile index 7e2411de..79a66216 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,16 @@ # Bazel wrapper BAZEL = ./tool/bazel +# protoc wrapper (hermetic; pinned by .protocversion) +PROTOC = ./tool/protoc + +# protoc plugins (hermetic; versions pinned by the `tool` directives in go.mod). +# Passed explicitly so protoc never resolves a plugin from the host $PATH. +PROTOC_PLUGINS = \ + --plugin=protoc-gen-go=$(CURDIR)/tool/protoc-gen-go \ + --plugin=protoc-gen-go-grpc=$(CURDIR)/tool/protoc-gen-go-grpc \ + --plugin=protoc-gen-yarpc-go=$(CURDIR)/tool/protoc-gen-yarpc-go + # Docker Compose wrapper COMPOSE = docker-compose @@ -40,7 +50,7 @@ define assert_clean fi endef -.PHONY: build build-all-linux build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-gateway-linux build-stovepipe-orchestrator-linux check-gazelle check-mocks check-tidy clean clean-proto deps e2e-test fmt gazelle integration-test integration-test-submitqueue-consumer integration-test-extensions integration-test-submitqueue-gateway integration-test-submitqueue-orchestrator license-fix lint lint-fmt lint-license local-submitqueue-clean local-submitqueue-gateway-start local-submitqueue-gateway-stop local-init-submitqueue-schemas local-init-stovepipe-queue-schema local-submitqueue-logs local-submitqueue-orchestrator-start local-submitqueue-orchestrator-stop local-submitqueue-ps local-submitqueue-restart local-submitqueue-start local-stop local-stovepipe-gateway-start local-stovepipe-orchestrator-start local-stovepipe-start mocks proto query-deps query-targets run-client-submitqueue-gateway run-client-submitqueue-orchestrator run-client-stovepipe-gateway run-client-stovepipe-orchestrator run-queue-admin test test-no-cache tidy tidy-bazel tidy-go help +.PHONY: build build-all-linux build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-gateway-linux build-stovepipe-orchestrator-linux check-gazelle check-mocks check-proto check-tidy clean clean-proto deps e2e-test fmt gazelle integration-test integration-test-submitqueue-consumer integration-test-extensions integration-test-submitqueue-gateway integration-test-submitqueue-orchestrator license-fix lint lint-fmt lint-license local-submitqueue-clean local-submitqueue-gateway-start local-submitqueue-gateway-stop local-init-submitqueue-schemas local-init-stovepipe-queue-schema local-submitqueue-logs local-submitqueue-orchestrator-start local-submitqueue-orchestrator-stop local-submitqueue-ps local-submitqueue-restart local-submitqueue-start local-stop local-stovepipe-gateway-start local-stovepipe-orchestrator-start local-stovepipe-start mocks proto query-deps query-targets run-client-submitqueue-gateway run-client-submitqueue-orchestrator run-client-stovepipe-gateway run-client-stovepipe-orchestrator run-queue-admin test test-no-cache tidy tidy-bazel tidy-go help build: ## Build all services and examples @@ -94,6 +104,10 @@ check-mocks: mocks ## Check mock files are up to date $(call assert_clean,make mocks) @echo "Mock files are up to date." +check-proto: proto ## Check generated proto files are up to date + $(call assert_clean,make proto) + @echo "Proto files are up to date." + check-tidy: tidy ## Check that go.mod and MODULE.bazel are tidy $(call assert_clean,make tidy) @echo "Module files are up to date." @@ -339,22 +353,26 @@ mocks: ## Generate mock files using mockgen proto: ## Generate protobuf files from .proto definitions @echo "Generating protobuf files with protoc..." - @protoc --go_out=submitqueue/gateway/protopb --go_opt=paths=source_relative \ + @$(PROTOC) $(PROTOC_PLUGINS) --go_out=submitqueue/gateway/protopb --go_opt=paths=source_relative \ --go-grpc_out=submitqueue/gateway/protopb --go-grpc_opt=paths=source_relative \ --yarpc-go_out=submitqueue/gateway/protopb --yarpc-go_opt=paths=source_relative \ --proto_path=submitqueue/gateway/proto submitqueue/gateway/proto/gateway.proto - @protoc --go_out=submitqueue/orchestrator/protopb --go_opt=paths=source_relative \ + @$(PROTOC) $(PROTOC_PLUGINS) --go_out=submitqueue/orchestrator/protopb --go_opt=paths=source_relative \ --go-grpc_out=submitqueue/orchestrator/protopb --go-grpc_opt=paths=source_relative \ --yarpc-go_out=submitqueue/orchestrator/protopb --yarpc-go_opt=paths=source_relative \ --proto_path=submitqueue/orchestrator/proto submitqueue/orchestrator/proto/orchestrator.proto - @protoc --go_out=stovepipe/gateway/protopb --go_opt=paths=source_relative \ + @$(PROTOC) $(PROTOC_PLUGINS) --go_out=stovepipe/gateway/protopb --go_opt=paths=source_relative \ --go-grpc_out=stovepipe/gateway/protopb --go-grpc_opt=paths=source_relative \ --yarpc-go_out=stovepipe/gateway/protopb --yarpc-go_opt=paths=source_relative \ --proto_path=stovepipe/gateway/proto stovepipe/gateway/proto/gateway.proto - @protoc --go_out=stovepipe/orchestrator/protopb --go_opt=paths=source_relative \ + @$(PROTOC) $(PROTOC_PLUGINS) --go_out=stovepipe/orchestrator/protopb --go_opt=paths=source_relative \ --go-grpc_out=stovepipe/orchestrator/protopb --go-grpc_opt=paths=source_relative \ --yarpc-go_out=stovepipe/orchestrator/protopb --yarpc-go_opt=paths=source_relative \ --proto_path=stovepipe/orchestrator/proto stovepipe/orchestrator/proto/orchestrator.proto + @echo "Formatting generated files with goimports..." + @go run golang.org/x/tools/cmd/goimports@$(GOIMPORTS_VERSION) -w \ + submitqueue/gateway/protopb submitqueue/orchestrator/protopb \ + stovepipe/gateway/protopb stovepipe/orchestrator/protopb @echo "Protobuf files generated successfully!" # Bazel query helpers diff --git a/doc/howto/DEVELOPMENT.md b/doc/howto/DEVELOPMENT.md index 9fccc2bd..281f6192 100644 --- a/doc/howto/DEVELOPMENT.md +++ b/doc/howto/DEVELOPMENT.md @@ -2,7 +2,7 @@ ## Prerequisites -- **Go 1.24+** — needed for `gopls`, `go mod`, and installing protoc plugins. Download from [go.dev/dl](https://go.dev/dl/). Note: Bazel manages its own Go toolchain for builds, but a local Go installation is required for editor tooling and dependency management. +- **Go 1.24+** — needed for `gopls`, `go mod`, and running the hermetic protoc plugins (via `go tool`). Download from [go.dev/dl](https://go.dev/dl/). Note: Bazel manages its own Go toolchain for builds, but a local Go installation is required for editor tooling and dependency management. - **Docker** and **Docker Compose** — for integration and e2e tests, and for running services locally. - **direnv** (recommended) — automatically loads `.envrc` so you can use `bazel` directly instead of `./tool/bazel`. @@ -85,15 +85,18 @@ GoLand works with Go modules automatically. Open the project root and GoLand wil ## Optional Tools ```bash -# macOS -brew install protobuf grpcurl - -# Go protoc plugins (only if modifying .proto files) -go install google.golang.org/protobuf/cmd/protoc-gen-go@latest -go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest -go install go.uber.org/yarpc/encoding/protobuf/protoc-gen-yarpc-go@latest +# macOS — grpcurl for poking at running services (optional) +brew install grpcurl ``` +Proto generation is fully hermetic and needs no manual installs: `make proto` +downloads a pinned `protoc` via `./tool/protoc` (see `.protocversion`) and runs +the `protoc-gen-go`, `protoc-gen-go-grpc`, and `protoc-gen-yarpc-go` plugins +at the versions pinned by the `tool` directives in `go.mod` (via `go tool`). It +needs a Go toolchain plus the standard shell utilities `./tool/protoc` shells +out to — `bash`, `curl`, `unzip`, and a SHA-256 tool (`sha256sum` or `shasum`) — +and network access on the first run to fetch protoc and the plugin modules. + ## Common Make Targets | Target | Description | @@ -138,8 +141,9 @@ See [TESTING.md](TESTING.md) for the full testing guide, including integration a ## Troubleshooting **Proto generation fails:** -- Ensure all three protoc plugins are installed (see Optional Tools above) -- Check that `protoc` is in your PATH: `which protoc` +- `make proto` is hermetic — it needs no host `protoc` or plugins, only a Go toolchain, the shell utilities `./tool/protoc` uses (`bash`, `curl`, `unzip`, `sha256sum`/`shasum`), and (on the first run) network access to download pinned `protoc` and the plugin modules. +- To bump versions: edit `.protocversion` (and add the new platform checksums in `./tool/protoc`) for protoc, or `go get -tool @` followed by `make tidy` for a plugin. +- Run `make check-proto` to confirm the committed generated files match a fresh `make proto`. **Build fails after proto changes:** - Run `make proto` to regenerate proto files diff --git a/go.mod b/go.mod index 12ec655d..e3987093 100644 --- a/go.mod +++ b/go.mod @@ -56,6 +56,13 @@ require ( golang.org/x/tools v0.41.0 // indirect golang.org/x/tools/go/expect v0.1.1-deprecated // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241230172942-26aa7a208def // indirect + google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect honnef.co/go/tools v0.4.3 // indirect ) + +tool ( + go.uber.org/yarpc/encoding/protobuf/protoc-gen-yarpc-go + google.golang.org/grpc/cmd/protoc-gen-go-grpc + google.golang.org/protobuf/cmd/protoc-gen-go +) diff --git a/go.sum b/go.sum index b12bd5d0..25779f92 100644 --- a/go.sum +++ b/go.sum @@ -296,6 +296,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20241230172942-26aa7a208def/go. google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1 h1:F29+wU6Ee6qgu9TddPgooOdaqsxTMunOoj8KA5yuS5A= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1/go.mod h1:5KF+wpkbTSbGcR9zteSqZV6fqFOWBl4Yde8En8MryZA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/tool/BUILD.bazel b/tool/BUILD.bazel index 4626b003..15c33589 100644 --- a/tool/BUILD.bazel +++ b/tool/BUILD.bazel @@ -3,4 +3,8 @@ exports_files([ "bazel", + "protoc", + "protoc-gen-go", + "protoc-gen-go-grpc", + "protoc-gen-yarpc-go", ]) diff --git a/tool/protoc b/tool/protoc new file mode 100755 index 00000000..88549fa3 --- /dev/null +++ b/tool/protoc @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2025 Uber Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Hermetic protoc wrapper. Downloads and pins protoc to the version in +# .protocversion, then execs it. This mirrors ./tool/bazel (Bazelisk) so that +# `make proto` produces identical output regardless of any host/homebrew protoc. +# +# The binary is cached under ${XDG_CACHE_HOME:-$HOME/.cache}/submitqueue-protoc/ +# and verified against a pinned SHA-256 before use. + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +version="$(tr -d '[:space:]' <"${repo_root}/.protocversion")" + +# Map the host OS/arch onto the protobuf release asset suffix. +os="$(uname -s)" +arch="$(uname -m)" +case "${os}" in + Darwin) os_part="osx" ;; + Linux) os_part="linux" ;; + *) echo "tool/protoc: unsupported OS: ${os}" >&2; exit 1 ;; +esac +case "${arch}" in + arm64 | aarch64) arch_part="aarch_64" ;; + x86_64 | amd64) arch_part="x86_64" ;; + *) echo "tool/protoc: unsupported arch: ${arch}" >&2; exit 1 ;; +esac +plat="${os_part}-${arch_part}" + +# Pinned SHA-256 of protoc-${version}-${plat}.zip. Add new entries when bumping +# .protocversion (see the release page on github.com/protocolbuffers/protobuf). +checksum="" +case "${version}:${plat}" in + "29.3:osx-aarch_64") checksum="2b8a3403cd097f95f3ba656e14b76c732b6b26d7f183330b11e36ef2bc028765" ;; + "29.3:osx-x86_64") checksum="9a788036d8f9854f7b03c305df4777cf0e54e5b081e25bf15252da87e0e90875" ;; + "29.3:linux-x86_64") checksum="3e866620c5be27664f3d2fa2d656b5f3e09b5152b42f1bedbf427b333e90021a" ;; + "29.3:linux-aarch_64") checksum="6427349140e01f06e049e707a58709a4f221ae73ab9a0425bc4a00c8d0e1ab32" ;; +esac + +if [[ -z "${checksum}" ]]; then + echo "tool/protoc: no pinned SHA-256 for ${version}:${plat}." >&2 + echo " Download protoc-${version}-${plat}.zip from" >&2 + echo " github.com/protocolbuffers/protobuf/releases, record its sha256 in" >&2 + echo " tool/protoc, then retry. Refusing to run an unverified protoc." >&2 + exit 1 +fi + +cache_root="${XDG_CACHE_HOME:-}" +if [[ -z "${cache_root}" ]]; then + if [[ -z "${HOME:-}" ]]; then + echo "tool/protoc: neither XDG_CACHE_HOME nor HOME is set; cannot locate a cache directory" >&2 + exit 1 + fi + cache_root="${HOME}/.cache" +fi +cache_root="${cache_root}/submitqueue-protoc" +install_dir="${cache_root}/${version}/${plat}" +protoc_bin="${install_dir}/bin/protoc" + +sha256() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +if [[ ! -x "${protoc_bin}" ]]; then + url="https://github.com/protocolbuffers/protobuf/releases/download/v${version}/protoc-${version}-${plat}.zip" + tmp="$(mktemp -d)" + trap 'rm -rf "${tmp}"' EXIT + echo "tool/protoc: downloading protoc ${version} (${plat})..." >&2 + curl -fsSL -o "${tmp}/protoc.zip" "${url}" + got="$(sha256 "${tmp}/protoc.zip")" + if [[ "${got}" != "${checksum}" ]]; then + echo "tool/protoc: checksum mismatch for protoc-${version}-${plat}.zip" >&2 + echo " expected ${checksum}" >&2 + echo " got ${got}" >&2 + exit 1 + fi + rm -rf "${install_dir}" + mkdir -p "${install_dir}" + unzip -q "${tmp}/protoc.zip" -d "${install_dir}" +fi + +exec "${protoc_bin}" "$@" diff --git a/tool/protoc-gen-go b/tool/protoc-gen-go new file mode 100755 index 00000000..c4e9b53a --- /dev/null +++ b/tool/protoc-gen-go @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2025 Uber Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Hermetic protoc-gen-go wrapper. Runs the version pinned by the `tool` +# directive in go.mod via the Go toolchain, so `make proto` produces identical +# output regardless of any host/`go install`ed protoc-gen-go. Mirrors +# ./tool/protoc (which pins protoc itself). protoc invokes this with no args and +# speaks the plugin protocol over stdin/stdout. + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +exec go -C "${repo_root}" tool protoc-gen-go "$@" diff --git a/tool/protoc-gen-go-grpc b/tool/protoc-gen-go-grpc new file mode 100755 index 00000000..a1fb2707 --- /dev/null +++ b/tool/protoc-gen-go-grpc @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2025 Uber Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Hermetic protoc-gen-go-grpc wrapper. Runs the version pinned by the `tool` +# directive in go.mod via the Go toolchain, so `make proto` produces identical +# output regardless of any host/`go install`ed protoc-gen-go-grpc. Mirrors +# ./tool/protoc (which pins protoc itself). protoc invokes this with no args and +# speaks the plugin protocol over stdin/stdout. + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +exec go -C "${repo_root}" tool protoc-gen-go-grpc "$@" diff --git a/tool/protoc-gen-yarpc-go b/tool/protoc-gen-yarpc-go new file mode 100755 index 00000000..4aba1008 --- /dev/null +++ b/tool/protoc-gen-yarpc-go @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2025 Uber Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Hermetic protoc-gen-yarpc-go wrapper. Runs the version pinned by the `tool` +# directive in go.mod via the Go toolchain, so `make proto` produces identical +# output regardless of any host/`go install`ed protoc-gen-yarpc-go. Mirrors +# ./tool/protoc (which pins protoc itself). protoc invokes this with no args and +# speaks the plugin protocol over stdin/stdout. + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +exec go -C "${repo_root}" tool protoc-gen-yarpc-go "$@"