diff --git a/MODULE.bazel b/MODULE.bazel index d44b8457..47162774 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -9,6 +9,9 @@ bazel_dep(name = "rules_proto", version = "7.1.0") # dependencies) from source on every cold build. Requires # --incompatible_enable_proto_toolchain_resolution (set in .bazelrc). protoc = use_extension("@toolchains_protoc//protoc:extensions.bzl", "protoc") +protoc.toolchain( + version = "v29.3", +) use_repo(protoc, "toolchains_protoc_hub") register_toolchains("@toolchains_protoc_hub//:all") @@ -37,6 +40,11 @@ go_sdk.host() go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps") go_deps.from_file(go_mod = "//:go.mod") +go_deps.module( + path = "google.golang.org/grpc/cmd/protoc-gen-go-grpc", + sum = "h1:F29+wU6Ee6qgu9TddPgooOdaqsxTMunOoj8KA5yuS5A=", + version = "v1.5.1", +) # All *direct* Go dependencies of the module have to be listed explicitly use_repo( @@ -49,6 +57,7 @@ use_repo( "com_github_uber_go_tally_v4", "in_gopkg_yaml_v3", "org_golang_google_grpc", + "org_golang_google_grpc_cmd_protoc_gen_go_grpc", "org_golang_google_protobuf", "org_golang_x_oauth2", "org_uber_go_fx", diff --git a/Makefile b/Makefile index 7e2411de..0230b3e9 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,11 @@ YAMLFMT_VERSION ?= v0.16.0 # goimports version for Go formatting + import fixing GOIMPORTS_VERSION ?= v0.33.0 +# Proto packages: / dirs whose protopb/ holds the generated +# stubs. Each is generated by Bazel into bazel-bin/tool/proto/_/ +# (the out_dir convention in tool/proto/BUILD.bazel) and copied back here. +PROTO_PACKAGES = submitqueue/gateway submitqueue/orchestrator stovepipe/gateway stovepipe/orchestrator + # Set REPO_ROOT for docker-compose export REPO_ROOT := $(shell pwd) @@ -106,10 +111,7 @@ clean: ## Clean generated files and binaries clean-proto: ## Clean generated proto files @echo "Cleaning generated proto files..." - @rm -rf submitqueue/gateway/protopb/*.pb.go - @rm -rf submitqueue/orchestrator/protopb/*.pb.go - @rm -rf stovepipe/gateway/protopb/*.pb.go - @rm -rf stovepipe/orchestrator/protopb/*.pb.go + @rm -f $(foreach p,$(PROTO_PACKAGES),$(p)/protopb/*.pb.go $(p)/protopb/*.pb.yarpc.go) @echo "Proto clean complete!" deps: tidy-go ## Download and tidy Go dependencies @@ -338,23 +340,15 @@ mocks: ## Generate mock files using mockgen @echo "Mocks generated successfully!" proto: ## Generate protobuf files from .proto definitions - @echo "Generating protobuf files with protoc..." - @protoc --go_out=submitqueue/gateway/protopb --go_opt=paths=source_relative \ - --go-grpc_out=submitqueue/gateway/protopb --go-grpc_opt=paths=source_relative \ - --yarpc-go_out=submitqueue/gateway/protopb --yarpc-go_opt=paths=source_relative \ - --proto_path=submitqueue/gateway/proto submitqueue/gateway/proto/gateway.proto - @protoc --go_out=submitqueue/orchestrator/protopb --go_opt=paths=source_relative \ - --go-grpc_out=submitqueue/orchestrator/protopb --go-grpc_opt=paths=source_relative \ - --yarpc-go_out=submitqueue/orchestrator/protopb --yarpc-go_opt=paths=source_relative \ - --proto_path=submitqueue/orchestrator/proto submitqueue/orchestrator/proto/orchestrator.proto - @protoc --go_out=stovepipe/gateway/protopb --go_opt=paths=source_relative \ - --go-grpc_out=stovepipe/gateway/protopb --go-grpc_opt=paths=source_relative \ - --yarpc-go_out=stovepipe/gateway/protopb --yarpc-go_opt=paths=source_relative \ - --proto_path=stovepipe/gateway/proto stovepipe/gateway/proto/gateway.proto - @protoc --go_out=stovepipe/orchestrator/protopb --go_opt=paths=source_relative \ - --go-grpc_out=stovepipe/orchestrator/protopb --go-grpc_opt=paths=source_relative \ - --yarpc-go_out=stovepipe/orchestrator/protopb --yarpc-go_opt=paths=source_relative \ - --proto_path=stovepipe/orchestrator/proto stovepipe/orchestrator/proto/orchestrator.proto + @echo "Generating protobuf files with Bazel..." + @$(BAZEL) build //tool/proto:generated + @set -e; for pkg in $(PROTO_PACKAGES); do \ + out=$$(echo $$pkg | tr / _); base=$$(basename $$pkg); \ + for f in $$base.pb.go $$base.pb.yarpc.go $${base}_grpc.pb.go; do \ + cp -f bazel-bin/tool/proto/$$out/$$f $$pkg/protopb/$$f; \ + done; \ + done + @$(BAZEL) run @rules_go//go -- run golang.org/x/tools/cmd/goimports@$(GOIMPORTS_VERSION) -w $(addsuffix /protopb,$(PROTO_PACKAGES)) @echo "Protobuf files generated successfully!" # Bazel query helpers diff --git a/doc/howto/DEVELOPMENT.md b/doc/howto/DEVELOPMENT.md index 9fccc2bd..0ba7fe23 100644 --- a/doc/howto/DEVELOPMENT.md +++ b/doc/howto/DEVELOPMENT.md @@ -86,12 +86,7 @@ GoLand works with Go modules automatically. Open the project root and GoLand wil ```bash # macOS -brew install protobuf grpcurl - -# Go protoc plugins (only if modifying .proto files) -go install google.golang.org/protobuf/cmd/protoc-gen-go@latest -go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest -go install go.uber.org/yarpc/encoding/protobuf/protoc-gen-yarpc-go@latest +brew install grpcurl ``` ## Common Make Targets @@ -138,8 +133,8 @@ See [TESTING.md](TESTING.md) for the full testing guide, including integration a ## Troubleshooting **Proto generation fails:** -- Ensure all three protoc plugins are installed (see Optional Tools above) -- Check that `protoc` is in your PATH: `which protoc` +- Run `make proto`; Bazel provides the pinned `protoc` and plugin toolchain. +- If Bazel cannot fetch tools, check network access and the repository cache configuration in `.bazelrc`. **Build fails after proto changes:** - Run `make proto` to regenerate proto files diff --git a/stovepipe/gateway/proto/BUILD.bazel b/stovepipe/gateway/proto/BUILD.bazel index 7019f19d..e2a54be0 100644 --- a/stovepipe/gateway/proto/BUILD.bazel +++ b/stovepipe/gateway/proto/BUILD.bazel @@ -2,6 +2,11 @@ load("@rules_go//go:def.bzl", "go_library") load("@rules_go//proto:def.bzl", "go_proto_library") load("@rules_proto//proto:defs.bzl", "proto_library") +exports_files( + ["gateway.proto"], + visibility = ["//tool/proto:__pkg__"], +) + proto_library( name = "gatewaypb_proto", srcs = ["gateway.proto"], diff --git a/stovepipe/orchestrator/proto/BUILD.bazel b/stovepipe/orchestrator/proto/BUILD.bazel index 68e13e74..62aa615e 100644 --- a/stovepipe/orchestrator/proto/BUILD.bazel +++ b/stovepipe/orchestrator/proto/BUILD.bazel @@ -2,6 +2,11 @@ load("@rules_go//go:def.bzl", "go_library") load("@rules_go//proto:def.bzl", "go_proto_library") load("@rules_proto//proto:defs.bzl", "proto_library") +exports_files( + ["orchestrator.proto"], + visibility = ["//tool/proto:__pkg__"], +) + proto_library( name = "orchestratorpb_proto", srcs = ["orchestrator.proto"], diff --git a/submitqueue/gateway/proto/BUILD.bazel b/submitqueue/gateway/proto/BUILD.bazel index b4fff6c1..17612245 100644 --- a/submitqueue/gateway/proto/BUILD.bazel +++ b/submitqueue/gateway/proto/BUILD.bazel @@ -2,6 +2,11 @@ load("@rules_go//go:def.bzl", "go_library") load("@rules_go//proto:def.bzl", "go_proto_library") load("@rules_proto//proto:defs.bzl", "proto_library") +exports_files( + ["gateway.proto"], + visibility = ["//tool/proto:__pkg__"], +) + proto_library( name = "gatewaypb_proto", srcs = ["gateway.proto"], diff --git a/submitqueue/orchestrator/proto/BUILD.bazel b/submitqueue/orchestrator/proto/BUILD.bazel index c0efa6e5..837b148c 100644 --- a/submitqueue/orchestrator/proto/BUILD.bazel +++ b/submitqueue/orchestrator/proto/BUILD.bazel @@ -2,6 +2,11 @@ load("@rules_go//go:def.bzl", "go_library") load("@rules_go//proto:def.bzl", "go_proto_library") load("@rules_proto//proto:defs.bzl", "proto_library") +exports_files( + ["orchestrator.proto"], + visibility = ["//tool/proto:__pkg__"], +) + proto_library( name = "orchestratorpb_proto", srcs = ["orchestrator.proto"], diff --git a/tool/proto/BUILD.bazel b/tool/proto/BUILD.bazel new file mode 100644 index 00000000..4dfab7b7 --- /dev/null +++ b/tool/proto/BUILD.bazel @@ -0,0 +1,35 @@ +load("//tool/proto:proto_codegen.bzl", "go_proto_generated_files") + +go_proto_generated_files( + name = "submitqueue_gateway", + src = "//submitqueue/gateway/proto:gateway.proto", + out_dir = "submitqueue_gateway", +) + +go_proto_generated_files( + name = "submitqueue_orchestrator", + src = "//submitqueue/orchestrator/proto:orchestrator.proto", + out_dir = "submitqueue_orchestrator", +) + +go_proto_generated_files( + name = "stovepipe_gateway", + src = "//stovepipe/gateway/proto:gateway.proto", + out_dir = "stovepipe_gateway", +) + +go_proto_generated_files( + name = "stovepipe_orchestrator", + src = "//stovepipe/orchestrator/proto:orchestrator.proto", + out_dir = "stovepipe_orchestrator", +) + +filegroup( + name = "generated", + srcs = [ + ":stovepipe_gateway", + ":stovepipe_orchestrator", + ":submitqueue_gateway", + ":submitqueue_orchestrator", + ], +) diff --git a/tool/proto/proto_codegen.bzl b/tool/proto/proto_codegen.bzl new file mode 100644 index 00000000..dbc087a3 --- /dev/null +++ b/tool/proto/proto_codegen.bzl @@ -0,0 +1,84 @@ +"""Hermetic protobuf code generation helpers.""" + +def _strip_proto_suffix(filename): + if not filename.endswith(".proto"): + fail("expected a .proto source, got {}".format(filename)) + return filename[:-len(".proto")] + +def _go_proto_generated_files_impl(ctx): + src = ctx.file.src + base = _strip_proto_suffix(src.basename) + + out_dir = ctx.attr.out_dir + outputs = [ + ctx.actions.declare_file("{}/{}.pb.go".format(out_dir, base)), + ctx.actions.declare_file("{}/{}.pb.yarpc.go".format(out_dir, base)), + ctx.actions.declare_file("{}/{}_grpc.pb.go".format(out_dir, base)), + ] + + proto_toolchain = ctx.toolchains["@rules_proto//proto:toolchain_type"].proto + protoc = proto_toolchain.proto_compiler + + args = ctx.actions.args() + for opt in proto_toolchain.protoc_opts: + args.add(opt) + args.add("--plugin=protoc-gen-go={}".format(ctx.executable._protoc_gen_go.path)) + args.add("--plugin=protoc-gen-go-grpc={}".format(ctx.executable._protoc_gen_go_grpc.path)) + args.add("--plugin=protoc-gen-yarpc-go={}".format(ctx.executable._protoc_gen_yarpc_go.path)) + args.add("--go_out={}".format(outputs[0].dirname)) + args.add("--go_opt=paths=source_relative") + args.add("--go-grpc_out={}".format(outputs[0].dirname)) + args.add("--go-grpc_opt=paths=source_relative") + args.add("--yarpc-go_out={}".format(outputs[0].dirname)) + args.add("--yarpc-go_opt=paths=source_relative") + args.add("--proto_path={}".format(src.dirname)) + args.add(src.path) + + tools = [ + protoc, + ctx.executable._protoc_gen_go, + ctx.executable._protoc_gen_go_grpc, + ctx.executable._protoc_gen_yarpc_go, + ] + + ctx.actions.run_shell( + inputs = [src], + outputs = outputs, + tools = tools, + command = "mkdir -p {out_dir} && {protoc} \"$@\"".format( + out_dir = outputs[0].dirname, + protoc = protoc.executable.path, + ), + arguments = [args], + mnemonic = "HermeticGoProto", + progress_message = "Generating Go protobuf files for {}".format(src.short_path), + ) + + return [DefaultInfo(files = depset(outputs))] + +go_proto_generated_files = rule( + implementation = _go_proto_generated_files_impl, + attrs = { + "src": attr.label( + allow_single_file = [".proto"], + mandatory = True, + ), + "out_dir": attr.string(mandatory = True), + "_protoc_gen_go": attr.label( + default = "@org_golang_google_protobuf//cmd/protoc-gen-go", + executable = True, + cfg = "exec", + ), + "_protoc_gen_go_grpc": attr.label( + default = "@org_golang_google_grpc_cmd_protoc_gen_go_grpc//:protoc-gen-go-grpc", + executable = True, + cfg = "exec", + ), + "_protoc_gen_yarpc_go": attr.label( + default = "@org_uber_go_yarpc//encoding/protobuf/protoc-gen-yarpc-go", + executable = True, + cfg = "exec", + ), + }, + toolchains = ["@rules_proto//proto:toolchain_type"], +)