From 47f6908bf525e948b6a5cfe23090b52ab83f84fb Mon Sep 17 00:00:00 2001 From: Long Ho Date: Fri, 6 Nov 2026 12:23:21 +0000 Subject: [PATCH 1/7] feat: extract messages from runtime dependency graphs --- README.md | 8 + docs/automatic-extraction.md | 123 +++++++++++++ examples/simple/MODULE.bazel | 1 + examples/simple/auto_extract/BUILD.bazel | 37 ++++ examples/simple/auto_extract/expected.json | 1 + examples/simple/auto_extract/extract.bzl | 16 ++ examples/simple/auto_extract/messages.py | 3 + formatjs/BUILD.bazel | 1 + formatjs/auto_extract.bzl | 114 ++++++++++++ formatjs/defs.bzl | 14 +- formatjs/extract.bzl | 151 +++++++++------- tests/auto_extract/BUILD.bazel | 201 +++++++++++++++++++++ tests/auto_extract/empty.json | 1 + tests/auto_extract/fixtures.bzl | 124 +++++++++++++ tests/auto_extract/foreign.py | 3 + tests/auto_extract/ignored.txt | 1 + tests/auto_extract/leaf.py | 3 + tests/auto_extract/legacy.tsx | 5 + tests/auto_extract/messages.json | 1 + tests/auto_extract/python.json | 1 + tests/auto_extract/report.json | 5 + tests/auto_extract/root.rs | 4 + tests/auto_extract/unsupported.json | 1 + 23 files changed, 754 insertions(+), 65 deletions(-) create mode 100644 docs/automatic-extraction.md create mode 100644 examples/simple/auto_extract/BUILD.bazel create mode 100644 examples/simple/auto_extract/expected.json create mode 100644 examples/simple/auto_extract/extract.bzl create mode 100644 examples/simple/auto_extract/messages.py create mode 100644 formatjs/auto_extract.bzl create mode 100644 tests/auto_extract/BUILD.bazel create mode 100644 tests/auto_extract/empty.json create mode 100644 tests/auto_extract/fixtures.bzl create mode 100644 tests/auto_extract/foreign.py create mode 100644 tests/auto_extract/ignored.txt create mode 100644 tests/auto_extract/leaf.py create mode 100644 tests/auto_extract/legacy.tsx create mode 100644 tests/auto_extract/messages.json create mode 100644 tests/auto_extract/python.json create mode 100644 tests/auto_extract/report.json create mode 100644 tests/auto_extract/root.rs create mode 100644 tests/auto_extract/unsupported.json diff --git a/README.md b/README.md index 855db5c..b95e6e4 100644 --- a/README.md +++ b/README.md @@ -260,6 +260,14 @@ Since `formatjs_extract` is a custom rule, you can attach Bazel aspects to it fo - Aggregating messages across multiple targets - Custom reporting and analysis +### Automatic extraction from runtime dependencies + +Use [`formatjs_auto_extract_aspect`](docs/automatic-extraction.md) to extract sources +when their configured dependencies enable FormatJS, then collect catalogs through +transitive depsets. Supply a runtime predicate or `FormatjsRuntimeInfo` marker; no +language-rule dependency is required. Custom rules/aspects can reuse +`formatjs_extract_action` instead of duplicating CLI wiring. + ### Message Aggregation Aspect The `formatjs_aggregate_aspect` collects and merges extracted messages from a target and all its dependencies into a single JSON file using `jq`. diff --git a/docs/automatic-extraction.md b/docs/automatic-extraction.md new file mode 100644 index 0000000..7c8e66a --- /dev/null +++ b/docs/automatic-extraction.md @@ -0,0 +1,123 @@ +# Dependency-triggered extraction + +`formatjs_auto_extract_aspect` extracts sources when a configured dependency enables +FormatJS. It follows `deps` and `srcs`, handles aliases and `select()`, and collects +original catalogs in a depset. It registers no intermediate merge actions and adds +no targets to compilation macros. + +The ruleset has no dependency on Python, Rust or JavaScript build rules. Runtime +integrations can return `FormatjsRuntimeInfo()` from their native rule. For external +runtimes, supply a predicate that recognizes the language's native provider. + +For example, a Rust consumer can define `build/i18n.bzl`: + +```starlark +load("@rules_formatjs//formatjs:defs.bzl", "formatjs_auto_extract_aspect") +load("@rules_rust//rust:defs.bzl", "rust_common") + +def _rust_runtime(target, _ctx): + return ( + rust_common.crate_info in target and + target[rust_common.crate_info].name == "formatjs_intl" + ) + +rust_messages = formatjs_auto_extract_aspect( + name = "rust_messages", + is_runtime = _rust_runtime, + source_extensions = ["rs"], + id_interpolation_pattern = "[sha512:contenthash:base64:10]", + extract_source_location = True, + flatten = True, +) +``` + +Then collect messages from an application: + +```sh +bazel build //app:main \ + --aspects=//build:i18n.bzl%rust_messages \ + --output_groups=all_messages +``` + +Detection uses resolved dependencies during analysis, not source imports or +macro-time inspection of dependency labels. Runtime recognition does not require +an implicit dependency on the runtime at every aspect node. A Python adapter can +inspect a wheel's metadata or native provider without configuring that wheel for +unrelated targets. + +## Collect at the consumer + +Consumers can access `FormatjsAutoExtractInfo.catalogs` to merge once, applying their +own duplicate-ID and source-location policies: + +```starlark +load("@rules_formatjs//formatjs:defs.bzl", "FormatjsAutoExtractInfo") + +def _catalogs_impl(ctx): + return [DefaultInfo(files = ctx.attr.target[FormatjsAutoExtractInfo].catalogs)] + +source_catalogs = rule( + implementation = _catalogs_impl, + attrs = {"target": attr.label(aspects = [rust_messages])}, +) +``` + +`FormatjsAutoExtractInfo.uses_runtime` carries the detection result; +`FormatjsAutoExtractInfo.srcs` identifies sources already represented on source +edges. Shared catalogs are deduplicated in the depset. Existing +`FormatjsExtractInfo` catalogs are collected, and their sources are reused rather +than extracted again. Newly extracted targets also expose `FormatjsExtractInfo`. + +Use one configured aspect for each collection pass. For mixed-language graphs, +combine runtime recognition and source extensions in that aspect. Give each +instantiated aspect a unique `name` to distinguish its output files. + +## Scope and options + +By default, only targets in the main repository get new extraction actions. +External dependencies can enable extraction, and their explicitly provided catalogs +are still collected, but their raw sources are not automatically scanned. Override +`should_extract(target, ctx)` to change this boundary. + +Only files owned by the target's repository are automatically extracted, including +when source labels point into another repository. + +`source_extensions` defaults to `ts`, `tsx`, `js`, `jsx`, `mjs`, `cjs`, `rs`, and `py`. +`dependency_attributes` controls dependency traversal; `source_attributes` controls +source inputs and is also traversed for extraction metadata. Both label and +label-list dependency attributes are supported. No extraction action is registered +without runtime detection and eligible, uncovered sources. + +Other keyword arguments are the extraction options accepted by +`formatjs_extract_action`, with the same defaults as `formatjs_extract`. + +This API collects source messages. Translation archive ownership, locale projection, +catalog merging and runtime installation remain consumer decisions. + +## Reuse the extraction action + +Custom rules and aspects can register extraction without duplicating CLI arguments, +source response-file handling or toolchain wiring: + +```starlark +load("@rules_formatjs//formatjs:defs.bzl", "formatjs_extract_action") + +def _extract_impl(ctx): + info = formatjs_extract_action( + ctx, + ctx.files.srcs, + ctx.actions.declare_file(ctx.label.name + ".json"), + extract_source_location = True, + ) + return [DefaultInfo(files = depset([info.messages])), info] + +extract = rule( + implementation = _extract_impl, + attrs = {"srcs": attr.label_list(allow_files = True)}, + toolchains = ["@rules_formatjs//formatjs_cli:toolchain_type"], +) +``` + +The caller declares the output and supplies source Files. The helper returns +`FormatjsExtractInfo`; it neither creates a target nor merges catalogs. The existing +`formatjs_extract` rule uses the same helper. diff --git a/examples/simple/MODULE.bazel b/examples/simple/MODULE.bazel index c2c44c1..68ab195 100644 --- a/examples/simple/MODULE.bazel +++ b/examples/simple/MODULE.bazel @@ -13,3 +13,4 @@ local_path_override( # Test utilities (not required by rules_formatjs users) bazel_dep(name = "aspect_bazel_lib", version = "2.22.5") +bazel_dep(name = "jq.bzl", version = "0.7.1") diff --git a/examples/simple/auto_extract/BUILD.bazel b/examples/simple/auto_extract/BUILD.bazel new file mode 100644 index 0000000..44e0146 --- /dev/null +++ b/examples/simple/auto_extract/BUILD.bazel @@ -0,0 +1,37 @@ +load("@jq.bzl//jq:jq.bzl", "jq_test") +load("@rules_formatjs//tests/auto_extract:fixtures.bzl", "catalogs", "graph") +load(":extract.bzl", "extract") + +graph( + name = "consumer", + srcs = [ + "messages.py", + "@rules_formatjs//tests/auto_extract:foreign.py", + ], + deps = ["@rules_formatjs//tests/auto_extract:runtime_with_sources"], +) + +catalogs( + name = "messages", + target = ":consumer", +) + +jq_test( + name = "main_repository_only_test", + file1 = ":messages", + file2 = "expected.json", + filter1 = "[.[] | (.defaultMessage // .message)]", +) + +extract( + name = "custom_rule", + srcs = ["messages.py"], + out = "custom_rule.json", +) + +jq_test( + name = "consumer_action_test", + file1 = ":custom_rule", + file2 = "expected.json", + filter1 = "[.[] | (.defaultMessage // .message)]", +) diff --git a/examples/simple/auto_extract/expected.json b/examples/simple/auto_extract/expected.json new file mode 100644 index 0000000..bdab156 --- /dev/null +++ b/examples/simple/auto_extract/expected.json @@ -0,0 +1 @@ +["Consumer message"] diff --git a/examples/simple/auto_extract/extract.bzl b/examples/simple/auto_extract/extract.bzl new file mode 100644 index 0000000..8f23acc --- /dev/null +++ b/examples/simple/auto_extract/extract.bzl @@ -0,0 +1,16 @@ +"""Consumer-defined rule using the public action helper.""" + +load("@rules_formatjs//formatjs:defs.bzl", "formatjs_extract_action") + +def _extract_impl(ctx): + info = formatjs_extract_action(ctx, ctx.files.srcs, ctx.outputs.out) + return [DefaultInfo(files = depset([info.messages])), info] + +extract = rule( + implementation = _extract_impl, + attrs = { + "srcs": attr.label_list(allow_files = [".py"]), + "out": attr.output(mandatory = True), + }, + toolchains = ["@rules_formatjs//formatjs_cli:toolchain_type"], +) diff --git a/examples/simple/auto_extract/messages.py b/examples/simple/auto_extract/messages.py new file mode 100644 index 0000000..11ec61c --- /dev/null +++ b/examples/simple/auto_extract/messages.py @@ -0,0 +1,3 @@ +from intl import define_message + +GREETING = define_message(id="consumer.message", default_message="Consumer message") diff --git a/formatjs/BUILD.bazel b/formatjs/BUILD.bazel index bb78422..1311f9b 100644 --- a/formatjs/BUILD.bazel +++ b/formatjs/BUILD.bazel @@ -2,6 +2,7 @@ exports_files([ "defs.bzl", + "auto_extract.bzl", "extract.bzl", "compile.bzl", "aggregate.bzl", diff --git a/formatjs/auto_extract.bzl b/formatjs/auto_extract.bzl new file mode 100644 index 0000000..17ac7db --- /dev/null +++ b/formatjs/auto_extract.bzl @@ -0,0 +1,114 @@ +"""Dependency-triggered source extraction without language-rule dependencies.""" + +load(":extract.bzl", "FormatjsExtractInfo", "formatjs_extract_action") + +FormatjsRuntimeInfo = provider( + doc = "Marks a runtime dependency that enables FormatJS extraction.", + fields = [], +) + +FormatjsAutoExtractInfo = provider( + doc = "Original catalogs and runtime detection propagated by automatic extraction.", + fields = { + "catalogs": "Depset of original catalogs; no intermediate merges.", + "uses_runtime": "Whether a configured dependency enables extraction.", + "srcs": "Depset of source files already represented on source edges.", + }, +) + +def _marked_runtime(target, _ctx): + return FormatjsRuntimeInfo in target + +def _main_repository(_target, ctx): + return ctx.label.repo_name == "" + +def _targets(ctx, attributes): + result = [] + for attribute in attributes: + value = getattr(ctx.rule.attr, attribute, []) + result.extend(value if type(value) == "list" else [value] if value != None else []) + return result + +def _extracted_sources(target): + if FormatjsAutoExtractInfo in target: + return target[FormatjsAutoExtractInfo].srcs + if FormatjsExtractInfo in target: + return getattr(target[FormatjsExtractInfo], "srcs", depset()) + return depset() + +def formatjs_auto_extract_aspect( + name, + is_runtime = _marked_runtime, + should_extract = _main_repository, + source_extensions = ["ts", "tsx", "js", "jsx", "mjs", "cjs", "rs", "py"], + dependency_attributes = ["deps", "srcs"], + source_attributes = ["srcs"], + **extraction_options): + """Creates an extraction/collection aspect enabled by configured dependencies. + + Args: + name: Unique suffix for this aspect's output files. + is_runtime: Predicate (target, ctx) identifying the runtime; defaults to + FormatjsRuntimeInfo. Language adapters can inspect native providers. + should_extract: Predicate (target, ctx) limiting source owners; defaults + to targets in the main repository. Collection still crosses others. + source_extensions: File extensions without leading dots. + dependency_attributes: Dependency edges followed for runtime detection + and catalog collection. Label and label-list attributes work. + source_attributes: Source attributes; also traversed for extraction metadata. + **extraction_options: Keyword options accepted by formatjs_extract_action. + + Returns: + An aspect exposing FormatjsAutoExtractInfo and all_messages output group. + Newly extracted targets also expose FormatjsExtractInfo. + """ + if not name: + fail("formatjs_auto_extract_aspect requires a nonempty name") + attributes = depset(dependency_attributes + source_attributes).to_list() + + def _impl(target, ctx): + dependencies = [ + dependency[FormatjsAutoExtractInfo] + for dependency in _targets(ctx, attributes) + if FormatjsAutoExtractInfo in dependency + ] + uses_runtime = is_runtime(target, ctx) or any([dependency.uses_runtime for dependency in dependencies]) + catalogs = [] + providers = [] + covered = depset(transitive = [_extracted_sources(dependency) for dependency in _targets(ctx, source_attributes)]) + own_sources = [] + if FormatjsExtractInfo in target: + catalogs.append(target[FormatjsExtractInfo].messages) + covered = depset(transitive = [covered, _extracted_sources(target)]) + elif uses_runtime and should_extract(target, ctx): + already_extracted = {source: True for source in covered.to_list()} + own_sources = depset([ + source + for attribute in source_attributes + for source in getattr(ctx.rule.files, attribute, []) + if source.extension in source_extensions and source.owner.repo_name == ctx.label.repo_name and source not in already_extracted + ]).to_list() + if own_sources: + info = formatjs_extract_action( + ctx, + own_sources, + ctx.actions.declare_file(ctx.label.name + "_" + name + ".json"), + **extraction_options + ) + catalogs.append(info.messages) + providers.append(info) + all_catalogs = depset(catalogs, transitive = [dependency.catalogs for dependency in dependencies]) + return providers + [ + FormatjsAutoExtractInfo( + catalogs = all_catalogs, + uses_runtime = uses_runtime, + srcs = depset(own_sources, transitive = [covered]), + ), + OutputGroupInfo(all_messages = all_catalogs), + ] + + return aspect( + implementation = _impl, + attr_aspects = attributes, + toolchains = [Label("//formatjs_cli:toolchain_type")], + ) diff --git a/formatjs/defs.bzl b/formatjs/defs.bzl index 59570e6..5338ca3 100644 --- a/formatjs/defs.bzl +++ b/formatjs/defs.bzl @@ -9,8 +9,14 @@ load( _formatjs_aggregate = "formatjs_aggregate", _formatjs_aggregate_aspect = "formatjs_aggregate_aspect", ) +load( + ":auto_extract.bzl", + _FormatjsAutoExtractInfo = "FormatjsAutoExtractInfo", + _FormatjsRuntimeInfo = "FormatjsRuntimeInfo", + _formatjs_auto_extract_aspect = "formatjs_auto_extract_aspect", +) load(":compile.bzl", _formatjs_compile = "formatjs_compile") -load(":extract.bzl", _FormatjsExtractInfo = "FormatjsExtractInfo", _formatjs_extract = "formatjs_extract") +load(":extract.bzl", _FormatjsExtractInfo = "FormatjsExtractInfo", _formatjs_extract = "formatjs_extract", _formatjs_extract_action = "formatjs_extract_action") load(":verify.bzl", _formatjs_verify_test = "formatjs_verify_test") # Public API - Rules @@ -25,3 +31,9 @@ formatjs_aggregate_aspect = _formatjs_aggregate_aspect # Public API - Providers FormatjsExtractInfo = _FormatjsExtractInfo FormatjsAggregateInfo = _FormatjsAggregateInfo + +# Public API - Rule/aspect integration +formatjs_extract_action = _formatjs_extract_action +formatjs_auto_extract_aspect = _formatjs_auto_extract_aspect +FormatjsAutoExtractInfo = _FormatjsAutoExtractInfo +FormatjsRuntimeInfo = _FormatjsRuntimeInfo diff --git a/formatjs/extract.bzl b/formatjs/extract.bzl index 766a612..50dc2e5 100644 --- a/formatjs/extract.bzl +++ b/formatjs/extract.bzl @@ -17,6 +17,8 @@ For more information about the FormatJS CLI and its extraction features, see: https://formatjs.github.io/docs/tooling/cli """ +_FORMATJS_TOOLCHAIN_TYPE = Label("//formatjs_cli:toolchain_type") + FormatjsExtractInfo = provider( doc = """Provider containing information about extracted FormatJS messages. @@ -30,87 +32,108 @@ FormatjsExtractInfo = provider( }, ) -def _formatjs_extract_impl(ctx): - """Implementation of the formatjs_extract rule.""" - - # Get FormatJS CLI from toolchain - toolchain = ctx.toolchains["@rules_formatjs//formatjs_cli:toolchain_type"] - formatjs_cli_info = toolchain.formatjs_cli_info - - # Determine output file - if ctx.outputs.out: - out_file = ctx.outputs.out - else: - out_file = ctx.actions.declare_file(ctx.label.name + ".json") - - # Build arguments for formatjs CLI +def formatjs_extract_action( + ctx, + srcs, + out, + format = "", + id_interpolation_pattern = "", + extract_source_location = False, + additional_component_names = [], + additional_function_names = [], + ignore = [], + throws = True, + pragma = "", + preserve_whitespace = False, + flatten = False): + """Registers extraction for a rule or aspect and returns its metadata. + + Args: + ctx: Rule or aspect context with the FormatJS CLI toolchain. + srcs: List of source Files. + out: Caller-declared output File. + format: CLI output formatter. + id_interpolation_pattern: Generated message ID pattern. + extract_source_location: Include source locations. + additional_component_names: Additional component names. + additional_function_names: Additional function names. + ignore: Source exclusion patterns. + throws: Fail on extraction errors. + pragma: Metadata pragma. + preserve_whitespace: Preserve message whitespace. + flatten: Hoist selectors. + + Returns: + FormatjsExtractInfo for the registered action. + """ args = ctx.actions.args() args.add("extract") - args.add("--out-file", out_file) - - if ctx.attr.format: - args.add("--format", ctx.attr.format) - - if ctx.attr.id_interpolation_pattern: - args.add("--id-interpolation-pattern", ctx.attr.id_interpolation_pattern) - - if ctx.attr.extract_source_location: + args.add("--out-file", out) + for flag, value in [ + ("--format", format), + ("--id-interpolation-pattern", id_interpolation_pattern), + ]: + if value: + args.add(flag, value) + if extract_source_location: args.add("--extract-source-location") - - for component in ctx.attr.additional_component_names: - args.add("--additional-component-names", component) - - for function in ctx.attr.additional_function_names: - args.add("--additional-function-names", function) - - for pattern in ctx.attr.ignore: - args.add("--ignore", pattern) - - if ctx.attr.throws: + for flag, values in [ + ("--additional-component-names", additional_component_names), + ("--additional-function-names", additional_function_names), + ("--ignore", ignore), + ]: + for value in values: + args.add(flag, value) + + if throws: args.add("--throws") - - if ctx.attr.pragma: - args.add("--pragma", ctx.attr.pragma) - - if ctx.attr.preserve_whitespace: + if pragma: + args.add("--pragma", pragma) + if preserve_whitespace: args.add("--preserve-whitespace") - - if ctx.attr.flatten: + if flatten: args.add("--flatten") - # Keep source lists below Windows command-line limits. sources = ctx.actions.args() - sources.add_all(ctx.files.srcs) + sources.add_all(srcs) sources.set_param_file_format("multiline") sources.use_param_file("--in-file=%s", use_always = True) - - # Run formatjs extract (v0.1.1+ sorts keys by default) ctx.actions.run( - executable = formatjs_cli_info.cli, + executable = ctx.toolchains[_FORMATJS_TOOLCHAIN_TYPE].formatjs_cli_info.cli, arguments = [args, sources], - inputs = depset(ctx.files.srcs), - outputs = [out_file], + inputs = depset(srcs), + outputs = [out], mnemonic = "FormatjsExtract", progress_message = "Extracting messages from %{label}", - env = { - "BAZEL_BINDIR": ".", - }, + env = {"BAZEL_BINDIR": "."}, + ) + return FormatjsExtractInfo( + messages = out, + srcs = depset(srcs), + id_interpolation_pattern = id_interpolation_pattern, ) +def _formatjs_extract_impl(ctx): + out = ctx.outputs.out or ctx.actions.declare_file(ctx.label.name + ".json") + info = formatjs_extract_action( + ctx, + ctx.files.srcs, + out, + format = ctx.attr.format, + id_interpolation_pattern = ctx.attr.id_interpolation_pattern, + extract_source_location = ctx.attr.extract_source_location, + additional_component_names = ctx.attr.additional_component_names, + additional_function_names = ctx.attr.additional_function_names, + ignore = ctx.attr.ignore, + throws = ctx.attr.throws, + pragma = ctx.attr.pragma, + preserve_whitespace = ctx.attr.preserve_whitespace, + flatten = ctx.attr.flatten, + ) return [ - DefaultInfo( - files = depset([out_file]), - # Make the output file available for direct reference - runfiles = ctx.runfiles(files = [out_file]), - ), - FormatjsExtractInfo( - messages = out_file, - srcs = depset(ctx.files.srcs), - id_interpolation_pattern = ctx.attr.id_interpolation_pattern, - ), - OutputGroupInfo( - json = depset([out_file]), - ), + DefaultInfo(files = depset([out]), runfiles = ctx.runfiles(files = [out])), + info, + OutputGroupInfo(json = depset([out])), ] formatjs_extract = rule( diff --git a/tests/auto_extract/BUILD.bazel b/tests/auto_extract/BUILD.bazel new file mode 100644 index 0000000..7225540 --- /dev/null +++ b/tests/auto_extract/BUILD.bazel @@ -0,0 +1,201 @@ +load("@jq.bzl", "jq") +load("@jq.bzl//jq:jq.bzl", "jq_test") +load("//formatjs:defs.bzl", "formatjs_extract") +load(":fixtures.bzl", "adapter_catalogs", "catalogs", "custom_action", "custom_catalogs", "graph", "metadata", "report", "runtime", "scoped_catalogs") + +exports_files( + ["foreign.py"], + visibility = ["//visibility:public"], +) + +runtime(name = "runtime") + +runtime( + name = "runtime_with_sources", + srcs = ["foreign.py"], + visibility = ["//visibility:public"], +) + +alias( + name = "runtime_alias", + actual = ":runtime", +) + +graph( + name = "leaf", + srcs = select({"//conditions:default": ["leaf.py"]}), + deps = select({"//conditions:default": [":runtime_alias"]}), +) + +alias( + name = "leaf_alias", + actual = ":leaf", +) + +graph( + name = "bridge", + deps = [":leaf_alias"], +) + +formatjs_extract( + name = "legacy", + srcs = ["legacy.tsx"], + extract_source_location = True, +) + +metadata( + name = "legacy_sources", + srcs = ["legacy.tsx"], + extraction = ":legacy", +) + +graph( + name = "root", + srcs = [ + "root.rs", + ":legacy_sources", + ], + deps = [ + ":bridge", + ":leaf", + ], +) + +catalogs( + name = "root_catalogs", + target = ":root", +) + +jq( + name = "merged", + srcs = [":root_catalogs"], + args = ["--slurp"], + filter = "add | [.[] | (.defaultMessage // .message)] | sort", +) + +jq_test( + name = "messages_test", + file1 = ":merged", + file2 = "messages.json", +) + +report( + name = "root_report", + target = ":root", +) + +jq_test( + name = "collection_test", + file1 = ":root_report", + file2 = "report.json", +) + +graph( + name = "plain", + srcs = ["leaf.py"], +) + +report( + name = "plain_report", + target = ":plain", +) + +jq_test( + name = "no_runtime_test", + file1 = ":plain_report", + file2 = "empty.json", +) + +graph( + name = "unsupported", + srcs = ["ignored.txt"], + deps = [":runtime"], +) + +report( + name = "unsupported_report", + target = ":unsupported", +) + +jq_test( + name = "unsupported_extension_test", + file1 = ":unsupported_report", + file2 = "unsupported.json", +) + +graph( + name = "adapter_runtime", + runtime = True, +) + +graph( + name = "adapter_source", + srcs = ["leaf.py"], + deps = [":adapter_runtime"], +) + +adapter_catalogs( + name = "adapter_output", + target = ":adapter_source", +) + +jq_test( + name = "runtime_adapter_test", + file1 = ":adapter_output", + file2 = "python.json", + filter1 = "[.[] | (.defaultMessage // .message)]", +) + +graph( + name = "custom_attributes", + runtime_dep = ":runtime_alias", + source = "leaf.py", +) + +custom_catalogs( + name = "custom_output", + target = ":custom_attributes", +) + +jq_test( + name = "custom_attributes_test", + file1 = ":custom_output", + file2 = "python.json", + filter1 = "[.[] | (.defaultMessage // .message)]", +) + +custom_action( + name = "action_output", + srcs = ["root.rs"], +) + +formatjs_extract( + name = "rule_output", + srcs = ["root.rs"], + extract_source_location = True, + id_interpolation_pattern = "[sha512:contenthash:base64:10]", +) + +jq_test( + name = "action_rule_parity_test", + file1 = ":action_output", + file2 = ":rule_output", +) + +graph( + name = "skipped", + srcs = ["root.rs"], + deps = [":leaf"], +) + +scoped_catalogs( + name = "scoped_output", + target = ":skipped", +) + +jq_test( + name = "custom_scope_test", + file1 = ":scoped_output", + file2 = "python.json", + filter1 = "[.[] | (.defaultMessage // .message)]", +) diff --git a/tests/auto_extract/empty.json b/tests/auto_extract/empty.json new file mode 100644 index 0000000..6dedc9f --- /dev/null +++ b/tests/auto_extract/empty.json @@ -0,0 +1 @@ +{ "catalogs": [], "sources": [], "uses_runtime": false } diff --git a/tests/auto_extract/fixtures.bzl b/tests/auto_extract/fixtures.bzl new file mode 100644 index 0000000..d049097 --- /dev/null +++ b/tests/auto_extract/fixtures.bzl @@ -0,0 +1,124 @@ +"""Real source graphs for the public automatic extraction API.""" + +load("//formatjs:defs.bzl", "FormatjsAutoExtractInfo", "FormatjsExtractInfo", "FormatjsRuntimeInfo", "formatjs_auto_extract_aspect", "formatjs_extract_action") + +def _graph_impl(ctx): + return [DefaultInfo(files = depset(ctx.files.srcs))] + +graph = rule( + implementation = _graph_impl, + attrs = { + "deps": attr.label_list(), + "srcs": attr.label_list(allow_files = True), + "runtime": attr.bool(), + "runtime_dep": attr.label(), + "source": attr.label(allow_single_file = True), + }, +) + +def _runtime_impl(ctx): + return [DefaultInfo(files = depset(ctx.files.srcs)), FormatjsRuntimeInfo()] + +runtime = rule( + implementation = _runtime_impl, + attrs = {"srcs": attr.label_list(allow_files = True)}, +) + +def _metadata_impl(ctx): + return [ + DefaultInfo(files = depset(ctx.files.srcs)), + ctx.attr.extraction[FormatjsExtractInfo], + ] + +metadata = rule( + implementation = _metadata_impl, + attrs = { + "srcs": attr.label_list(allow_files = True), + "extraction": attr.label(providers = [FormatjsExtractInfo]), + }, +) + +messages_aspect = formatjs_auto_extract_aspect( + name = "messages", + extract_source_location = True, + id_interpolation_pattern = "[sha512:contenthash:base64:10]", +) + +def _catalogs_impl(ctx): + return [DefaultInfo(files = ctx.attr.target[FormatjsAutoExtractInfo].catalogs)] + +def _report_impl(ctx): + info = ctx.attr.target[FormatjsAutoExtractInfo] + output = ctx.actions.declare_file(ctx.label.name + ".json") + ctx.actions.write(output, json.encode({ + "catalogs": sorted([file.basename for file in info.catalogs.to_list()]), + "sources": sorted([file.basename for file in info.srcs.to_list()]), + "uses_runtime": info.uses_runtime, + })) + return [DefaultInfo(files = depset([output]))] + +catalogs = rule( + implementation = _catalogs_impl, + attrs = {"target": attr.label(aspects = [messages_aspect])}, +) + +report = rule( + implementation = _report_impl, + attrs = {"target": attr.label(aspects = [messages_aspect])}, +) + +def _adapter_runtime(_target, ctx): + return getattr(ctx.rule.attr, "runtime", False) + +adapter_aspect = formatjs_auto_extract_aspect( + name = "adapter", + is_runtime = _adapter_runtime, + source_extensions = ["py"], +) + +adapter_catalogs = rule( + implementation = _catalogs_impl, + attrs = {"target": attr.label(aspects = [adapter_aspect])}, +) + +custom_attributes_aspect = formatjs_auto_extract_aspect( + name = "custom", + dependency_attributes = ["runtime_dep"], + source_attributes = ["source"], + source_extensions = ["py"], +) + +custom_catalogs = rule( + implementation = _catalogs_impl, + attrs = {"target": attr.label(aspects = [custom_attributes_aspect])}, +) + +def _action_impl(ctx): + output = ctx.actions.declare_file(ctx.label.name + ".json") + info = formatjs_extract_action( + ctx, + ctx.files.srcs, + output, + extract_source_location = True, + id_interpolation_pattern = "[sha512:contenthash:base64:10]", + ) + return [DefaultInfo(files = depset([output])), info] + +custom_action = rule( + implementation = _action_impl, + attrs = {"srcs": attr.label_list(allow_files = True)}, + toolchains = ["//formatjs_cli:toolchain_type"], +) + +def _scope(_target, ctx): + return ctx.label.name != "skipped" + +scoped_aspect = formatjs_auto_extract_aspect( + name = "scoped", + should_extract = _scope, +) + +scoped_catalogs = rule( + implementation = _catalogs_impl, + attrs = {"target": attr.label(aspects = [scoped_aspect])}, +) diff --git a/tests/auto_extract/foreign.py b/tests/auto_extract/foreign.py new file mode 100644 index 0000000..8fc61fd --- /dev/null +++ b/tests/auto_extract/foreign.py @@ -0,0 +1,3 @@ +from intl import define_message + +FOREIGN = define_message(id="foreign.runtime", default_message="External runtime internals") diff --git a/tests/auto_extract/ignored.txt b/tests/auto_extract/ignored.txt new file mode 100644 index 0000000..dbfdbec --- /dev/null +++ b/tests/auto_extract/ignored.txt @@ -0,0 +1 @@ +Not a supported source extension. diff --git a/tests/auto_extract/leaf.py b/tests/auto_extract/leaf.py new file mode 100644 index 0000000..98a95fc --- /dev/null +++ b/tests/auto_extract/leaf.py @@ -0,0 +1,3 @@ +from intl import define_message + +GREETING = define_message(id="auto.python", default_message="Automatic Python") diff --git a/tests/auto_extract/legacy.tsx b/tests/auto_extract/legacy.tsx new file mode 100644 index 0000000..888c37d --- /dev/null +++ b/tests/auto_extract/legacy.tsx @@ -0,0 +1,5 @@ +import { FormattedMessage } from "react-intl"; + +export const Message = () => ( + +); diff --git a/tests/auto_extract/messages.json b/tests/auto_extract/messages.json new file mode 100644 index 0000000..07e3f2e --- /dev/null +++ b/tests/auto_extract/messages.json @@ -0,0 +1 @@ +["Automatic Python", "Automatic Rust", "Explicit TypeScript"] diff --git a/tests/auto_extract/python.json b/tests/auto_extract/python.json new file mode 100644 index 0000000..57e3977 --- /dev/null +++ b/tests/auto_extract/python.json @@ -0,0 +1 @@ +["Automatic Python"] diff --git a/tests/auto_extract/report.json b/tests/auto_extract/report.json new file mode 100644 index 0000000..38a5884 --- /dev/null +++ b/tests/auto_extract/report.json @@ -0,0 +1,5 @@ +{ + "catalogs": ["leaf_messages.json", "legacy.json", "root_messages.json"], + "sources": ["legacy.tsx", "root.rs"], + "uses_runtime": true +} diff --git a/tests/auto_extract/root.rs b/tests/auto_extract/root.rs new file mode 100644 index 0000000..10983b3 --- /dev/null +++ b/tests/auto_extract/root.rs @@ -0,0 +1,4 @@ +const MESSAGE: formatjs_intl::MessageDescriptor = formatjs_intl::message_descriptor!( + id: "auto.rust", + default_message: "Automatic Rust" +); diff --git a/tests/auto_extract/unsupported.json b/tests/auto_extract/unsupported.json new file mode 100644 index 0000000..2c588fb --- /dev/null +++ b/tests/auto_extract/unsupported.json @@ -0,0 +1 @@ +{ "catalogs": [], "sources": [], "uses_runtime": true } From fd610f580a287a082bbd87cbb9f0d37cf4f2fa5a Mon Sep 17 00:00:00 2001 From: Long Ho Date: Fri, 6 Nov 2026 12:57:48 +0000 Subject: [PATCH 2/7] refactor: unify collection and dependency-triggered extraction --- README.md | 4 +- docs/automatic-extraction.md | 41 +++++++++-------- formatjs/BUILD.bazel | 2 +- formatjs/aspects.bzl | 32 ++------------ formatjs/{auto_extract.bzl => collector.bzl} | 46 +++++++++++--------- formatjs/defs.bzl | 10 ++--- tests/auto_extract/BUILD.bazel | 24 +++++++++- tests/auto_extract/fixtures.bzl | 36 ++++++++++++--- 8 files changed, 111 insertions(+), 84 deletions(-) rename formatjs/{auto_extract.bzl => collector.bzl} (72%) diff --git a/README.md b/README.md index b95e6e4..31e89a1 100644 --- a/README.md +++ b/README.md @@ -262,9 +262,9 @@ Since `formatjs_extract` is a custom rule, you can attach Bazel aspects to it fo ### Automatic extraction from runtime dependencies -Use [`formatjs_auto_extract_aspect`](docs/automatic-extraction.md) to extract sources +Use [`formatjs_message_collector_aspect`](docs/automatic-extraction.md) to extract sources when their configured dependencies enable FormatJS, then collect catalogs through -transitive depsets. Supply a runtime predicate or `FormatjsRuntimeInfo` marker; no +transitive depsets. Supply runtime labels or a runtime predicate; no language-rule dependency is required. Custom rules/aspects can reuse `formatjs_extract_action` instead of duplicating CLI wiring. diff --git a/docs/automatic-extraction.md b/docs/automatic-extraction.md index 7c8e66a..490b466 100644 --- a/docs/automatic-extraction.md +++ b/docs/automatic-extraction.md @@ -1,29 +1,23 @@ # Dependency-triggered extraction -`formatjs_auto_extract_aspect` extracts sources when a configured dependency enables +`formatjs_message_collector_aspect` extracts sources when a configured dependency enables FormatJS. It follows `deps` and `srcs`, handles aliases and `select()`, and collects original catalogs in a depset. It registers no intermediate merge actions and adds no targets to compilation macros. The ruleset has no dependency on Python, Rust or JavaScript build rules. Runtime -integrations can return `FormatjsRuntimeInfo()` from their native rule. For external -runtimes, supply a predicate that recognizes the language's native provider. +integrations supply `runtime_labels` or an `is_runtime(target, ctx)` predicate. +Labels can differ across consumers; no Intl package name is hardcoded. A predicate +can inspect native providers or a `FormatjsRuntimeInfo` marker. -For example, a Rust consumer can define `build/i18n.bzl`: +For example, define `build/i18n.bzl` with your runtime labels: ```starlark -load("@rules_formatjs//formatjs:defs.bzl", "formatjs_auto_extract_aspect") -load("@rules_rust//rust:defs.bzl", "rust_common") +load("@rules_formatjs//formatjs:defs.bzl", "formatjs_message_collector_aspect") -def _rust_runtime(target, _ctx): - return ( - rust_common.crate_info in target and - target[rust_common.crate_info].name == "formatjs_intl" - ) - -rust_messages = formatjs_auto_extract_aspect( +rust_messages = formatjs_message_collector_aspect( name = "rust_messages", - is_runtime = _rust_runtime, + runtime_labels = [Label("@my_crates//:formatjs_intl")], source_extensions = ["rs"], id_interpolation_pattern = "[sha512:contenthash:base64:10]", extract_source_location = True, @@ -31,6 +25,15 @@ rust_messages = formatjs_auto_extract_aspect( ) ``` +Use `Label(...)` in the calling module so repository mappings resolve there. +Alternatively, `is_runtime(target, ctx)` can recognize a crate or wheel by its +native provider without relying on a particular target label. Either match enables +extraction; detection propagates through configured dependencies. + +Without runtime labels or a predicate, the same factory only collects existing +catalogs and requires no CLI toolchain. The existing `message_collector_aspect` +entrypoint uses this configuration. + Then collect messages from an application: ```sh @@ -47,14 +50,14 @@ unrelated targets. ## Collect at the consumer -Consumers can access `FormatjsAutoExtractInfo.catalogs` to merge once, applying their +Consumers can access `FormatjsMessagesInfo.catalogs` to merge once, applying their own duplicate-ID and source-location policies: ```starlark -load("@rules_formatjs//formatjs:defs.bzl", "FormatjsAutoExtractInfo") +load("@rules_formatjs//formatjs:defs.bzl", "FormatjsMessagesInfo") def _catalogs_impl(ctx): - return [DefaultInfo(files = ctx.attr.target[FormatjsAutoExtractInfo].catalogs)] + return [DefaultInfo(files = ctx.attr.target[FormatjsMessagesInfo].catalogs)] source_catalogs = rule( implementation = _catalogs_impl, @@ -62,8 +65,8 @@ source_catalogs = rule( ) ``` -`FormatjsAutoExtractInfo.uses_runtime` carries the detection result; -`FormatjsAutoExtractInfo.srcs` identifies sources already represented on source +`FormatjsMessagesInfo.uses_runtime` carries the detection result; +`FormatjsMessagesInfo.srcs` identifies sources already represented on source edges. Shared catalogs are deduplicated in the depset. Existing `FormatjsExtractInfo` catalogs are collected, and their sources are reused rather than extracted again. Newly extracted targets also expose `FormatjsExtractInfo`. diff --git a/formatjs/BUILD.bazel b/formatjs/BUILD.bazel index 1311f9b..2b88caa 100644 --- a/formatjs/BUILD.bazel +++ b/formatjs/BUILD.bazel @@ -2,7 +2,7 @@ exports_files([ "defs.bzl", - "auto_extract.bzl", + "collector.bzl", "extract.bzl", "compile.bzl", "aggregate.bzl", diff --git a/formatjs/aspects.bzl b/formatjs/aspects.bzl index d7f9313..a802a8b 100644 --- a/formatjs/aspects.bzl +++ b/formatjs/aspects.bzl @@ -3,6 +3,7 @@ This module demonstrates how to create aspects that work with formatjs_extract. """ +load(":collector.bzl", "formatjs_message_collector_aspect") load(":extract.bzl", "FormatjsExtractInfo") def _message_stats_aspect_impl(target, ctx): @@ -113,32 +114,5 @@ message_validator_aspect = aspect( """, ) -def _message_collector_aspect_impl(target, ctx): - direct = [target[FormatjsExtractInfo].messages] if FormatjsExtractInfo in target else [] - transitive = [] - for attr_name in ["deps", "srcs"]: - for dep in getattr(ctx.rule.attr, attr_name, []): - if OutputGroupInfo in dep and hasattr(dep[OutputGroupInfo], "all_messages"): - transitive.append(dep[OutputGroupInfo].all_messages) - - return [OutputGroupInfo(all_messages = depset(direct, transitive = transitive))] - -message_collector_aspect = aspect( - implementation = _message_collector_aspect_impl, - attr_aspects = ["deps", "srcs"], - doc = """Collect original message catalogs without registering merge actions. - - Follows both deps and srcs, including libraries that carry extraction - metadata through a source target. Transitive depsets retain shared catalogs - only once without flattening the dependency graph at each target. No jq - toolchain is required. Empty graphs expose an empty all_messages group. - - Consumers can merge the original catalogs with their own duplicate-ID and - source-location policies. This aspect does not extract messages from raw - sources: extraction targets must provide FormatjsExtractInfo. - - Usage: - bazel build //path/to:target --aspects=//formatjs:aspects.bzl%message_collector_aspect \\ - --output_groups=all_messages - """, -) +# Preserve the collection-only entrypoint without requiring a CLI toolchain. +message_collector_aspect = formatjs_message_collector_aspect() diff --git a/formatjs/auto_extract.bzl b/formatjs/collector.bzl similarity index 72% rename from formatjs/auto_extract.bzl rename to formatjs/collector.bzl index 17ac7db..6d17002 100644 --- a/formatjs/auto_extract.bzl +++ b/formatjs/collector.bzl @@ -1,4 +1,4 @@ -"""Dependency-triggered source extraction without language-rule dependencies.""" +"""Catalog collection with optional dependency-triggered source extraction.""" load(":extract.bzl", "FormatjsExtractInfo", "formatjs_extract_action") @@ -7,7 +7,7 @@ FormatjsRuntimeInfo = provider( fields = [], ) -FormatjsAutoExtractInfo = provider( +FormatjsMessagesInfo = provider( doc = "Original catalogs and runtime detection propagated by automatic extraction.", fields = { "catalogs": "Depset of original catalogs; no intermediate merges.", @@ -16,9 +16,6 @@ FormatjsAutoExtractInfo = provider( }, ) -def _marked_runtime(target, _ctx): - return FormatjsRuntimeInfo in target - def _main_repository(_target, ctx): return ctx.label.repo_name == "" @@ -30,26 +27,28 @@ def _targets(ctx, attributes): return result def _extracted_sources(target): - if FormatjsAutoExtractInfo in target: - return target[FormatjsAutoExtractInfo].srcs + if FormatjsMessagesInfo in target: + return target[FormatjsMessagesInfo].srcs if FormatjsExtractInfo in target: return getattr(target[FormatjsExtractInfo], "srcs", depset()) return depset() -def formatjs_auto_extract_aspect( - name, - is_runtime = _marked_runtime, +def formatjs_message_collector_aspect( + name = "messages", + runtime_labels = [], + is_runtime = None, should_extract = _main_repository, source_extensions = ["ts", "tsx", "js", "jsx", "mjs", "cjs", "rs", "py"], dependency_attributes = ["deps", "srcs"], source_attributes = ["srcs"], **extraction_options): - """Creates an extraction/collection aspect enabled by configured dependencies. + """Creates a collector that can extract sources based on configured dependencies. Args: name: Unique suffix for this aspect's output files. - is_runtime: Predicate (target, ctx) identifying the runtime; defaults to - FormatjsRuntimeInfo. Language adapters can inspect native providers. + runtime_labels: Labels of runtime targets that enable extraction. + is_runtime: Optional predicate (target, ctx) for native providers or + FormatjsRuntimeInfo. Extraction is disabled when neither is supplied. should_extract: Predicate (target, ctx) limiting source owners; defaults to targets in the main repository. Collection still crosses others. source_extensions: File extensions without leading dots. @@ -59,20 +58,27 @@ def formatjs_auto_extract_aspect( **extraction_options: Keyword options accepted by formatjs_extract_action. Returns: - An aspect exposing FormatjsAutoExtractInfo and all_messages output group. + An aspect exposing FormatjsMessagesInfo and all_messages output group. Newly extracted targets also expose FormatjsExtractInfo. """ if not name: - fail("formatjs_auto_extract_aspect requires a nonempty name") + fail("formatjs_message_collector_aspect requires a nonempty name") + runtime_labels = [Label(label) if type(label) == "string" else label for label in runtime_labels] + extraction_enabled = bool(runtime_labels) or is_runtime != None attributes = depset(dependency_attributes + source_attributes).to_list() def _impl(target, ctx): dependencies = [ - dependency[FormatjsAutoExtractInfo] + dependency[FormatjsMessagesInfo] for dependency in _targets(ctx, attributes) - if FormatjsAutoExtractInfo in dependency + if FormatjsMessagesInfo in dependency ] - uses_runtime = is_runtime(target, ctx) or any([dependency.uses_runtime for dependency in dependencies]) + uses_runtime = ( + target.label in runtime_labels or + ctx.label in runtime_labels or + (is_runtime != None and is_runtime(target, ctx)) or + any([dependency.uses_runtime for dependency in dependencies]) + ) catalogs = [] providers = [] covered = depset(transitive = [_extracted_sources(dependency) for dependency in _targets(ctx, source_attributes)]) @@ -99,7 +105,7 @@ def formatjs_auto_extract_aspect( providers.append(info) all_catalogs = depset(catalogs, transitive = [dependency.catalogs for dependency in dependencies]) return providers + [ - FormatjsAutoExtractInfo( + FormatjsMessagesInfo( catalogs = all_catalogs, uses_runtime = uses_runtime, srcs = depset(own_sources, transitive = [covered]), @@ -110,5 +116,5 @@ def formatjs_auto_extract_aspect( return aspect( implementation = _impl, attr_aspects = attributes, - toolchains = [Label("//formatjs_cli:toolchain_type")], + toolchains = [Label("//formatjs_cli:toolchain_type")] if extraction_enabled else [], ) diff --git a/formatjs/defs.bzl b/formatjs/defs.bzl index 5338ca3..13fd9f5 100644 --- a/formatjs/defs.bzl +++ b/formatjs/defs.bzl @@ -10,10 +10,10 @@ load( _formatjs_aggregate_aspect = "formatjs_aggregate_aspect", ) load( - ":auto_extract.bzl", - _FormatjsAutoExtractInfo = "FormatjsAutoExtractInfo", + ":collector.bzl", + _FormatjsMessagesInfo = "FormatjsMessagesInfo", _FormatjsRuntimeInfo = "FormatjsRuntimeInfo", - _formatjs_auto_extract_aspect = "formatjs_auto_extract_aspect", + _formatjs_message_collector_aspect = "formatjs_message_collector_aspect", ) load(":compile.bzl", _formatjs_compile = "formatjs_compile") load(":extract.bzl", _FormatjsExtractInfo = "FormatjsExtractInfo", _formatjs_extract = "formatjs_extract", _formatjs_extract_action = "formatjs_extract_action") @@ -34,6 +34,6 @@ FormatjsAggregateInfo = _FormatjsAggregateInfo # Public API - Rule/aspect integration formatjs_extract_action = _formatjs_extract_action -formatjs_auto_extract_aspect = _formatjs_auto_extract_aspect -FormatjsAutoExtractInfo = _FormatjsAutoExtractInfo +formatjs_message_collector_aspect = _formatjs_message_collector_aspect +FormatjsMessagesInfo = _FormatjsMessagesInfo FormatjsRuntimeInfo = _FormatjsRuntimeInfo diff --git a/tests/auto_extract/BUILD.bazel b/tests/auto_extract/BUILD.bazel index 7225540..f5a6207 100644 --- a/tests/auto_extract/BUILD.bazel +++ b/tests/auto_extract/BUILD.bazel @@ -1,7 +1,7 @@ load("@jq.bzl", "jq") load("@jq.bzl//jq:jq.bzl", "jq_test") load("//formatjs:defs.bzl", "formatjs_extract") -load(":fixtures.bzl", "adapter_catalogs", "catalogs", "custom_action", "custom_catalogs", "graph", "metadata", "report", "runtime", "scoped_catalogs") +load(":fixtures.bzl", "adapter_catalogs", "catalogs", "collection_only_catalogs", "custom_action", "custom_catalogs", "graph", "metadata", "report", "runtime", "scoped_catalogs", "unmatched_report") exports_files( ["foreign.py"], @@ -199,3 +199,25 @@ jq_test( file2 = "python.json", filter1 = "[.[] | (.defaultMessage // .message)]", ) + +collection_only_catalogs( + name = "collection_only_output", + target = ":root", +) + +jq_test( + name = "collection_only_test", + file1 = ":collection_only_output", + file2 = ":legacy", +) + +unmatched_report( + name = "unmatched_output", + target = ":leaf", +) + +jq_test( + name = "unmatched_runtime_test", + file1 = ":unmatched_output", + file2 = "empty.json", +) diff --git a/tests/auto_extract/fixtures.bzl b/tests/auto_extract/fixtures.bzl index d049097..4fad451 100644 --- a/tests/auto_extract/fixtures.bzl +++ b/tests/auto_extract/fixtures.bzl @@ -1,6 +1,6 @@ """Real source graphs for the public automatic extraction API.""" -load("//formatjs:defs.bzl", "FormatjsAutoExtractInfo", "FormatjsExtractInfo", "FormatjsRuntimeInfo", "formatjs_auto_extract_aspect", "formatjs_extract_action") +load("//formatjs:defs.bzl", "FormatjsExtractInfo", "FormatjsMessagesInfo", "FormatjsRuntimeInfo", "formatjs_extract_action", "formatjs_message_collector_aspect") def _graph_impl(ctx): return [DefaultInfo(files = depset(ctx.files.srcs))] @@ -38,17 +38,21 @@ metadata = rule( }, ) -messages_aspect = formatjs_auto_extract_aspect( +messages_aspect = formatjs_message_collector_aspect( name = "messages", + runtime_labels = [ + Label("//tests/auto_extract:runtime"), + Label("//tests/auto_extract:runtime_with_sources"), + ], extract_source_location = True, id_interpolation_pattern = "[sha512:contenthash:base64:10]", ) def _catalogs_impl(ctx): - return [DefaultInfo(files = ctx.attr.target[FormatjsAutoExtractInfo].catalogs)] + return [DefaultInfo(files = ctx.attr.target[FormatjsMessagesInfo].catalogs)] def _report_impl(ctx): - info = ctx.attr.target[FormatjsAutoExtractInfo] + info = ctx.attr.target[FormatjsMessagesInfo] output = ctx.actions.declare_file(ctx.label.name + ".json") ctx.actions.write(output, json.encode({ "catalogs": sorted([file.basename for file in info.catalogs.to_list()]), @@ -70,7 +74,7 @@ report = rule( def _adapter_runtime(_target, ctx): return getattr(ctx.rule.attr, "runtime", False) -adapter_aspect = formatjs_auto_extract_aspect( +adapter_aspect = formatjs_message_collector_aspect( name = "adapter", is_runtime = _adapter_runtime, source_extensions = ["py"], @@ -81,8 +85,9 @@ adapter_catalogs = rule( attrs = {"target": attr.label(aspects = [adapter_aspect])}, ) -custom_attributes_aspect = formatjs_auto_extract_aspect( +custom_attributes_aspect = formatjs_message_collector_aspect( name = "custom", + runtime_labels = [Label("//tests/auto_extract:runtime")], dependency_attributes = ["runtime_dep"], source_attributes = ["source"], source_extensions = ["py"], @@ -113,8 +118,9 @@ custom_action = rule( def _scope(_target, ctx): return ctx.label.name != "skipped" -scoped_aspect = formatjs_auto_extract_aspect( +scoped_aspect = formatjs_message_collector_aspect( name = "scoped", + runtime_labels = [Label("//tests/auto_extract:runtime")], should_extract = _scope, ) @@ -122,3 +128,19 @@ scoped_catalogs = rule( implementation = _catalogs_impl, attrs = {"target": attr.label(aspects = [scoped_aspect])}, ) + +collection_only_aspect = formatjs_message_collector_aspect() + +collection_only_catalogs = rule( + implementation = _catalogs_impl, + attrs = {"target": attr.label(aspects = [collection_only_aspect])}, +) + +unmatched_aspect = formatjs_message_collector_aspect( + runtime_labels = [Label("//tests/auto_extract:adapter_runtime")], +) + +unmatched_report = rule( + implementation = _report_impl, + attrs = {"target": attr.label(aspects = [unmatched_aspect])}, +) From c700f0724d40c5066fde746f508ccae576fdb668 Mon Sep 17 00:00:00 2001 From: Long Ho Date: Fri, 6 Nov 2026 13:48:44 +0000 Subject: [PATCH 3/7] fix: keep automatic extraction lazy and preserve uncovered sources --- MODULE.bazel | 2 +- docs/automatic-extraction.md | 24 +++- examples/aggregate/MODULE.bazel.lock | 3 +- examples/custom_version/MODULE.bazel.lock | 3 +- examples/simple/MODULE.bazel.lock | 3 +- formatjs/collector.bzl | 145 +++++++++++++++----- formatjs/extract.bzl | 15 +- tests/auto_extract/BUILD.bazel | 60 +++++++- tests/auto_extract/chain_catalog_count.json | 1 + tests/auto_extract/chain_count.json | 1 + tests/auto_extract/empty_catalog.json | 1 + tests/auto_extract/fixtures.bzl | 23 +++- tests/auto_extract/partial.json | 1 + tests/auto_extract/report.json | 4 +- tests/auto_extract/unsupported.json | 6 +- 15 files changed, 236 insertions(+), 56 deletions(-) create mode 100644 tests/auto_extract/chain_catalog_count.json create mode 100644 tests/auto_extract/chain_count.json create mode 100644 tests/auto_extract/empty_catalog.json create mode 100644 tests/auto_extract/partial.json diff --git a/MODULE.bazel b/MODULE.bazel index 956e7b2..4cb29a3 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -31,12 +31,12 @@ module( ) # Core dependencies required by rules_formatjs +bazel_dep(name = "bazel_lib", version = "3.7.2") bazel_dep(name = "jq.bzl", version = "0.7.1") # JSON manipulation in aggregation aspect bazel_dep(name = "package_metadata", version = "0.0.13") # Package metadata for BCR bazel_dep(name = "platforms", version = "1.1.0") # Development dependencies -bazel_dep(name = "bazel_lib", version = "3.7.2", dev_dependency = True) bazel_dep(name = "bazelrc-preset.bzl", version = "1.10.0", dev_dependency = True) bazel_dep(name = "buildifier_prebuilt", version = "10.1.0", dev_dependency = True) bazel_dep(name = "gazelle", version = "0.54.0", dev_dependency = True, repo_name = "bazel_gazelle") diff --git a/docs/automatic-extraction.md b/docs/automatic-extraction.md index 490b466..8ac5410 100644 --- a/docs/automatic-extraction.md +++ b/docs/automatic-extraction.md @@ -25,7 +25,8 @@ rust_messages = formatjs_message_collector_aspect( ) ``` -Use `Label(...)` in the calling module so repository mappings resolve there. +Runtime labels must be `Label` objects, not strings. Use `Label(...)` in the +calling module so repository mappings resolve there. Alternatively, `is_runtime(target, ctx)` can recognize a crate or wheel by its native provider without relying on a particular target label. Either match enables extraction; detection propagates through configured dependencies. @@ -66,10 +67,25 @@ source_catalogs = rule( ``` `FormatjsMessagesInfo.uses_runtime` carries the detection result; -`FormatjsMessagesInfo.srcs` identifies sources already represented on source -edges. Shared catalogs are deduplicated in the depset. Existing +`FormatjsMessagesInfo.covered_srcs` identifies sources already represented by +explicit extraction metadata. Shared catalogs are deduplicated in the depset. Existing `FormatjsExtractInfo` catalogs are collected, and their sources are reused rather -than extracted again. Newly extracted targets also expose `FormatjsExtractInfo`. +than extracted again. Partial metadata excludes only its declared sources; +uncovered files are still extracted. Targets without explicit metadata also expose +`FormatjsExtractInfo` for the automatic catalog. + +The aspect carries candidate source depsets by declaring repository and explicit +catalogs through the graph. Each requested root exposes one automatic catalog plus +its explicit catalogs. Parent outputs do not depend on children's automatic +catalogs, so requesting one root runs one automatic extraction. Source lists and +coverage subtraction are materialized at execution, without flattening transitive +coverage during analysis. An empty filtered list produces an empty catalog. + +Automatic extraction uses the FormatJS CLI, jq and bazel_lib's `spawn_binary` +toolchains. Filtering runs pinned native executables; no host shell or Python is +used. Collection-only mode requires none of these toolchains. Source artifacts are +declared conservatively as action inputs, including files later excluded by scope, +extension or explicit metadata. Use one configured aspect for each collection pass. For mixed-language graphs, combine runtime recognition and source extensions in that aspect. Give each diff --git a/examples/aggregate/MODULE.bazel.lock b/examples/aggregate/MODULE.bazel.lock index c259f1d..6dbf1f2 100644 --- a/examples/aggregate/MODULE.bazel.lock +++ b/examples/aggregate/MODULE.bazel.lock @@ -46,7 +46,8 @@ "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_lib/3.0.0-rc.0/MODULE.bazel": "d6e00979a98ac14ada5e31c8794708b41434d461e7e7ca39b59b765e6d233b18", "https://bcr.bazel.build/modules/bazel_lib/3.0.0/MODULE.bazel": "22b70b80ac89ad3f3772526cd9feee2fa412c2b01933fea7ed13238a448d370d", - "https://bcr.bazel.build/modules/bazel_lib/3.0.0/source.json": "895f21909c6fba01d7c17914bb6c8e135982275a1b18cdaa4e62272217ef1751", + "https://bcr.bazel.build/modules/bazel_lib/3.7.2/MODULE.bazel": "cbc920fff00a55b2fd663d40850560a6fb17fd5ef3a9f3d6cbf0d32908dd8a40", + "https://bcr.bazel.build/modules/bazel_lib/3.7.2/source.json": "7a9c2b06a113c4962d329bb68befdb90d0447788a19aabaa32b476a0458e7d28", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", diff --git a/examples/custom_version/MODULE.bazel.lock b/examples/custom_version/MODULE.bazel.lock index 19fe8d5..1abc27d 100644 --- a/examples/custom_version/MODULE.bazel.lock +++ b/examples/custom_version/MODULE.bazel.lock @@ -40,7 +40,8 @@ "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_lib/3.0.0-rc.0/MODULE.bazel": "d6e00979a98ac14ada5e31c8794708b41434d461e7e7ca39b59b765e6d233b18", - "https://bcr.bazel.build/modules/bazel_lib/3.0.0-rc.0/source.json": "7051768079aa19302df2b9446ad0889839fd08b7d59851eba2c99234d665c9ba", + "https://bcr.bazel.build/modules/bazel_lib/3.7.2/MODULE.bazel": "cbc920fff00a55b2fd663d40850560a6fb17fd5ef3a9f3d6cbf0d32908dd8a40", + "https://bcr.bazel.build/modules/bazel_lib/3.7.2/source.json": "7a9c2b06a113c4962d329bb68befdb90d0447788a19aabaa32b476a0458e7d28", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", diff --git a/examples/simple/MODULE.bazel.lock b/examples/simple/MODULE.bazel.lock index c259f1d..6dbf1f2 100644 --- a/examples/simple/MODULE.bazel.lock +++ b/examples/simple/MODULE.bazel.lock @@ -46,7 +46,8 @@ "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_lib/3.0.0-rc.0/MODULE.bazel": "d6e00979a98ac14ada5e31c8794708b41434d461e7e7ca39b59b765e6d233b18", "https://bcr.bazel.build/modules/bazel_lib/3.0.0/MODULE.bazel": "22b70b80ac89ad3f3772526cd9feee2fa412c2b01933fea7ed13238a448d370d", - "https://bcr.bazel.build/modules/bazel_lib/3.0.0/source.json": "895f21909c6fba01d7c17914bb6c8e135982275a1b18cdaa4e62272217ef1751", + "https://bcr.bazel.build/modules/bazel_lib/3.7.2/MODULE.bazel": "cbc920fff00a55b2fd663d40850560a6fb17fd5ef3a9f3d6cbf0d32908dd8a40", + "https://bcr.bazel.build/modules/bazel_lib/3.7.2/source.json": "7a9c2b06a113c4962d329bb68befdb90d0447788a19aabaa32b476a0458e7d28", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", diff --git a/formatjs/collector.bzl b/formatjs/collector.bzl index 6d17002..ce6a8f2 100644 --- a/formatjs/collector.bzl +++ b/formatjs/collector.bzl @@ -12,7 +12,9 @@ FormatjsMessagesInfo = provider( fields = { "catalogs": "Depset of original catalogs; no intermediate merges.", "uses_runtime": "Whether a configured dependency enables extraction.", - "srcs": "Depset of source files already represented on source edges.", + "covered_srcs": "Depset of source files represented by explicit extraction metadata.", + "explicit_catalogs": "Original explicit catalogs, excluding this aspect's extraction actions.", + "sources_by_repository": "Candidate source depsets grouped by declaring repository.", }, ) @@ -26,12 +28,67 @@ def _targets(ctx, attributes): result.extend(value if type(value) == "list" else [value] if value != None else []) return result -def _extracted_sources(target): - if FormatjsMessagesInfo in target: - return target[FormatjsMessagesInfo].srcs - if FormatjsExtractInfo in target: - return getattr(target[FormatjsExtractInfo], "srcs", depset()) - return depset() +_JQ_TOOLCHAIN = Label("@jq.bzl//jq/toolchain:type") +_SPAWN_TOOLCHAIN = Label("@bazel_lib//lib:spawn_binary_toolchain_type") + +_SOURCE_FILTER = """ + ($covered | split("\\n") | map(select(length > 0)) | INDEX(.)) as $excluded + | $candidates + | split("\\n")[] + | select(length > 0) + | select($excluded[.] == null) +""" + +def _source_mapper(repository, extensions): + def _path(source): + if source.extension in extensions and source.owner.repo_name == repository: + return source.path + return None + + return _path + +def _source_list(ctx, name, sources_by_repository, covered, source_extensions): + excluded = ctx.actions.declare_file(ctx.label.name + "_" + name + ".covered") + excluded_args = ctx.actions.args() + excluded_args.add_all(covered) + excluded_args.set_param_file_format("multiline") + ctx.actions.write(excluded, excluded_args) + + candidates = ctx.actions.declare_file(ctx.label.name + "_" + name + ".candidates") + candidate_args = ctx.actions.args() + for repository, sources in sources_by_repository.items(): + candidate_args.add_all( + sources, + map_each = _source_mapper(repository, source_extensions), + allow_closure = True, + ) + candidate_args.set_param_file_format("multiline") + ctx.actions.write(candidates, candidate_args) + + output = ctx.actions.declare_file(ctx.label.name + "_" + name + ".sources") + jq = ctx.toolchains[_JQ_TOOLCHAIN].jqinfo.bin + spawn = ctx.toolchains[_SPAWN_TOOLCHAIN].spawn_binary_info.bin + args = ctx.actions.args() + args.add("--stdout", output) + args.add("--") + args.add(jq) + args.add_all(["--null-input", "--raw-output", _SOURCE_FILTER]) + args.add("--rawfile") + args.add("covered") + args.add(excluded) + args.add("--rawfile") + args.add("candidates") + args.add(candidates) + ctx.actions.run( + executable = spawn, + arguments = [args], + inputs = [excluded, candidates], + tools = [jq], + outputs = [output], + mnemonic = "FormatjsSourceList", + toolchain = _SPAWN_TOOLCHAIN, + ) + return output def formatjs_message_collector_aspect( name = "messages", @@ -46,7 +103,7 @@ def formatjs_message_collector_aspect( Args: name: Unique suffix for this aspect's output files. - runtime_labels: Labels of runtime targets that enable extraction. + runtime_labels: Label objects for runtime targets that enable extraction. is_runtime: Optional predicate (target, ctx) for native providers or FormatjsRuntimeInfo. Extraction is disabled when neither is supplied. should_extract: Predicate (target, ctx) limiting source owners; defaults @@ -63,7 +120,9 @@ def formatjs_message_collector_aspect( """ if not name: fail("formatjs_message_collector_aspect requires a nonempty name") - runtime_labels = [Label(label) if type(label) == "string" else label for label in runtime_labels] + for label in runtime_labels: + if type(label) != "Label": + fail("runtime_labels must contain Label objects; use Label(...) in the calling .bzl module") extraction_enabled = bool(runtime_labels) or is_runtime != None attributes = depset(dependency_attributes + source_attributes).to_list() @@ -79,42 +138,56 @@ def formatjs_message_collector_aspect( (is_runtime != None and is_runtime(target, ctx)) or any([dependency.uses_runtime for dependency in dependencies]) ) - catalogs = [] - providers = [] - covered = depset(transitive = [_extracted_sources(dependency) for dependency in _targets(ctx, source_attributes)]) - own_sources = [] + explicit_catalogs = depset( + [target[FormatjsExtractInfo].messages] if FormatjsExtractInfo in target else [], + transitive = [dependency.explicit_catalogs for dependency in dependencies], + ) + covered_inputs = [dependency.covered_srcs for dependency in dependencies] if FormatjsExtractInfo in target: - catalogs.append(target[FormatjsExtractInfo].messages) - covered = depset(transitive = [covered, _extracted_sources(target)]) - elif uses_runtime and should_extract(target, ctx): - already_extracted = {source: True for source in covered.to_list()} - own_sources = depset([ - source - for attribute in source_attributes - for source in getattr(ctx.rule.files, attribute, []) - if source.extension in source_extensions and source.owner.repo_name == ctx.label.repo_name and source not in already_extracted - ]).to_list() - if own_sources: - info = formatjs_extract_action( - ctx, - own_sources, - ctx.actions.declare_file(ctx.label.name + "_" + name + ".json"), - **extraction_options - ) - catalogs.append(info.messages) + covered_inputs.append(getattr(target[FormatjsExtractInfo], "srcs", depset())) + covered = depset(transitive = covered_inputs) + groups = {} + for dependency in dependencies: + for repository, sources in dependency.sources_by_repository.items(): + groups.setdefault(repository, []).append(sources) + if uses_runtime and should_extract(target, ctx): + groups.setdefault(ctx.label.repo_name, []).extend([ + dependency[DefaultInfo].files + for dependency in _targets(ctx, source_attributes) + if DefaultInfo in dependency + ]) + sources_by_repository = { + repository: depset(transitive = sources) + for repository, sources in groups.items() + } + sources = depset(transitive = sources_by_repository.values()) + catalogs = explicit_catalogs + providers = [] + if sources: + source_list = _source_list(ctx, name, sources_by_repository, covered, source_extensions) + info = formatjs_extract_action( + ctx, + sources, + ctx.actions.declare_file(ctx.label.name + "_" + name + ".json"), + source_list = source_list, + **extraction_options + ) + catalogs = depset([info.messages], transitive = [explicit_catalogs]) + if FormatjsExtractInfo not in target: providers.append(info) - all_catalogs = depset(catalogs, transitive = [dependency.catalogs for dependency in dependencies]) return providers + [ FormatjsMessagesInfo( - catalogs = all_catalogs, + catalogs = catalogs, uses_runtime = uses_runtime, - srcs = depset(own_sources, transitive = [covered]), + covered_srcs = covered, + explicit_catalogs = explicit_catalogs, + sources_by_repository = sources_by_repository, ), - OutputGroupInfo(all_messages = all_catalogs), + OutputGroupInfo(all_messages = catalogs), ] return aspect( implementation = _impl, attr_aspects = attributes, - toolchains = [Label("//formatjs_cli:toolchain_type")] if extraction_enabled else [], + toolchains = [Label("//formatjs_cli:toolchain_type"), _JQ_TOOLCHAIN, _SPAWN_TOOLCHAIN] if extraction_enabled else [], ) diff --git a/formatjs/extract.bzl b/formatjs/extract.bzl index 50dc2e5..4fda213 100644 --- a/formatjs/extract.bzl +++ b/formatjs/extract.bzl @@ -45,12 +45,13 @@ def formatjs_extract_action( throws = True, pragma = "", preserve_whitespace = False, - flatten = False): + flatten = False, + source_list = None): """Registers extraction for a rule or aspect and returns its metadata. Args: ctx: Rule or aspect context with the FormatJS CLI toolchain. - srcs: List of source Files. + srcs: List or depset of source Files. out: Caller-declared output File. format: CLI output formatter. id_interpolation_pattern: Generated message ID pattern. @@ -62,6 +63,7 @@ def formatjs_extract_action( pragma: Metadata pragma. preserve_whitespace: Preserve message whitespace. flatten: Hoist selectors. + source_list: Optional prefiltered newline-delimited source list File. Returns: FormatjsExtractInfo for the registered action. @@ -94,14 +96,15 @@ def formatjs_extract_action( if flatten: args.add("--flatten") + source_files = srcs if type(srcs) == "depset" else depset(srcs) sources = ctx.actions.args() - sources.add_all(srcs) + sources.add_all(source_files) sources.set_param_file_format("multiline") sources.use_param_file("--in-file=%s", use_always = True) ctx.actions.run( executable = ctx.toolchains[_FORMATJS_TOOLCHAIN_TYPE].formatjs_cli_info.cli, - arguments = [args, sources], - inputs = depset(srcs), + arguments = [args, "--in-file=" + source_list.path] if source_list else [args, sources], + inputs = depset([source_list] if source_list else [], transitive = [source_files]), outputs = [out], mnemonic = "FormatjsExtract", progress_message = "Extracting messages from %{label}", @@ -109,7 +112,7 @@ def formatjs_extract_action( ) return FormatjsExtractInfo( messages = out, - srcs = depset(srcs), + srcs = source_files, id_interpolation_pattern = id_interpolation_pattern, ) diff --git a/tests/auto_extract/BUILD.bazel b/tests/auto_extract/BUILD.bazel index f5a6207..3258014 100644 --- a/tests/auto_extract/BUILD.bazel +++ b/tests/auto_extract/BUILD.bazel @@ -1,7 +1,7 @@ load("@jq.bzl", "jq") load("@jq.bzl//jq:jq.bzl", "jq_test") load("//formatjs:defs.bzl", "formatjs_extract") -load(":fixtures.bzl", "adapter_catalogs", "catalogs", "collection_only_catalogs", "custom_action", "custom_catalogs", "graph", "metadata", "report", "runtime", "scoped_catalogs", "unmatched_report") +load(":fixtures.bzl", "adapter_catalogs", "catalogs", "collection_only_catalogs", "custom_action", "custom_catalogs", "graph", "metadata", "report", "runtime", "scoped_catalogs", "source_chain", "unmatched_report") exports_files( ["foreign.py"], @@ -221,3 +221,61 @@ jq_test( file1 = ":unmatched_output", file2 = "empty.json", ) + +metadata( + name = "partial_metadata", + srcs = [ + "leaf.py", + "legacy.tsx", + ], + extraction = ":legacy", + deps = [":runtime"], +) + +catalogs( + name = "partial_catalogs", + target = ":partial_metadata", +) + +jq( + name = "partial_merged", + srcs = [":partial_catalogs"], + args = ["--slurp"], + filter = "add | [.[] | (.defaultMessage // .message)] | sort", +) + +jq_test( + name = "partial_metadata_test", + file1 = ":partial_merged", + file2 = "partial.json", +) + +source_chain( + name = "chain", + count = 100, +) + +jq_test( + name = "source_chain_messages_test", + file1 = ":chain_catalogs", + file2 = "chain_count.json", + filter1 = "length", +) + +jq_test( + name = "source_chain_catalog_test", + file1 = ":chain_report", + file2 = "chain_catalog_count.json", + filter1 = ".catalogs | length", +) + +catalogs( + name = "unsupported_catalogs", + target = ":unsupported", +) + +jq_test( + name = "empty_filtered_catalog_test", + file1 = ":unsupported_catalogs", + file2 = "empty_catalog.json", +) diff --git a/tests/auto_extract/chain_catalog_count.json b/tests/auto_extract/chain_catalog_count.json new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/tests/auto_extract/chain_catalog_count.json @@ -0,0 +1 @@ +1 diff --git a/tests/auto_extract/chain_count.json b/tests/auto_extract/chain_count.json new file mode 100644 index 0000000..29d6383 --- /dev/null +++ b/tests/auto_extract/chain_count.json @@ -0,0 +1 @@ +100 diff --git a/tests/auto_extract/empty_catalog.json b/tests/auto_extract/empty_catalog.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/tests/auto_extract/empty_catalog.json @@ -0,0 +1 @@ +{} diff --git a/tests/auto_extract/fixtures.bzl b/tests/auto_extract/fixtures.bzl index 4fad451..1f63301 100644 --- a/tests/auto_extract/fixtures.bzl +++ b/tests/auto_extract/fixtures.bzl @@ -3,7 +3,7 @@ load("//formatjs:defs.bzl", "FormatjsExtractInfo", "FormatjsMessagesInfo", "FormatjsRuntimeInfo", "formatjs_extract_action", "formatjs_message_collector_aspect") def _graph_impl(ctx): - return [DefaultInfo(files = depset(ctx.files.srcs))] + return [DefaultInfo(files = depset(transitive = [source[DefaultInfo].files for source in ctx.attr.srcs]))] graph = rule( implementation = _graph_impl, @@ -35,6 +35,7 @@ metadata = rule( attrs = { "srcs": attr.label_list(allow_files = True), "extraction": attr.label(providers = [FormatjsExtractInfo]), + "deps": attr.label_list(), }, ) @@ -56,7 +57,7 @@ def _report_impl(ctx): output = ctx.actions.declare_file(ctx.label.name + ".json") ctx.actions.write(output, json.encode({ "catalogs": sorted([file.basename for file in info.catalogs.to_list()]), - "sources": sorted([file.basename for file in info.srcs.to_list()]), + "sources": sorted([file.basename for file in info.covered_srcs.to_list()]), "uses_runtime": info.uses_runtime, })) return [DefaultInfo(files = depset([output]))] @@ -144,3 +145,21 @@ unmatched_report = rule( implementation = _report_impl, attrs = {"target": attr.label(aspects = [unmatched_aspect])}, ) + +def _python_source_impl(ctx): + output = ctx.actions.declare_file(ctx.label.name + ".py") + ctx.actions.write(output, "from py_intl import define_message\nmessage = define_message(default_message=%s)\n" % json.encode(ctx.label.name)) + return [DefaultInfo(files = depset([output]))] + +python_source = rule(implementation = _python_source_impl) + +def source_chain(name, count): + for index in range(count): + python_source(name = name + "_source_" + str(index)) + graph( + name = name + "_" + str(index), + srcs = [":" + name + "_source_" + str(index)] + ([":" + name + "_" + str(index - 1)] if index else []), + deps = [] if index else [":runtime"], + ) + catalogs(name = name + "_catalogs", target = ":" + name + "_" + str(count - 1)) + report(name = name + "_report", target = ":" + name + "_" + str(count - 1)) diff --git a/tests/auto_extract/partial.json b/tests/auto_extract/partial.json new file mode 100644 index 0000000..dbed1a3 --- /dev/null +++ b/tests/auto_extract/partial.json @@ -0,0 +1 @@ +["Automatic Python", "Explicit TypeScript"] diff --git a/tests/auto_extract/report.json b/tests/auto_extract/report.json index 38a5884..b2e03e4 100644 --- a/tests/auto_extract/report.json +++ b/tests/auto_extract/report.json @@ -1,5 +1,5 @@ { - "catalogs": ["leaf_messages.json", "legacy.json", "root_messages.json"], - "sources": ["legacy.tsx", "root.rs"], + "catalogs": ["legacy.json", "root_messages.json"], + "sources": ["legacy.tsx"], "uses_runtime": true } diff --git a/tests/auto_extract/unsupported.json b/tests/auto_extract/unsupported.json index 2c588fb..0d6e445 100644 --- a/tests/auto_extract/unsupported.json +++ b/tests/auto_extract/unsupported.json @@ -1 +1,5 @@ -{ "catalogs": [], "sources": [], "uses_runtime": true } +{ + "catalogs": ["unsupported_messages.json"], + "sources": [], + "uses_runtime": true +} From 3eb584c50348fdc8f9f0d18c189a37a0c3a676cd Mon Sep 17 00:00:00 2001 From: Long Ho Date: Fri, 6 Nov 2026 14:33:06 +0000 Subject: [PATCH 4/7] fix: resolve runtime aliases and extract only selected sources --- README.md | 3 +- docs/automatic-extraction.md | 179 ++++++++++---------- formatjs/collector.bzl | 142 +++++++--------- formatjs/extract.bzl | 8 +- tests/auto_extract/BUILD.bazel | 164 +++++++++++++++++- tests/auto_extract/batch_catalog_count.json | 1 + tests/auto_extract/chain_catalog_count.json | 2 +- tests/auto_extract/empty.json | 6 +- tests/auto_extract/fixtures.bzl | 67 +++++++- tests/auto_extract/report.json | 2 +- tests/auto_extract/shared_locations.json | 1 + tests/auto_extract/shared_one.py | 3 + tests/auto_extract/shared_two.py | 3 + tests/auto_extract/unsupported.json | 2 +- 14 files changed, 396 insertions(+), 187 deletions(-) create mode 100644 tests/auto_extract/batch_catalog_count.json create mode 100644 tests/auto_extract/shared_locations.json create mode 100644 tests/auto_extract/shared_one.py create mode 100644 tests/auto_extract/shared_two.py diff --git a/README.md b/README.md index 31e89a1..f534144 100644 --- a/README.md +++ b/README.md @@ -266,7 +266,8 @@ Use [`formatjs_message_collector_aspect`](docs/automatic-extraction.md) to extra when their configured dependencies enable FormatJS, then collect catalogs through transitive depsets. Supply runtime labels or a runtime predicate; no language-rule dependency is required. Custom rules/aspects can reuse -`formatjs_extract_action` instead of duplicating CLI wiring. +`formatjs_extract_action` instead of duplicating CLI wiring. Select extraction roots +with `collect_at` for rule adapters or `formatjs_root` for CLI use. ### Message Aggregation Aspect diff --git a/docs/automatic-extraction.md b/docs/automatic-extraction.md index 8ac5410..74e9eb6 100644 --- a/docs/automatic-extraction.md +++ b/docs/automatic-extraction.md @@ -1,16 +1,12 @@ # Dependency-triggered extraction -`formatjs_message_collector_aspect` extracts sources when a configured dependency enables -FormatJS. It follows `deps` and `srcs`, handles aliases and `select()`, and collects -original catalogs in a depset. It registers no intermediate merge actions and adds -no targets to compilation macros. +`formatjs_message_collector_aspect` follows configured `deps` and `srcs`, detects +Intl usage, and carries source plans and explicit catalogs in depsets. Extraction +runs only at selected roots. Libraries do not flatten transitive source sets. -The ruleset has no dependency on Python, Rust or JavaScript build rules. Runtime -integrations supply `runtime_labels` or an `is_runtime(target, ctx)` predicate. -Labels can differ across consumers; no Intl package name is hardcoded. A predicate -can inspect native providers or a `FormatjsRuntimeInfo` marker. +## CLI use -For example, define `build/i18n.bzl` with your runtime labels: +Define `build/i18n.bzl`: ```starlark load("@rules_formatjs//formatjs:defs.bzl", "formatjs_message_collector_aspect") @@ -25,34 +21,51 @@ rust_messages = formatjs_message_collector_aspect( ) ``` -Runtime labels must be `Label` objects, not strings. Use `Label(...)` in the -calling module so repository mappings resolve there. -Alternatively, `is_runtime(target, ctx)` can recognize a crate or wheel by its -native provider without relying on a particular target label. Either match enables -extraction; detection propagates through configured dependencies. - -Without runtime labels or a predicate, the same factory only collects existing -catalogs and requires no CLI toolchain. The existing `message_collector_aspect` -entrypoint uses this configuration. - -Then collect messages from an application: +Select the extraction root explicitly: ```sh bazel build //app:main \ --aspects=//build:i18n.bzl%rust_messages \ + --aspects_parameters=formatjs_root=//app:main \ --output_groups=all_messages ``` -Detection uses resolved dependencies during analysis, not source imports or -macro-time inspection of dependency labels. Runtime recognition does not require -an implicit dependency on the runtime at every aspect node. A Python adapter can -inspect a wheel's metadata or native provider without configuring that wheel for -unrelated targets. +`formatjs_root` must be the actual target's absolute canonical label, not an alias +or relative label. It scopes materialization to that consumer. Other nodes only +propagate source plans and existing catalogs. + +Runtime labels must be caller-created `Label` objects. This preserves the caller's +repository mappings. Aliases, including configurable aliases, are resolved in the +target configuration through private resolver dependencies. They are not traversed +for Intl detection, so configuring a resolver does not opt unrelated graphs into +extraction. Their files are not extraction inputs and are not built for detection. +Resolution can configure runtime dependencies in otherwise unrelated graphs; those +configured targets are shared within each configuration. + +Alternatively, supply `is_runtime(target, ctx)` to inspect native language providers +or a `FormatjsRuntimeInfo` marker. Predicate-only adapters add no resolver dependency +on the runtime. No language build rules are imported by this ruleset. + +## Rule adapters + +Rule-propagated aspects use a `collect_at(target, ctx)` predicate instead of the CLI +parameter. Choose application or catalog-consumer roots, not every library: -## Collect at the consumer +```starlark +def _at_application(_target, ctx): + return ctx.label == Label("//app:main") + +rust_messages = formatjs_message_collector_aspect( + name = "rust_messages", + runtime_labels = [Label("@my_crates//:formatjs_intl")], + source_extensions = ["rs"], + collect_at = _at_application, + extract_source_location = True, +) +``` -Consumers can access `FormatjsMessagesInfo.catalogs` to merge once, applying their -own duplicate-ID and source-location policies: +This configuration also works from the CLI without `--aspects_parameters`. +Consumers access `FormatjsMessagesInfo.catalogs` on a selected root: ```starlark load("@rules_formatjs//formatjs:defs.bzl", "FormatjsMessagesInfo") @@ -66,77 +79,59 @@ source_catalogs = rule( ) ``` -`FormatjsMessagesInfo.uses_runtime` carries the detection result; -`FormatjsMessagesInfo.covered_srcs` identifies sources already represented by -explicit extraction metadata. Shared catalogs are deduplicated in the depset. Existing -`FormatjsExtractInfo` catalogs are collected, and their sources are reused rather -than extracted again. Partial metadata excludes only its declared sources; -uncovered files are still extracted. Targets without explicit metadata also expose -`FormatjsExtractInfo` for the automatic catalog. - -The aspect carries candidate source depsets by declaring repository and explicit -catalogs through the graph. Each requested root exposes one automatic catalog plus -its explicit catalogs. Parent outputs do not depend on children's automatic -catalogs, so requesting one root runs one automatic extraction. Source lists and -coverage subtraction are materialized at execution, without flattening transitive -coverage during analysis. An empty filtered list produces an empty catalog. - -Automatic extraction uses the FormatJS CLI, jq and bazel_lib's `spawn_binary` -toolchains. Filtering runs pinned native executables; no host shell or Python is -used. Collection-only mode requires none of these toolchains. Source artifacts are -declared conservatively as action inputs, including files later excluded by scope, -extension or explicit metadata. - -Use one configured aspect for each collection pass. For mixed-language graphs, -combine runtime recognition and source extensions in that aspect. Give each -instantiated aspect a unique `name` to distinguish its output files. +At intermediate nodes, `catalogs` contains only explicit catalogs. The provider also +exposes `uses_runtime`, `covered_srcs`, `explicit_catalogs`, and source depsets grouped +by declaring repository. A predicate that always returns false gives metadata-only +propagation for adapters that materialize catalogs themselves. -## Scope and options +## Catalogs and execution -By default, only targets in the main repository get new extraction actions. -External dependencies can enable extraction, and their explicitly provided catalogs -are still collected, but their raw sources are not automatically scanned. Override -`should_extract(target, ctx)` to change this boundary. +Roots flatten candidate and covered sources once. Ownership, extension and explicit +coverage filters run before extraction actions are registered. Excluded generated +artifacts are not action inputs, so their producers do not run for catalog builds. +Partial explicit metadata excludes only its declared sources. -Only files owned by the target's repository are automatically extracted, including -when source labels point into another repository. +Without source locations, uncovered sources use one batch extraction per root. +With `extract_source_location = True`, each file gets its own catalog: the native +CLI keeps only one location for duplicate IDs within a batch. Per-file catalogs +preserve declarations across files and allow unchanged files to reuse cached +extraction actions. Duplicate occurrences within one file retain the native CLI's +behavior. Catalog order is deterministic; no intermediate merge actions are added. +Consumers choose their duplicate-ID and source-location merge policies. -`source_extensions` defaults to `ts`, `tsx`, `js`, `jsx`, `mjs`, `cjs`, `rs`, and `py`. -`dependency_attributes` controls dependency traversal; `source_attributes` controls -source inputs and is also traversed for extraction metadata. Both label and -label-list dependency attributes are supported. No extraction action is registered -without runtime detection and eligible, uncovered sources. +Existing `FormatjsExtractInfo` catalogs are collected unchanged. Automatic outputs +are exposed through `FormatjsMessagesInfo` and `all_messages`; they do not replace +or add a singular `FormatjsExtractInfo` on the underlying target. + +Automatic mode requires only the native FormatJS CLI toolchain. No host shell, +Python or filtering toolchain is used. Without runtime labels or a predicate, the +factory only collects existing catalogs and requires no CLI toolchain. The existing +`message_collector_aspect` entrypoint keeps that collection-only behavior. -Other keyword arguments are the extraction options accepted by -`formatjs_extract_action`, with the same defaults as `formatjs_extract`. +## Scope and options -This API collects source messages. Translation archive ownership, locale projection, -catalog merging and runtime installation remain consumer decisions. +By default, only targets in the main repository contribute new source candidates. +External dependencies can enable extraction and provide explicit catalogs without +having their implementation scanned. Override `should_extract(target, ctx)` to +change that target boundary. Only files owned by a contributing target's repository +are eligible, including when source labels point into another repository. -## Reuse the extraction action +`source_extensions` defaults to `ts`, `tsx`, `js`, `jsx`, `mjs`, `cjs`, `rs`, and `py`. +`dependency_attributes` controls detection/collection edges; `source_attributes` +controls source inputs and is also traversed for extraction metadata. Both label +and label-list attributes work. Aliases and `select()` are handled after resolution. -Custom rules and aspects can register extraction without duplicating CLI arguments, -source response-file handling or toolchain wiring: +Other keyword arguments are options accepted by `formatjs_extract_action`. +Use one configured aspect per collection pass. Mixed-language adapters can combine +runtime recognition and source extensions in that aspect. Give separate instances +unique `name` values to distinguish output files. -```starlark -load("@rules_formatjs//formatjs:defs.bzl", "formatjs_extract_action") - -def _extract_impl(ctx): - info = formatjs_extract_action( - ctx, - ctx.files.srcs, - ctx.actions.declare_file(ctx.label.name + ".json"), - extract_source_location = True, - ) - return [DefaultInfo(files = depset([info.messages])), info] - -extract = rule( - implementation = _extract_impl, - attrs = {"srcs": attr.label_list(allow_files = True)}, - toolchains = ["@rules_formatjs//formatjs_cli:toolchain_type"], -) -``` +Archive ownership, translation projection, catalog merging and runtime installation +remain consumer decisions. + +## Custom extraction actions -The caller declares the output and supplies source Files. The helper returns -`FormatjsExtractInfo`; it neither creates a target nor merges catalogs. The existing -`formatjs_extract` rule uses the same helper. +`formatjs_extract_action(ctx, srcs, out, **options)` registers the native action and +returns `FormatjsExtractInfo`. `srcs` accepts a list or depset of source Files; the +caller declares the output and CLI toolchain. The existing `formatjs_extract` rule +uses the same helper, including its options and response-file handling. diff --git a/formatjs/collector.bzl b/formatjs/collector.bzl index ce6a8f2..39b9298 100644 --- a/formatjs/collector.bzl +++ b/formatjs/collector.bzl @@ -28,73 +28,46 @@ def _targets(ctx, attributes): result.extend(value if type(value) == "list" else [value] if value != None else []) return result -_JQ_TOOLCHAIN = Label("@jq.bzl//jq/toolchain:type") -_SPAWN_TOOLCHAIN = Label("@bazel_lib//lib:spawn_binary_toolchain_type") - -_SOURCE_FILTER = """ - ($covered | split("\\n") | map(select(length > 0)) | INDEX(.)) as $excluded - | $candidates - | split("\\n")[] - | select(length > 0) - | select($excluded[.] == null) -""" - -def _source_mapper(repository, extensions): - def _path(source): - if source.extension in extensions and source.owner.repo_name == repository: - return source.path - return None - - return _path - -def _source_list(ctx, name, sources_by_repository, covered, source_extensions): - excluded = ctx.actions.declare_file(ctx.label.name + "_" + name + ".covered") - excluded_args = ctx.actions.args() - excluded_args.add_all(covered) - excluded_args.set_param_file_format("multiline") - ctx.actions.write(excluded, excluded_args) - - candidates = ctx.actions.declare_file(ctx.label.name + "_" + name + ".candidates") - candidate_args = ctx.actions.args() - for repository, sources in sources_by_repository.items(): - candidate_args.add_all( +def _extract_catalogs(ctx, name, sources_by_repository, covered, extensions, options): + excluded = {source: True for source in covered.to_list()} + sources = [ + source + for repository, candidates in sources_by_repository.items() + for source in candidates.to_list() + if source.extension in extensions and source.owner.repo_name == repository and source not in excluded + ] + if not sources: + return [] + if not options.get("extract_source_location", False): + return [formatjs_extract_action( + ctx, sources, - map_each = _source_mapper(repository, source_extensions), - allow_closure = True, - ) - candidate_args.set_param_file_format("multiline") - ctx.actions.write(candidates, candidate_args) - - output = ctx.actions.declare_file(ctx.label.name + "_" + name + ".sources") - jq = ctx.toolchains[_JQ_TOOLCHAIN].jqinfo.bin - spawn = ctx.toolchains[_SPAWN_TOOLCHAIN].spawn_binary_info.bin - args = ctx.actions.args() - args.add("--stdout", output) - args.add("--") - args.add(jq) - args.add_all(["--null-input", "--raw-output", _SOURCE_FILTER]) - args.add("--rawfile") - args.add("covered") - args.add(excluded) - args.add("--rawfile") - args.add("candidates") - args.add(candidates) - ctx.actions.run( - executable = spawn, - arguments = [args], - inputs = [excluded, candidates], - tools = [jq], - outputs = [output], - mnemonic = "FormatjsSourceList", - toolchain = _SPAWN_TOOLCHAIN, - ) - return output + ctx.actions.declare_file(ctx.label.name + "_" + name + ".json"), + **options + ).messages] + + # The CLI keeps only one location for duplicate IDs within a batch. + catalogs = [] + names = {} + for source in sources: + filename = source.basename[:80] + "." + str(hash(source.path)) + ".json" + if filename in names: + fail("Extraction output collision between %s and %s" % (source.path, names[filename])) + names[filename] = source.path + catalogs.append(formatjs_extract_action( + ctx, + [source], + ctx.actions.declare_file(ctx.label.name + "_" + name + "/" + filename), + **options + ).messages) + return catalogs def formatjs_message_collector_aspect( name = "messages", runtime_labels = [], is_runtime = None, should_extract = _main_repository, + collect_at = None, source_extensions = ["ts", "tsx", "js", "jsx", "mjs", "cjs", "rs", "py"], dependency_attributes = ["deps", "srcs"], source_attributes = ["srcs"], @@ -108,6 +81,8 @@ def formatjs_message_collector_aspect( FormatjsRuntimeInfo. Extraction is disabled when neither is supplied. should_extract: Predicate (target, ctx) limiting source owners; defaults to targets in the main repository. Collection still crosses others. + collect_at: Optional predicate (target, ctx) selecting extraction roots for + rule adapters. Without it, CLI users set --aspects_parameters=formatjs_root=//app:main. source_extensions: File extensions without leading dots. dependency_attributes: Dependency edges followed for runtime detection and catalog collection. Label and label-list attributes work. @@ -116,7 +91,7 @@ def formatjs_message_collector_aspect( Returns: An aspect exposing FormatjsMessagesInfo and all_messages output group. - Newly extracted targets also expose FormatjsExtractInfo. + Explicit extraction providers remain unchanged on the underlying target. """ if not name: fail("formatjs_message_collector_aspect requires a nonempty name") @@ -132,9 +107,10 @@ def formatjs_message_collector_aspect( for dependency in _targets(ctx, attributes) if FormatjsMessagesInfo in dependency ] + runtime_targets = {runtime.label: True for runtime in ctx.attr._formatjs_runtime_targets} uses_runtime = ( - target.label in runtime_labels or - ctx.label in runtime_labels or + target.label in runtime_targets or + ctx.label in runtime_targets or (is_runtime != None and is_runtime(target, ctx)) or any([dependency.uses_runtime for dependency in dependencies]) ) @@ -160,22 +136,24 @@ def formatjs_message_collector_aspect( repository: depset(transitive = sources) for repository, sources in groups.items() } - sources = depset(transitive = sources_by_repository.values()) catalogs = explicit_catalogs - providers = [] - if sources: - source_list = _source_list(ctx, name, sources_by_repository, covered, source_extensions) - info = formatjs_extract_action( - ctx, - sources, - ctx.actions.declare_file(ctx.label.name + "_" + name + ".json"), - source_list = source_list, - **extraction_options - ) - catalogs = depset([info.messages], transitive = [explicit_catalogs]) - if FormatjsExtractInfo not in target: - providers.append(info) - return providers + [ + if extraction_enabled: + if collect_at != None: + is_root = collect_at(target, ctx) + else: + if not ctx.attr.formatjs_root: + fail("Set --aspects_parameters=formatjs_root=//app:main or configure collect_at") + is_root = str(ctx.label) == ctx.attr.formatjs_root + if is_root: + catalogs = depset(_extract_catalogs( + ctx, + name, + sources_by_repository, + covered, + source_extensions, + extraction_options, + ), transitive = [explicit_catalogs]) + return [ FormatjsMessagesInfo( catalogs = catalogs, uses_runtime = uses_runtime, @@ -186,8 +164,14 @@ def formatjs_message_collector_aspect( OutputGroupInfo(all_messages = catalogs), ] + attrs = { + "_formatjs_runtime_targets": attr.label_list(default = runtime_labels, cfg = "target"), + } + if extraction_enabled and collect_at == None: + attrs["formatjs_root"] = attr.string(default = "") return aspect( implementation = _impl, attr_aspects = attributes, - toolchains = [Label("//formatjs_cli:toolchain_type"), _JQ_TOOLCHAIN, _SPAWN_TOOLCHAIN] if extraction_enabled else [], + attrs = attrs, + toolchains = [Label("//formatjs_cli:toolchain_type")] if extraction_enabled else [], ) diff --git a/formatjs/extract.bzl b/formatjs/extract.bzl index 4fda213..ad0000e 100644 --- a/formatjs/extract.bzl +++ b/formatjs/extract.bzl @@ -45,8 +45,7 @@ def formatjs_extract_action( throws = True, pragma = "", preserve_whitespace = False, - flatten = False, - source_list = None): + flatten = False): """Registers extraction for a rule or aspect and returns its metadata. Args: @@ -63,7 +62,6 @@ def formatjs_extract_action( pragma: Metadata pragma. preserve_whitespace: Preserve message whitespace. flatten: Hoist selectors. - source_list: Optional prefiltered newline-delimited source list File. Returns: FormatjsExtractInfo for the registered action. @@ -103,8 +101,8 @@ def formatjs_extract_action( sources.use_param_file("--in-file=%s", use_always = True) ctx.actions.run( executable = ctx.toolchains[_FORMATJS_TOOLCHAIN_TYPE].formatjs_cli_info.cli, - arguments = [args, "--in-file=" + source_list.path] if source_list else [args, sources], - inputs = depset([source_list] if source_list else [], transitive = [source_files]), + arguments = [args, sources], + inputs = source_files, outputs = [out], mnemonic = "FormatjsExtract", progress_message = "Extracting messages from %{label}", diff --git a/tests/auto_extract/BUILD.bazel b/tests/auto_extract/BUILD.bazel index 3258014..269804b 100644 --- a/tests/auto_extract/BUILD.bazel +++ b/tests/auto_extract/BUILD.bazel @@ -1,7 +1,7 @@ load("@jq.bzl", "jq") load("@jq.bzl//jq:jq.bzl", "jq_test") load("//formatjs:defs.bzl", "formatjs_extract") -load(":fixtures.bzl", "adapter_catalogs", "catalogs", "collection_only_catalogs", "custom_action", "custom_catalogs", "graph", "metadata", "report", "runtime", "scoped_catalogs", "source_chain", "unmatched_report") +load(":fixtures.bzl", "adapter_catalogs", "alias_catalogs", "batch_catalogs", "batch_report", "catalogs", "collection_only_catalogs", "custom_action", "custom_catalogs", "graph", "ignored", "metadata", "report", "runtime", "scoped_catalogs", "selected_alias_catalogs", "source_chain", "unmatched_report") exports_files( ["foreign.py"], @@ -257,7 +257,7 @@ source_chain( jq_test( name = "source_chain_messages_test", - file1 = ":chain_catalogs", + file1 = ":chain_merged", file2 = "chain_count.json", filter1 = "length", ) @@ -266,7 +266,7 @@ jq_test( name = "source_chain_catalog_test", file1 = ":chain_report", file2 = "chain_catalog_count.json", - filter1 = ".catalogs | length", + filter1 = ".catalog_count", ) catalogs( @@ -276,6 +276,162 @@ catalogs( jq_test( name = "empty_filtered_catalog_test", - file1 = ":unsupported_catalogs", + file1 = ":unsupported_merged", file2 = "empty_catalog.json", ) + +jq( + name = "unsupported_merged", + srcs = [":unsupported_catalogs"], + args = ["--slurp"], + filter = "(. // []) | add // {}", +) + +alias_catalogs( + name = "alias_output", + target = ":leaf", +) + +jq_test( + name = "aliased_runtime_label_test", + file1 = ":alias_output", + file2 = "python.json", + filter1 = "[.[] | (.defaultMessage // .message)]", +) + +ignored( + name = "ignored_generated", + tags = ["manual"], +) + +graph( + name = "with_ignored", + srcs = [ + "leaf.py", + ":ignored_generated", + ], + tags = ["manual"], + deps = [":runtime"], +) + +catalogs( + name = "ignored_output", + target = ":with_ignored", +) + +jq_test( + name = "ignored_producer_test", + file1 = ":ignored_output", + file2 = "python.json", + filter1 = "[.[] | (.defaultMessage // .message)]", +) + +graph( + name = "shared_one", + srcs = ["shared_one.py"], + deps = [":runtime"], +) + +graph( + name = "shared_two", + srcs = ["shared_two.py"], + deps = [":runtime"], +) + +graph( + name = "shared_root", + deps = [ + ":shared_one", + ":shared_two", + ], +) + +catalogs( + name = "shared_catalogs", + target = ":shared_root", +) + +jq( + name = "shared_locations", + srcs = [":shared_catalogs"], + args = ["--slurp"], + filter = "[.[] | .[\"shared.message\"].file] | sort", +) + +jq_test( + name = "shared_locations_test", + file1 = ":shared_locations", + file2 = "shared_locations.json", +) + +jq( + name = "chain_merged", + srcs = [":chain_catalogs"], + args = ["--slurp"], + filter = "add", +) + +config_setting( + name = "windows", + constraint_values = ["@platforms//os:windows"], +) + +graph(name = "runtime_windows") + +alias( + name = "runtime_selected_alias", + actual = select({ + ":windows": ":runtime_windows", + "//conditions:default": ":runtime", + }), +) + +graph( + name = "leaf_select", + srcs = ["leaf.py"], + deps = [":runtime_selected_alias"], +) + +selected_alias_catalogs( + name = "selected_alias_output", + target = ":leaf_select", +) + +jq_test( + name = "selected_runtime_alias_test", + file1 = ":selected_alias_output", + file2 = "python.json", + filter1 = "[.[] | (.defaultMessage // .message)]", +) + +platform( + name = "windows_platform", + constraint_values = [ + "@platforms//os:windows", + "@platforms//cpu:x86_64", + ], +) + +batch_catalogs( + name = "batch_chain_catalogs", + target = ":chain_99", +) + +batch_report( + name = "batch_chain_report", + target = ":chain_99", +) + +jq_test( + name = "batch_messages_test", + file1 = ":batch_chain_catalogs", + file2 = "chain_count.json", + filter1 = "length", +) + +jq_test( + name = "batch_catalog_test", + file1 = ":batch_chain_report", + file2 = "batch_catalog_count.json", + filter1 = ".catalog_count", +) diff --git a/tests/auto_extract/batch_catalog_count.json b/tests/auto_extract/batch_catalog_count.json new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/tests/auto_extract/batch_catalog_count.json @@ -0,0 +1 @@ +1 diff --git a/tests/auto_extract/chain_catalog_count.json b/tests/auto_extract/chain_catalog_count.json index d00491f..29d6383 100644 --- a/tests/auto_extract/chain_catalog_count.json +++ b/tests/auto_extract/chain_catalog_count.json @@ -1 +1 @@ -1 +100 diff --git a/tests/auto_extract/empty.json b/tests/auto_extract/empty.json index 6dedc9f..1d17ce8 100644 --- a/tests/auto_extract/empty.json +++ b/tests/auto_extract/empty.json @@ -1 +1,5 @@ -{ "catalogs": [], "sources": [], "uses_runtime": false } +{ + "catalog_count": 0, + "sources": [], + "uses_runtime": false +} diff --git a/tests/auto_extract/fixtures.bzl b/tests/auto_extract/fixtures.bzl index 1f63301..69192a9 100644 --- a/tests/auto_extract/fixtures.bzl +++ b/tests/auto_extract/fixtures.bzl @@ -39,8 +39,26 @@ metadata = rule( }, ) +def _collect_at(_target, ctx): + return ctx.label.name in [ + "root", + "leaf", + "plain", + "unsupported", + "adapter_source", + "custom_attributes", + "skipped", + "partial_metadata", + "chain_99", + "consumer", + "shared_root", + "with_ignored", + "leaf_select", + ] + messages_aspect = formatjs_message_collector_aspect( name = "messages", + collect_at = _collect_at, runtime_labels = [ Label("//tests/auto_extract:runtime"), Label("//tests/auto_extract:runtime_with_sources"), @@ -56,7 +74,7 @@ def _report_impl(ctx): info = ctx.attr.target[FormatjsMessagesInfo] output = ctx.actions.declare_file(ctx.label.name + ".json") ctx.actions.write(output, json.encode({ - "catalogs": sorted([file.basename for file in info.catalogs.to_list()]), + "catalog_count": len(info.catalogs.to_list()), "sources": sorted([file.basename for file in info.covered_srcs.to_list()]), "uses_runtime": info.uses_runtime, })) @@ -77,6 +95,7 @@ def _adapter_runtime(_target, ctx): adapter_aspect = formatjs_message_collector_aspect( name = "adapter", + collect_at = _collect_at, is_runtime = _adapter_runtime, source_extensions = ["py"], ) @@ -88,6 +107,7 @@ adapter_catalogs = rule( custom_attributes_aspect = formatjs_message_collector_aspect( name = "custom", + collect_at = _collect_at, runtime_labels = [Label("//tests/auto_extract:runtime")], dependency_attributes = ["runtime_dep"], source_attributes = ["source"], @@ -121,6 +141,7 @@ def _scope(_target, ctx): scoped_aspect = formatjs_message_collector_aspect( name = "scoped", + collect_at = _collect_at, runtime_labels = [Label("//tests/auto_extract:runtime")], should_extract = _scope, ) @@ -138,7 +159,8 @@ collection_only_catalogs = rule( ) unmatched_aspect = formatjs_message_collector_aspect( - runtime_labels = [Label("//tests/auto_extract:adapter_runtime")], + collect_at = _collect_at, + runtime_labels = [Label("//tests/auto_extract:ignored_generated")], ) unmatched_report = rule( @@ -163,3 +185,44 @@ def source_chain(name, count): ) catalogs(name = name + "_catalogs", target = ":" + name + "_" + str(count - 1)) report(name = name + "_report", target = ":" + name + "_" + str(count - 1)) + +alias_aspect = formatjs_message_collector_aspect( + runtime_labels = [Label("//tests/auto_extract:runtime_alias")], + collect_at = _collect_at, +) + +alias_catalogs = rule( + implementation = _catalogs_impl, + attrs = {"target": attr.label(aspects = [alias_aspect])}, +) + +def _ignored_impl(ctx): + output = ctx.actions.declare_file(ctx.label.name + ".txt") + ctx.actions.run_shell(outputs = [output], command = "exit 1", mnemonic = "IgnoredProducer") + return [DefaultInfo(files = depset([output]))] + +ignored = rule(implementation = _ignored_impl) + +cli_aspect = formatjs_message_collector_aspect( + runtime_labels = [Label("//tests/auto_extract:runtime_alias"), Label("//tests/auto_extract:runtime_with_sources")], + extract_source_location = True, +) + +selected_alias_aspect = formatjs_message_collector_aspect( + runtime_labels = [Label("//tests/auto_extract:runtime_selected_alias")], + collect_at = _collect_at, +) + +selected_alias_catalogs = rule( + implementation = _catalogs_impl, + attrs = {"target": attr.label(aspects = [selected_alias_aspect])}, +) + +batch_aspect = formatjs_message_collector_aspect( + name = "batch", + runtime_labels = [Label("//tests/auto_extract:runtime_alias")], + collect_at = _collect_at, +) + +batch_catalogs = rule(implementation = _catalogs_impl, attrs = {"target": attr.label(aspects = [batch_aspect])}) +batch_report = rule(implementation = _report_impl, attrs = {"target": attr.label(aspects = [batch_aspect])}) diff --git a/tests/auto_extract/report.json b/tests/auto_extract/report.json index b2e03e4..8891551 100644 --- a/tests/auto_extract/report.json +++ b/tests/auto_extract/report.json @@ -1,5 +1,5 @@ { - "catalogs": ["legacy.json", "root_messages.json"], + "catalog_count": 3, "sources": ["legacy.tsx"], "uses_runtime": true } diff --git a/tests/auto_extract/shared_locations.json b/tests/auto_extract/shared_locations.json new file mode 100644 index 0000000..ebdbd23 --- /dev/null +++ b/tests/auto_extract/shared_locations.json @@ -0,0 +1 @@ +["tests/auto_extract/shared_one.py", "tests/auto_extract/shared_two.py"] diff --git a/tests/auto_extract/shared_one.py b/tests/auto_extract/shared_one.py new file mode 100644 index 0000000..5b3634d --- /dev/null +++ b/tests/auto_extract/shared_one.py @@ -0,0 +1,3 @@ +from intl import define_message + +MESSAGE = define_message(id="shared.message", default_message="Shared message") diff --git a/tests/auto_extract/shared_two.py b/tests/auto_extract/shared_two.py new file mode 100644 index 0000000..5b3634d --- /dev/null +++ b/tests/auto_extract/shared_two.py @@ -0,0 +1,3 @@ +from intl import define_message + +MESSAGE = define_message(id="shared.message", default_message="Shared message") diff --git a/tests/auto_extract/unsupported.json b/tests/auto_extract/unsupported.json index 0d6e445..a0179c0 100644 --- a/tests/auto_extract/unsupported.json +++ b/tests/auto_extract/unsupported.json @@ -1,5 +1,5 @@ { - "catalogs": ["unsupported_messages.json"], + "catalog_count": 0, "sources": [], "uses_runtime": true } From 78ed97d505a1444356ef9723464fb743e59816fe Mon Sep 17 00:00:00 2001 From: Long Ho Date: Fri, 6 Nov 2026 14:38:59 +0000 Subject: [PATCH 5/7] build: keep bazel_lib as a development dependency --- MODULE.bazel | 2 +- examples/aggregate/MODULE.bazel.lock | 3 +-- examples/custom_version/MODULE.bazel.lock | 3 +-- examples/simple/MODULE.bazel.lock | 3 +-- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 4cb29a3..956e7b2 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -31,12 +31,12 @@ module( ) # Core dependencies required by rules_formatjs -bazel_dep(name = "bazel_lib", version = "3.7.2") bazel_dep(name = "jq.bzl", version = "0.7.1") # JSON manipulation in aggregation aspect bazel_dep(name = "package_metadata", version = "0.0.13") # Package metadata for BCR bazel_dep(name = "platforms", version = "1.1.0") # Development dependencies +bazel_dep(name = "bazel_lib", version = "3.7.2", dev_dependency = True) bazel_dep(name = "bazelrc-preset.bzl", version = "1.10.0", dev_dependency = True) bazel_dep(name = "buildifier_prebuilt", version = "10.1.0", dev_dependency = True) bazel_dep(name = "gazelle", version = "0.54.0", dev_dependency = True, repo_name = "bazel_gazelle") diff --git a/examples/aggregate/MODULE.bazel.lock b/examples/aggregate/MODULE.bazel.lock index 6dbf1f2..c259f1d 100644 --- a/examples/aggregate/MODULE.bazel.lock +++ b/examples/aggregate/MODULE.bazel.lock @@ -46,8 +46,7 @@ "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_lib/3.0.0-rc.0/MODULE.bazel": "d6e00979a98ac14ada5e31c8794708b41434d461e7e7ca39b59b765e6d233b18", "https://bcr.bazel.build/modules/bazel_lib/3.0.0/MODULE.bazel": "22b70b80ac89ad3f3772526cd9feee2fa412c2b01933fea7ed13238a448d370d", - "https://bcr.bazel.build/modules/bazel_lib/3.7.2/MODULE.bazel": "cbc920fff00a55b2fd663d40850560a6fb17fd5ef3a9f3d6cbf0d32908dd8a40", - "https://bcr.bazel.build/modules/bazel_lib/3.7.2/source.json": "7a9c2b06a113c4962d329bb68befdb90d0447788a19aabaa32b476a0458e7d28", + "https://bcr.bazel.build/modules/bazel_lib/3.0.0/source.json": "895f21909c6fba01d7c17914bb6c8e135982275a1b18cdaa4e62272217ef1751", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", diff --git a/examples/custom_version/MODULE.bazel.lock b/examples/custom_version/MODULE.bazel.lock index 1abc27d..19fe8d5 100644 --- a/examples/custom_version/MODULE.bazel.lock +++ b/examples/custom_version/MODULE.bazel.lock @@ -40,8 +40,7 @@ "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_lib/3.0.0-rc.0/MODULE.bazel": "d6e00979a98ac14ada5e31c8794708b41434d461e7e7ca39b59b765e6d233b18", - "https://bcr.bazel.build/modules/bazel_lib/3.7.2/MODULE.bazel": "cbc920fff00a55b2fd663d40850560a6fb17fd5ef3a9f3d6cbf0d32908dd8a40", - "https://bcr.bazel.build/modules/bazel_lib/3.7.2/source.json": "7a9c2b06a113c4962d329bb68befdb90d0447788a19aabaa32b476a0458e7d28", + "https://bcr.bazel.build/modules/bazel_lib/3.0.0-rc.0/source.json": "7051768079aa19302df2b9446ad0889839fd08b7d59851eba2c99234d665c9ba", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", diff --git a/examples/simple/MODULE.bazel.lock b/examples/simple/MODULE.bazel.lock index 6dbf1f2..c259f1d 100644 --- a/examples/simple/MODULE.bazel.lock +++ b/examples/simple/MODULE.bazel.lock @@ -46,8 +46,7 @@ "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_lib/3.0.0-rc.0/MODULE.bazel": "d6e00979a98ac14ada5e31c8794708b41434d461e7e7ca39b59b765e6d233b18", "https://bcr.bazel.build/modules/bazel_lib/3.0.0/MODULE.bazel": "22b70b80ac89ad3f3772526cd9feee2fa412c2b01933fea7ed13238a448d370d", - "https://bcr.bazel.build/modules/bazel_lib/3.7.2/MODULE.bazel": "cbc920fff00a55b2fd663d40850560a6fb17fd5ef3a9f3d6cbf0d32908dd8a40", - "https://bcr.bazel.build/modules/bazel_lib/3.7.2/source.json": "7a9c2b06a113c4962d329bb68befdb90d0447788a19aabaa32b476a0458e7d28", + "https://bcr.bazel.build/modules/bazel_lib/3.0.0/source.json": "895f21909c6fba01d7c17914bb6c8e135982275a1b18cdaa4e62272217ef1751", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", From 05b5888016816b0f527c40fdcc05b1bc22480e55 Mon Sep 17 00:00:00 2001 From: Long Ho Date: Fri, 6 Nov 2026 14:49:53 +0000 Subject: [PATCH 6/7] fix: preserve source scope and handle output name collisions --- docs/automatic-extraction.md | 8 ++- formatjs/collector.bzl | 75 +++++++++++++++++---- tests/auto_extract/Aa/messages.py | 2 + tests/auto_extract/BB/messages.py | 2 + tests/auto_extract/BUILD.bazel | 99 +++++++++++++++++++++++++++- tests/auto_extract/collisions.json | 1 + tests/auto_extract/fixtures.bzl | 5 +- tests/auto_extract/scope_direct.json | 1 + 8 files changed, 175 insertions(+), 18 deletions(-) create mode 100644 tests/auto_extract/Aa/messages.py create mode 100644 tests/auto_extract/BB/messages.py create mode 100644 tests/auto_extract/collisions.json create mode 100644 tests/auto_extract/scope_direct.json diff --git a/docs/automatic-extraction.md b/docs/automatic-extraction.md index 74e9eb6..8258e51 100644 --- a/docs/automatic-extraction.md +++ b/docs/automatic-extraction.md @@ -97,7 +97,8 @@ CLI keeps only one location for duplicate IDs within a batch. Per-file catalogs preserve declarations across files and allow unchanged files to reuse cached extraction actions. Duplicate occurrences within one file retain the native CLI's behavior. Catalog order is deterministic; no intermediate merge actions are added. -Consumers choose their duplicate-ID and source-location merge policies. +Consumers choose their duplicate-ID and source-location merge policies. Output-name +hash collisions receive deterministic suffixes instead of failing analysis. Existing `FormatjsExtractInfo` catalogs are collected unchanged. Automatic outputs are exposed through `FormatjsMessagesInfo` and `all_messages`; they do not replace @@ -113,7 +114,10 @@ factory only collects existing catalogs and requires no CLI toolchain. The exist By default, only targets in the main repository contribute new source candidates. External dependencies can enable extraction and provide explicit catalogs without having their implementation scanned. Override `should_extract(target, ctx)` to -change that target boundary. Only files owned by a contributing target's repository +change that target boundary. Rejected sources stay excluded when ancestors forward +them through source wrappers. Independent allowed declarations of the same source +remain eligible. These scope decisions stay in depsets until the extraction root. +Only files owned by a contributing target's repository are eligible, including when source labels point into another repository. `source_extensions` defaults to `ts`, `tsx`, `js`, `jsx`, `mjs`, `cjs`, `rs`, and `py`. diff --git a/formatjs/collector.bzl b/formatjs/collector.bzl index 39b9298..2495e79 100644 --- a/formatjs/collector.bzl +++ b/formatjs/collector.bzl @@ -15,6 +15,10 @@ FormatjsMessagesInfo = provider( "covered_srcs": "Depset of source files represented by explicit extraction metadata.", "explicit_catalogs": "Original explicit catalogs, excluding this aspect's extraction actions.", "sources_by_repository": "Candidate source depsets grouped by declaring repository.", + "scope_excluded_srcs": "Files rejected by source scope, including forwarded outputs.", + "scope_direct_srcs": "Direct source files explicitly allowed by an opted-in target.", + "scope_allowed_owners": "Allowed source targets explicitly referenced by an opted-in target.", + "allows_sources": "Whether this target passes the source-scope predicate.", }, ) @@ -28,13 +32,21 @@ def _targets(ctx, attributes): result.extend(value if type(value) == "list" else [value] if value != None else []) return result -def _extract_catalogs(ctx, name, sources_by_repository, covered, extensions, options): +def _source_path(source): + return source.path + +def _extract_catalogs(ctx, name, sources_by_repository, covered, scope_excluded, scope_direct, scope_owners, extensions, options): excluded = {source: True for source in covered.to_list()} + rejected = {source: True for source in scope_excluded.to_list()} + direct = {source: True for source in scope_direct.to_list()} + owners = {owner: True for owner in scope_owners.to_list()} sources = [ source for repository, candidates in sources_by_repository.items() for source in candidates.to_list() - if source.extension in extensions and source.owner.repo_name == repository and source not in excluded + if source.extension in extensions and source.owner.repo_name == repository and source not in excluded and ( + source not in rejected or source in direct or source.owner in owners + ) ] if not sources: return [] @@ -50,16 +62,18 @@ def _extract_catalogs(ctx, name, sources_by_repository, covered, extensions, opt catalogs = [] names = {} for source in sources: - filename = source.basename[:80] + "." + str(hash(source.path)) + ".json" - if filename in names: - fail("Extraction output collision between %s and %s" % (source.path, names[filename])) - names[filename] = source.path - catalogs.append(formatjs_extract_action( - ctx, - [source], - ctx.actions.declare_file(ctx.label.name + "_" + name + "/" + filename), - **options - ).messages) + filename = source.basename[:80] + "." + str(hash(source.path)) + names.setdefault(filename, []).append(source) + for filename, files in names.items(): + # Disambiguate hash collisions without lengthening every source path. + for index, source in enumerate(sorted(files, key = _source_path) if len(files) > 1 else files): + suffix = "_" + str(index) if len(files) > 1 else "" + catalogs.append(formatjs_extract_action( + ctx, + [source], + ctx.actions.declare_file(ctx.label.name + "_" + name + "/" + filename + suffix + ".json"), + **options + ).messages) return catalogs def formatjs_message_collector_aspect( @@ -126,12 +140,38 @@ def formatjs_message_collector_aspect( for dependency in dependencies: for repository, sources in dependency.sources_by_repository.items(): groups.setdefault(repository, []).append(sources) - if uses_runtime and should_extract(target, ctx): + source_targets = _targets(ctx, source_attributes) + allowed = extraction_enabled and should_extract(target, ctx) + rejected = [] + direct = [] + if extraction_enabled and not allowed: + rejected = [dependency[DefaultInfo].files for dependency in source_targets if DefaultInfo in dependency] + if DefaultInfo in target: + rejected.append(target[DefaultInfo].files) + if uses_runtime and allowed: groups.setdefault(ctx.label.repo_name, []).extend([ dependency[DefaultInfo].files - for dependency in _targets(ctx, source_attributes) + for dependency in source_targets if DefaultInfo in dependency ]) + direct = [ + dependency[DefaultInfo].files + for dependency in source_targets + if DefaultInfo in dependency and FormatjsMessagesInfo not in dependency + ] + + # Ancestors can forward rejected files through DefaultInfo. Keep scope + # decisions lazy, while preserving independent allowed declarations. + scope_excluded = depset(transitive = rejected + [dependency.scope_excluded_srcs for dependency in dependencies]) + scope_direct = depset(transitive = direct + [dependency.scope_direct_srcs for dependency in dependencies]) + scope_owners = depset( + [ + dependency.label + for dependency in source_targets + if FormatjsMessagesInfo in dependency and dependency[FormatjsMessagesInfo].allows_sources + ] if uses_runtime and allowed else [], + transitive = [dependency.scope_allowed_owners for dependency in dependencies], + ) sources_by_repository = { repository: depset(transitive = sources) for repository, sources in groups.items() @@ -150,6 +190,9 @@ def formatjs_message_collector_aspect( name, sources_by_repository, covered, + scope_excluded, + scope_direct, + scope_owners, source_extensions, extraction_options, ), transitive = [explicit_catalogs]) @@ -160,6 +203,10 @@ def formatjs_message_collector_aspect( covered_srcs = covered, explicit_catalogs = explicit_catalogs, sources_by_repository = sources_by_repository, + scope_excluded_srcs = scope_excluded, + scope_direct_srcs = scope_direct, + scope_allowed_owners = scope_owners, + allows_sources = allowed, ), OutputGroupInfo(all_messages = catalogs), ] diff --git a/tests/auto_extract/Aa/messages.py b/tests/auto_extract/Aa/messages.py new file mode 100644 index 0000000..6394a8c --- /dev/null +++ b/tests/auto_extract/Aa/messages.py @@ -0,0 +1,2 @@ +from py_intl import define_message +message = define_message(default_message="First collision message") diff --git a/tests/auto_extract/BB/messages.py b/tests/auto_extract/BB/messages.py new file mode 100644 index 0000000..5104ccb --- /dev/null +++ b/tests/auto_extract/BB/messages.py @@ -0,0 +1,2 @@ +from py_intl import define_message +message = define_message(default_message="Second collision message") diff --git a/tests/auto_extract/BUILD.bazel b/tests/auto_extract/BUILD.bazel index 269804b..355dca7 100644 --- a/tests/auto_extract/BUILD.bazel +++ b/tests/auto_extract/BUILD.bazel @@ -1,7 +1,7 @@ load("@jq.bzl", "jq") load("@jq.bzl//jq:jq.bzl", "jq_test") load("//formatjs:defs.bzl", "formatjs_extract") -load(":fixtures.bzl", "adapter_catalogs", "alias_catalogs", "batch_catalogs", "batch_report", "catalogs", "collection_only_catalogs", "custom_action", "custom_catalogs", "graph", "ignored", "metadata", "report", "runtime", "scoped_catalogs", "selected_alias_catalogs", "source_chain", "unmatched_report") +load(":fixtures.bzl", "adapter_catalogs", "alias_catalogs", "batch_catalogs", "batch_report", "catalogs", "collection_only_catalogs", "custom_action", "custom_catalogs", "graph", "ignored", "metadata", "python_source", "report", "runtime", "scoped_catalogs", "selected_alias_catalogs", "source_chain", "unmatched_report") exports_files( ["foreign.py"], @@ -435,3 +435,100 @@ jq_test( file2 = "batch_catalog_count.json", filter1 = ".catalog_count", ) + +python_source(name = "skipped_generated") + +python_source(name = "allowed_generated") + +graph( + name = "skipped_generated_wrapper", + srcs = [":allowed_generated"], +) + +graph( + name = "scope_wrapper", + srcs = [ + ":skipped", + ":skipped_generated_wrapper", + ], +) + +graph( + name = "scope_wrapper_root", + srcs = [ + ":scope_wrapper", + ":skipped_generated", + ], +) + +scoped_catalogs( + name = "scope_wrapper_output", + target = ":scope_wrapper_root", +) + +jq( + name = "scope_wrapper_merged", + srcs = [":scope_wrapper_output"], + args = ["--slurp"], + filter = "add | [.[] | (.defaultMessage // .message)] | sort", +) + +jq_test( + name = "scope_wrapper_test", + file1 = ":scope_wrapper_merged", + file2 = "python.json", +) + +graph( + name = "scope_direct_root", + srcs = [ + "root.rs", + ":allowed_generated", + ":scope_wrapper", + ], +) + +scoped_catalogs( + name = "scope_direct_output", + target = ":scope_direct_root", +) + +jq( + name = "scope_direct_merged", + srcs = [":scope_direct_output"], + args = ["--slurp"], + filter = "add | [.[] | (.defaultMessage // .message)] | sort", +) + +jq_test( + name = "scope_direct_test", + file1 = ":scope_direct_merged", + file2 = "scope_direct.json", +) + +graph( + name = "collision_root", + srcs = [ + "Aa/messages.py", + "BB/messages.py", + ], + deps = [":runtime"], +) + +catalogs( + name = "collision_output", + target = ":collision_root", +) + +jq( + name = "collision_merged", + srcs = [":collision_output"], + args = ["--slurp"], + filter = "add | [.[] | (.defaultMessage // .message)] | sort", +) + +jq_test( + name = "output_collision_test", + file1 = ":collision_merged", + file2 = "collisions.json", +) diff --git a/tests/auto_extract/collisions.json b/tests/auto_extract/collisions.json new file mode 100644 index 0000000..31d186d --- /dev/null +++ b/tests/auto_extract/collisions.json @@ -0,0 +1 @@ +["First collision message", "Second collision message"] diff --git a/tests/auto_extract/fixtures.bzl b/tests/auto_extract/fixtures.bzl index 69192a9..e31468a 100644 --- a/tests/auto_extract/fixtures.bzl +++ b/tests/auto_extract/fixtures.bzl @@ -54,6 +54,9 @@ def _collect_at(_target, ctx): "shared_root", "with_ignored", "leaf_select", + "scope_wrapper_root", + "scope_direct_root", + "collision_root", ] messages_aspect = formatjs_message_collector_aspect( @@ -137,7 +140,7 @@ custom_action = rule( ) def _scope(_target, ctx): - return ctx.label.name != "skipped" + return ctx.label.name not in ["skipped", "skipped_generated", "skipped_generated_wrapper"] scoped_aspect = formatjs_message_collector_aspect( name = "scoped", diff --git a/tests/auto_extract/scope_direct.json b/tests/auto_extract/scope_direct.json new file mode 100644 index 0000000..8ead4f4 --- /dev/null +++ b/tests/auto_extract/scope_direct.json @@ -0,0 +1 @@ +["Automatic Python", "Automatic Rust", "allowed_generated"] From 93a4932c65385a89ee0f1bfb2d7394cd29549653 Mon Sep 17 00:00:00 2001 From: Long Ho Date: Fri, 6 Nov 2026 15:59:47 +0000 Subject: [PATCH 7/7] refactor: keep source planning private and simplify scope selection --- docs/automatic-extraction.md | 15 ++- formatjs/collector.bzl | 140 +++++++++----------- tests/auto_extract/configured/BUILD.bazel | 41 ++++++ tests/auto_extract/configured/expected.json | 1 + tests/auto_extract/configured/fixtures.bzl | 39 ++++++ tests/auto_extract/configured/input.py | 2 + tests/auto_extract/empty.json | 1 - tests/auto_extract/fixtures.bzl | 1 - tests/auto_extract/report.json | 1 - tests/auto_extract/unsupported.json | 1 - 10 files changed, 153 insertions(+), 89 deletions(-) create mode 100644 tests/auto_extract/configured/BUILD.bazel create mode 100644 tests/auto_extract/configured/expected.json create mode 100644 tests/auto_extract/configured/fixtures.bzl create mode 100644 tests/auto_extract/configured/input.py diff --git a/docs/automatic-extraction.md b/docs/automatic-extraction.md index 8258e51..a5ed3ef 100644 --- a/docs/automatic-extraction.md +++ b/docs/automatic-extraction.md @@ -79,10 +79,9 @@ source_catalogs = rule( ) ``` -At intermediate nodes, `catalogs` contains only explicit catalogs. The provider also -exposes `uses_runtime`, `covered_srcs`, `explicit_catalogs`, and source depsets grouped -by declaring repository. A predicate that always returns false gives metadata-only -propagation for adapters that materialize catalogs themselves. +The public provider has two fields: `catalogs` and `uses_runtime`. At intermediate +nodes, `catalogs` contains only explicit catalogs. A private provider carries source +plans and explicit coverage; consumers do not depend on that internal state. ## Catalogs and execution @@ -116,9 +115,11 @@ External dependencies can enable extraction and provide explicit catalogs withou having their implementation scanned. Override `should_extract(target, ctx)` to change that target boundary. Rejected sources stay excluded when ancestors forward them through source wrappers. Independent allowed declarations of the same source -remain eligible. These scope decisions stay in depsets until the extraction root. -Only files owned by a contributing target's repository -are eligible, including when source labels point into another repository. +remain eligible. Source plans retain configured identity, so allowing a generator +in one configuration cannot enable its rejected output in another. Plans stay in +depsets until the extraction root; each configured node is visited once there. +Only files owned by a contributing target's repository are eligible, including +when source labels point into another repository. `source_extensions` defaults to `ts`, `tsx`, `js`, `jsx`, `mjs`, `cjs`, `rs`, and `py`. `dependency_attributes` controls detection/collection edges; `source_attributes` diff --git a/formatjs/collector.bzl b/formatjs/collector.bzl index 2495e79..eb8a632 100644 --- a/formatjs/collector.bzl +++ b/formatjs/collector.bzl @@ -12,16 +12,11 @@ FormatjsMessagesInfo = provider( fields = { "catalogs": "Depset of original catalogs; no intermediate merges.", "uses_runtime": "Whether a configured dependency enables extraction.", - "covered_srcs": "Depset of source files represented by explicit extraction metadata.", - "explicit_catalogs": "Original explicit catalogs, excluding this aspect's extraction actions.", - "sources_by_repository": "Candidate source depsets grouped by declaring repository.", - "scope_excluded_srcs": "Files rejected by source scope, including forwarded outputs.", - "scope_direct_srcs": "Direct source files explicitly allowed by an opted-in target.", - "scope_allowed_owners": "Allowed source targets explicitly referenced by an opted-in target.", - "allows_sources": "Whether this target passes the source-scope predicate.", }, ) +_SourcePlansInfo = provider(fields = ["catalogs", "covered", "plan", "plans"]) + def _main_repository(_target, ctx): return ctx.label.repo_name == "" @@ -35,19 +30,38 @@ def _targets(ctx, attributes): def _source_path(source): return source.path -def _extract_catalogs(ctx, name, sources_by_repository, covered, scope_excluded, scope_direct, scope_owners, extensions, options): +def _source_files(plans, covered, extensions): + requested = {} + candidates = {} + direct = [] + owners = {} + + # Parents request source views before their children are visited. Each + # configured node is visited once, even through shared source wrappers. + for plan in plans.to_list()[::-1]: + if not plan.allowed or not (plan.enabled or plan.key in requested): + continue + if plan.key in requested: + owners[plan.key] = True + for source in plan.sources: + if plan.enabled: + candidates.setdefault(plan.key[0].repo_name, []).append(source.files) + if source.key == None: + direct.append(source.files) + else: + requested[source.key] = True + eligible = {source: True for source in depset(transitive = direct).to_list()} excluded = {source: True for source in covered.to_list()} - rejected = {source: True for source in scope_excluded.to_list()} - direct = {source: True for source in scope_direct.to_list()} - owners = {owner: True for owner in scope_owners.to_list()} - sources = [ + return [ source - for repository, candidates in sources_by_repository.items() - for source in candidates.to_list() + for repository, inputs in candidates.items() + for source in depset(transitive = inputs).to_list() if source.extension in extensions and source.owner.repo_name == repository and source not in excluded and ( - source not in rejected or source in direct or source.owner in owners + source in eligible or (source.owner, source.root.path) in owners ) ] + +def _extract_catalogs(ctx, name, sources, options): if not sources: return [] if not options.get("extract_source_location", False): @@ -116,66 +130,47 @@ def formatjs_message_collector_aspect( attributes = depset(dependency_attributes + source_attributes).to_list() def _impl(target, ctx): - dependencies = [ - dependency[FormatjsMessagesInfo] - for dependency in _targets(ctx, attributes) - if FormatjsMessagesInfo in dependency - ] + dependencies = _targets(ctx, attributes) + inherited = [dependency[_SourcePlansInfo] for dependency in dependencies if _SourcePlansInfo in dependency] runtime_targets = {runtime.label: True for runtime in ctx.attr._formatjs_runtime_targets} uses_runtime = ( target.label in runtime_targets or ctx.label in runtime_targets or (is_runtime != None and is_runtime(target, ctx)) or - any([dependency.uses_runtime for dependency in dependencies]) + any([dependency[FormatjsMessagesInfo].uses_runtime for dependency in dependencies if FormatjsMessagesInfo in dependency]) ) explicit_catalogs = depset( [target[FormatjsExtractInfo].messages] if FormatjsExtractInfo in target else [], - transitive = [dependency.explicit_catalogs for dependency in dependencies], + transitive = [dependency.catalogs for dependency in inherited], ) - covered_inputs = [dependency.covered_srcs for dependency in dependencies] - if FormatjsExtractInfo in target: - covered_inputs.append(getattr(target[FormatjsExtractInfo], "srcs", depset())) - covered = depset(transitive = covered_inputs) - groups = {} - for dependency in dependencies: - for repository, sources in dependency.sources_by_repository.items(): - groups.setdefault(repository, []).append(sources) - source_targets = _targets(ctx, source_attributes) - allowed = extraction_enabled and should_extract(target, ctx) - rejected = [] - direct = [] - if extraction_enabled and not allowed: - rejected = [dependency[DefaultInfo].files for dependency in source_targets if DefaultInfo in dependency] - if DefaultInfo in target: - rejected.append(target[DefaultInfo].files) - if uses_runtime and allowed: - groups.setdefault(ctx.label.repo_name, []).extend([ - dependency[DefaultInfo].files - for dependency in source_targets - if DefaultInfo in dependency - ]) - direct = [ - dependency[DefaultInfo].files - for dependency in source_targets - if DefaultInfo in dependency and FormatjsMessagesInfo not in dependency - ] - - # Ancestors can forward rejected files through DefaultInfo. Keep scope - # decisions lazy, while preserving independent allowed declarations. - scope_excluded = depset(transitive = rejected + [dependency.scope_excluded_srcs for dependency in dependencies]) - scope_direct = depset(transitive = direct + [dependency.scope_direct_srcs for dependency in dependencies]) - scope_owners = depset( - [ - dependency.label - for dependency in source_targets - if FormatjsMessagesInfo in dependency and dependency[FormatjsMessagesInfo].allows_sources - ] if uses_runtime and allowed else [], - transitive = [dependency.scope_allowed_owners for dependency in dependencies], + covered = depset( + transitive = [dependency.covered for dependency in inherited] + ( + [getattr(target[FormatjsExtractInfo], "srcs", depset())] if FormatjsExtractInfo in target else [] + ), + ) + plan = None + if extraction_enabled: + allowed = should_extract(target, ctx) + + # Labels alone conflate generators reached through transitions. + plan = struct( + key = (target.label, ctx.bin_dir.path), + allowed = allowed, + enabled = allowed and uses_runtime, + sources = tuple([ + struct( + key = dependency[_SourcePlansInfo].plan.key if _SourcePlansInfo in dependency else None, + files = dependency[DefaultInfo].files, + ) + for dependency in _targets(ctx, source_attributes) + if DefaultInfo in dependency + ]), + ) + plans = depset( + [plan] if plan != None else [], + transitive = [dependency.plans for dependency in inherited], + order = "postorder", ) - sources_by_repository = { - repository: depset(transitive = sources) - for repository, sources in groups.items() - } catalogs = explicit_catalogs if extraction_enabled: if collect_at != None: @@ -188,26 +183,15 @@ def formatjs_message_collector_aspect( catalogs = depset(_extract_catalogs( ctx, name, - sources_by_repository, - covered, - scope_excluded, - scope_direct, - scope_owners, - source_extensions, + _source_files(plans, covered, source_extensions), extraction_options, ), transitive = [explicit_catalogs]) return [ FormatjsMessagesInfo( catalogs = catalogs, uses_runtime = uses_runtime, - covered_srcs = covered, - explicit_catalogs = explicit_catalogs, - sources_by_repository = sources_by_repository, - scope_excluded_srcs = scope_excluded, - scope_direct_srcs = scope_direct, - scope_allowed_owners = scope_owners, - allows_sources = allowed, ), + _SourcePlansInfo(catalogs = explicit_catalogs, covered = covered, plan = plan, plans = plans), OutputGroupInfo(all_messages = catalogs), ] diff --git a/tests/auto_extract/configured/BUILD.bazel b/tests/auto_extract/configured/BUILD.bazel new file mode 100644 index 0000000..86b955c --- /dev/null +++ b/tests/auto_extract/configured/BUILD.bazel @@ -0,0 +1,41 @@ +load("@jq.bzl//jq:jq.bzl", "jq_test") +load(":fixtures.bzl", "catalogs", "generate", "graph", "wrapper") + +config_setting( + name = "off", + values = {"compilation_mode": "dbg"}, +) + +generate( + name = "source", + srcs = ["input.py"], + scope_allow = select({ + ":off": False, + "//conditions:default": True, + }), +) + +wrapper( + name = "off_wrapper", + srcs = [":source"], +) + +graph( + name = "root", + srcs = [ + ":off_wrapper", + ":source", + ], +) + +catalogs( + name = "output", + target = ":root", +) + +jq_test( + name = "configured_scope_test", + file1 = ":output", + file2 = "expected.json", + filter1 = "[.[] | (.defaultMessage // .message)]", +) diff --git a/tests/auto_extract/configured/expected.json b/tests/auto_extract/configured/expected.json new file mode 100644 index 0000000..8c6b88b --- /dev/null +++ b/tests/auto_extract/configured/expected.json @@ -0,0 +1 @@ +["Allowed"] diff --git a/tests/auto_extract/configured/fixtures.bzl b/tests/auto_extract/configured/fixtures.bzl new file mode 100644 index 0000000..362a346 --- /dev/null +++ b/tests/auto_extract/configured/fixtures.bzl @@ -0,0 +1,39 @@ +load("//formatjs:defs.bzl", "FormatjsMessagesInfo", "formatjs_message_collector_aspect") + +def _off_impl(_settings, _attr): + return {"//command_line_option:compilation_mode": "dbg"} + +_off = transition(implementation = _off_impl, inputs = [], outputs = ["//command_line_option:compilation_mode"]) + +def _gen(ctx): + out = ctx.actions.declare_file(ctx.label.name + ".py") + text = "Allowed" if ctx.attr.scope_allow else "Rejected" + ctx.actions.write(out, 'from py_intl import define_message\nmessage = define_message(default_message="' + text + '")\n') + return [DefaultInfo(files = depset([out]))] + +generate = rule(implementation = _gen, attrs = {"scope_allow": attr.bool(), "srcs": attr.label_list(allow_files = True)}) + +def _graph(ctx): + return [DefaultInfo(files = depset(transitive = [s[DefaultInfo].files for s in ctx.attr.srcs]))] + +graph = rule(implementation = _graph, attrs = {"srcs": attr.label_list()}) +wrapper = rule(implementation = _graph, attrs = { + "srcs": attr.label_list(cfg = _off), + "_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"), +}) + +def _runtime(_target, ctx): + return ctx.label.name == "root" + +def _scope(_target, ctx): + return getattr(ctx.rule.attr, "scope_allow", True) + +def _root(_target, ctx): + return ctx.label.name == "root" + +messages = formatjs_message_collector_aspect(is_runtime = _runtime, should_extract = _scope, collect_at = _root, extract_source_location = True) + +def _catalogs(ctx): + return [DefaultInfo(files = ctx.attr.target[FormatjsMessagesInfo].catalogs)] + +catalogs = rule(implementation = _catalogs, attrs = {"target": attr.label(aspects = [messages])}) diff --git a/tests/auto_extract/configured/input.py b/tests/auto_extract/configured/input.py new file mode 100644 index 0000000..bd45338 --- /dev/null +++ b/tests/auto_extract/configured/input.py @@ -0,0 +1,2 @@ +from py_intl import define_message +message = define_message(default_message="Generator input stays unscanned") diff --git a/tests/auto_extract/empty.json b/tests/auto_extract/empty.json index 1d17ce8..dd0c467 100644 --- a/tests/auto_extract/empty.json +++ b/tests/auto_extract/empty.json @@ -1,5 +1,4 @@ { "catalog_count": 0, - "sources": [], "uses_runtime": false } diff --git a/tests/auto_extract/fixtures.bzl b/tests/auto_extract/fixtures.bzl index e31468a..d3f801f 100644 --- a/tests/auto_extract/fixtures.bzl +++ b/tests/auto_extract/fixtures.bzl @@ -78,7 +78,6 @@ def _report_impl(ctx): output = ctx.actions.declare_file(ctx.label.name + ".json") ctx.actions.write(output, json.encode({ "catalog_count": len(info.catalogs.to_list()), - "sources": sorted([file.basename for file in info.covered_srcs.to_list()]), "uses_runtime": info.uses_runtime, })) return [DefaultInfo(files = depset([output]))] diff --git a/tests/auto_extract/report.json b/tests/auto_extract/report.json index 8891551..052aeca 100644 --- a/tests/auto_extract/report.json +++ b/tests/auto_extract/report.json @@ -1,5 +1,4 @@ { "catalog_count": 3, - "sources": ["legacy.tsx"], "uses_runtime": true } diff --git a/tests/auto_extract/unsupported.json b/tests/auto_extract/unsupported.json index a0179c0..bb20424 100644 --- a/tests/auto_extract/unsupported.json +++ b/tests/auto_extract/unsupported.json @@ -1,5 +1,4 @@ { "catalog_count": 0, - "sources": [], "uses_runtime": true }