From 82532035951f9aa05c137ab60059d623112814f4 Mon Sep 17 00:00:00 2001 From: Oliver Kriska Date: Wed, 10 Jun 2026 14:02:43 +0200 Subject: [PATCH 1/4] Add recompile-free extraction via persisted module attributes mix gettext.extract can only discover messages by force-recompiling the whole project, because extraction happens during macro expansion and the results only exist in the extractor agent for the duration of that compilation. On large projects this dominates extraction time. Implement the design proposed by @maennchen in #373: during normal compilation, when the current Mix environment is included in the new :extraction_environments gettext configuration (default [:dev]), the gettext macros persist each message into an accumulated, persisted @__gettext_messages__ attribute of the calling module, and backends persist a @__gettext_backend_module__ marker. A new experimental --from-attributes flag on mix gettext.extract then runs a normal incremental compile, reads the persisted entries back from the compiled BEAM files with :beam_lib (without loading modules), feeds them into the existing extractor agent, and reuses the unchanged Gettext.Extractor.pot_files/2 merge logic. Staleness needs no bookkeeping: changed files are recompiled by the incremental compile and get fresh attributes, and deleted files' beams are pruned by the compiler. --check-up-to-date composes with the new flag and gets the same speedup. Environments not listed in :extraction_environments (such as :prod) persist nothing, so release artifacts are unaffected. If the scan finds no persisted messages or backends at all, the task raises with instructions instead of silently producing empty output. The force-recompile path is untouched and remains the default. Old-path and new-path POT output is byte-identical, covered by tests including macro-generated messages (gettext calls injected into modules that do not literally mention them) and plural/context/comment/noop variants. --- CHANGELOG.md | 6 + lib/gettext/compiler.ex | 2 + lib/gettext/extractor.ex | 160 +++++++++++- lib/gettext/macros.ex | 44 ++-- lib/mix/tasks/gettext.extract.ex | 80 +++++- .../gettext.extract_from_attributes_test.exs | 234 ++++++++++++++++++ 6 files changed, 500 insertions(+), 26 deletions(-) create mode 100644 test/mix/tasks/gettext.extract_from_attributes_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cd9c7c..72e2b33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ * Parallelize per-locale merging in `mix gettext.merge`. + * Add experimental `--from-attributes` flag to `mix gettext.extract`: messages + are persisted as module attributes during normal compilation (in the + environments listed in the `:extraction_environments` gettext configuration, + `[:dev]` by default) and read back from the compiled BEAM files, so + extraction no longer needs to force-recompile the project. + ## v1.0.2 * Only skip manifest removal on Elixir v1.19.3+ diff --git a/lib/gettext/compiler.ex b/lib/gettext/compiler.ex index 0598d49..24e9204 100644 --- a/lib/gettext/compiler.ex +++ b/lib/gettext/compiler.ex @@ -38,6 +38,8 @@ defmodule Gettext.Compiler do interpolation = opts[:interpolation] || Gettext.Interpolation.Default + Gettext.Extractor.persist_backend_marker(env) + quote do @behaviour Gettext.Backend diff --git a/lib/gettext/extractor.ex b/lib/gettext/extractor.ex index 284b794..e1d02f8 100644 --- a/lib/gettext/extractor.ex +++ b/lib/gettext/extractor.ex @@ -21,6 +21,9 @@ defmodule Gettext.Extractor do @extracted_messages_flag "elixir-autogen" + @persisted_messages_attribute :__gettext_messages__ + @persisted_backend_attribute :__gettext_backend_module__ + @new_pot_comment String.split( """ # This file is a PO Template file. @@ -69,14 +72,20 @@ defmodule Gettext.Extractor do Note that this function doesn't perform any operation on the filesystem. """ @spec extract( - Macro.Env.t(), + Macro.Env.t() | {file :: String.t(), line :: pos_integer}, backend :: module, domain :: binary | :default, msgctxt :: binary, id :: binary | {binary, binary}, extracted_comments :: [binary] ) :: :ok + def extract(caller_or_location, backend, domain, msgctxt, id, extracted_comments) + def extract(%Macro.Env{} = caller, backend, domain, msgctxt, id, extracted_comments) do + extract({caller.file, caller.line}, backend, domain, msgctxt, id, extracted_comments) + end + + def extract({file, line}, backend, domain, msgctxt, id, extracted_comments) do format_flag = backend.__gettext__(:interpolation).message_format() domain = @@ -89,8 +98,8 @@ defmodule Gettext.Extractor do create_message_struct( id, msgctxt, - caller.file, - caller.line, + file, + line, extracted_comments, format_flag ) @@ -98,6 +107,151 @@ defmodule Gettext.Extractor do ExtractorAgent.add_message(backend, domain, message) end + @doc """ + Tells whether gettext macros should persist the messages they extract as + module attributes in the module that is currently being compiled. + + This is true when there is a module being compiled (`env.module` is not + `nil`) and the current Mix environment is included in the + `:extraction_environments` gettext configuration (defaults to `[:dev]`). + Outside of Mix (for example, when modules are compiled at runtime in a + release) this is always false. + """ + @spec persisting_to_attributes?(Macro.Env.t()) :: boolean + def persisting_to_attributes?(%Macro.Env{module: nil}), do: false + def persisting_to_attributes?(%Macro.Env{}), do: extraction_environment?() + + defp extraction_environment? do + cond do + not Code.ensure_loaded?(Mix) -> + false + + is_nil(Mix.Project.get()) -> + false + + true -> + gettext_config = Mix.Project.config()[:gettext] || [] + + extraction_environments = + gettext_config[:extraction_environments] || + Application.get_env(:gettext, :extraction_environments) || + [:dev] + + Mix.env() in extraction_environments + end + end + + @doc """ + Persists a message in a module attribute of the module currently being + compiled, so that it can be read back from the compiled BEAM file by + `fill_from_compiled_beams/1` without recompiling the module. + """ + @spec persist_message( + Macro.Env.t(), + backend :: module, + domain :: binary | :default, + msgctxt :: binary, + id :: binary | {binary, binary}, + extracted_comments :: [binary] + ) :: :ok + def persist_message(%Macro.Env{module: module} = env, backend, domain, msgctxt, id, comments) + when not is_nil(module) do + unless Module.has_attribute?(module, @persisted_messages_attribute) do + Module.register_attribute(module, @persisted_messages_attribute, + accumulate: true, + persist: true + ) + end + + {msgid, msgid_plural} = + case id do + {msgid, msgid_plural} -> {msgid, msgid_plural} + msgid when is_binary(msgid) -> {msgid, nil} + end + + Module.put_attribute(module, @persisted_messages_attribute, %{ + backend: backend, + domain: domain, + msgctxt: msgctxt, + msgid: msgid, + msgid_plural: msgid_plural, + file: env.file, + line: env.line, + comments: comments + }) + + :ok + end + + @doc """ + Persists a marker attribute in the Gettext backend module currently being + compiled, so that `fill_from_compiled_beams/1` can find all backends + without recompiling. + """ + @spec persist_backend_marker(Macro.Env.t()) :: :ok + def persist_backend_marker(%Macro.Env{} = env) do + if persisting_to_attributes?(env) do + Module.register_attribute(env.module, @persisted_backend_attribute, persist: true) + Module.put_attribute(env.module, @persisted_backend_attribute, true) + end + + :ok + end + + @doc """ + Reads messages and backends persisted as module attributes from all the + BEAM files in `compile_path` and stores them in the extractor agent, as if + they had just been extracted during compilation. + + Returns `{backends_found, messages_found}`. + """ + @spec fill_from_compiled_beams(Path.t()) :: + {backends_found :: non_neg_integer, messages_found :: non_neg_integer} + def fill_from_compiled_beams(compile_path) do + compile_path + |> Path.join("*.beam") + |> Path.wildcard() + |> Enum.reduce({0, 0}, fn beam_path, {backends_acc, messages_acc} -> + {backends, messages} = fill_from_beam(beam_path) + {backends_acc + backends, messages_acc + messages} + end) + end + + defp fill_from_beam(beam_path) do + case :beam_lib.chunks(String.to_charlist(beam_path), [:attributes]) do + {:ok, {module, [attributes: attributes]}} -> + backend? = attributes[@persisted_backend_attribute] == [true] + + if backend? do + ExtractorAgent.add_backend(module) + end + + entries = attributes[@persisted_messages_attribute] || [] + + Enum.each(entries, fn entry -> + id = + case entry do + %{msgid_plural: nil, msgid: msgid} -> msgid + %{msgid_plural: msgid_plural, msgid: msgid} -> {msgid, msgid_plural} + end + + extract( + {entry.file, entry.line}, + entry.backend, + entry.domain, + entry.msgctxt, + id, + entry.comments + ) + end) + + {if(backend?, do: 1, else: 0), length(entries)} + + {:error, :beam_lib, _reason} -> + {0, 0} + end + end + @doc """ Returns a list of POT files based on the results of the extraction. diff --git a/lib/gettext/macros.ex b/lib/gettext/macros.ex index f950ac6..4e324de 100644 --- a/lib/gettext/macros.ex +++ b/lib/gettext/macros.ex @@ -596,16 +596,7 @@ defmodule Gettext.Macros do msgid = expand_to_binary(msgid, "msgid", env) msgctxt = expand_to_binary(msgctxt, "msgctxt", env) - if Extractor.extracting?() do - Extractor.extract( - env, - backend, - domain, - msgctxt, - msgid, - get_and_flush_extracted_comments() - ) - end + extract_message(env, backend, domain, msgctxt, msgid) msgid end @@ -617,20 +608,33 @@ defmodule Gettext.Macros do msgctxt = expand_to_binary(msgctxt, "msgctxt", env) msgid_plural = expand_to_binary(msgid_plural, "msgid_plural", env) - if Extractor.extracting?() do - Extractor.extract( - env, - backend, - domain, - msgctxt, - {msgid, msgid_plural}, - get_and_flush_extracted_comments() - ) - end + extract_message(env, backend, domain, msgctxt, {msgid, msgid_plural}) {msgid, msgid_plural} end + defp extract_message(env, backend, domain, msgctxt, id) do + # Extractor.extracting?() returns nil (not false) when the extractor + # agent is not running, such as when compiling a dependency that uses + # Gettext while the :gettext application is not started. + extracting? = Extractor.extracting?() + persisting? = Extractor.persisting_to_attributes?(env) + + if extracting? || persisting? do + comments = get_and_flush_extracted_comments() + + if extracting? do + Extractor.extract(env, backend, domain, msgctxt, id, comments) + end + + if persisting? do + Extractor.persist_message(env, backend, domain, msgctxt, id, comments) + end + end + + :ok + end + defp singular_extract_and_translate(env, backend, domain, msgctxt, msgid, bindings) do domain = expand_domain(domain, env) msgid = extract_singular_translation(env, backend, domain, msgctxt, msgid) diff --git a/lib/mix/tasks/gettext.extract.ex b/lib/mix/tasks/gettext.extract.ex index 2988649..57c1388 100644 --- a/lib/mix/tasks/gettext.extract.ex +++ b/lib/mix/tasks/gettext.extract.ex @@ -55,9 +55,40 @@ defmodule Mix.Tasks.Gettext.Extract do mix gettext.extract --merge --no-fuzzy ``` + ## Extraction Without Recompiling (Experimental) + + By default, this task extracts messages by **force-recompiling** the whole + project, because extraction happens during the expansion of the Gettext + macros. As an experimental alternative, the `--from-attributes` flag reads + messages back from the compiled BEAM files instead: + + ```bash + mix gettext.extract --from-attributes + ``` + + For this to work, messages must have been persisted as module attributes + during normal compilation. This happens automatically when the current Mix + environment is included in the `:extraction_environments` Gettext + configuration in your `mix.exs`, which defaults to `[:dev]`: + + def project do + [ + # ... + gettext: [extraction_environments: [:dev]] + ] + end + + With this flag, the task only runs a normal **incremental** compilation + (changed files are recompiled and get fresh attributes; unchanged BEAM + files already carry theirs), then scans the compiled BEAM files. Since + environments outside `:extraction_environments` (such as `:prod`) never + persist these attributes, release artifacts are unaffected. + + `--from-attributes` can be combined with `--merge` and `--check-up-to-date`. + """ - @switches [merge: :boolean, check_up_to_date: :boolean] + @switches [merge: :boolean, check_up_to_date: :boolean, from_attributes: :boolean] @impl true def run(args) do @@ -65,7 +96,7 @@ defmodule Mix.Tasks.Gettext.Extract do _ = Mix.Project.get!() mix_config = Mix.Project.config() {opts, _} = OptionParser.parse!(args, switches: @switches) - pot_files = extract(mix_config[:app], mix_config[:gettext] || []) + pot_files = extract(mix_config[:app], mix_config[:gettext] || [], opts) if opts[:check_up_to_date] do run_up_to_date_check(pot_files) @@ -103,7 +134,15 @@ defmodule Mix.Tasks.Gettext.Extract do end end - defp extract(app, gettext_config) do + defp extract(app, gettext_config, opts) do + if opts[:from_attributes] do + extract_from_attributes(app, gettext_config) + else + extract_via_recompilation(app, gettext_config) + end + end + + defp extract_via_recompilation(app, gettext_config) do Gettext.Extractor.enable() force_compile() Gettext.Extractor.pot_files(app, gettext_config) @@ -111,6 +150,41 @@ defmodule Mix.Tasks.Gettext.Extract do Gettext.Extractor.disable() end + defp extract_from_attributes(app, gettext_config) do + incremental_compile() + + {backends, messages} = + Gettext.Extractor.fill_from_compiled_beams(Mix.Project.compile_path()) + + if backends == 0 and messages == 0 do + Mix.raise(""" + mix gettext.extract --from-attributes found no persisted Gettext messages \ + or backends in #{Path.relative_to_cwd(Mix.Project.compile_path())}. + + Messages are persisted to module attributes during normal compilation only \ + when the current Mix environment is included in the :extraction_environments \ + Gettext configuration (which defaults to [:dev]). The current environment \ + is #{inspect(Mix.env())}. + + If you just updated Gettext or changed this configuration, force a recompile \ + so that up-to-date modules get their attributes written: + + mix compile --force + """) + end + + Gettext.Extractor.pot_files(app, gettext_config) + end + + defp incremental_compile do + # Same reenabling dance as force_compile/0, but without forcing: if + # "compile" already ran in this VM, reenabling "compile.elixir" lets us + # still pick up source changes incrementally. + Mix.Task.reenable("compile.elixir") + Mix.Task.run("compile", []) + Mix.Task.run("compile.elixir", []) + end + defp force_compile do # For old Elixir versions, we have to clean the manifest, # otherwise we are forced to compile all dependencies. diff --git a/test/mix/tasks/gettext.extract_from_attributes_test.exs b/test/mix/tasks/gettext.extract_from_attributes_test.exs new file mode 100644 index 0000000..835f262 --- /dev/null +++ b/test/mix/tasks/gettext.extract_from_attributes_test.exs @@ -0,0 +1,234 @@ +defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do + use ExUnit.Case + + import ExUnit.CaptureIO + import GettextTest.MixProjectHelpers + + @moduletag :tmp_dir + + setup_all do + # To suppress the `redefining module MyApp` warnings for the test modules + Code.compiler_options(ignore_module_conflict: true) + :ok + end + + setup do + # Mix task invocation state and loaded fixture modules are global to the + # test VM; reset both so that each test compiles its own project and + # resolves backends against its own modules. + for task <- ~w(compile compile.all compile.elixir compile.app loadpaths) do + Mix.Task.reenable(task) + end + + # Fully unload fixture modules: with lazy module loading (Elixir 1.19+), + # a stale MyApp.Gettext loaded by a previous test would otherwise keep + # answering __gettext__/1 with the previous test's configuration. + for module <- [MyApp.Gettext, MyApp, MyApp.Labels, MyApp.Consumer] do + :code.purge(module) + :code.delete(module) + :code.purge(module) + end + + # Also drop previous test projects' ebin dirs from the code path + # (in_project is called with prune_code_paths: false), so that the + # unloaded fixture modules cannot be lazily re-loaded from a previous + # test's _build. + tmp_root = Path.join(File.cwd!(), "tmp") + + for path <- :code.get_path(), + path_string = List.to_string(path), + String.starts_with?(path_string, tmp_root) do + :code.del_path(path) + end + + # Drain extractor state left behind by other test files (their fixture + # projects use the same MyApp.Gettext module name, so leftover messages + # would be popped into this test's POT files). + Agent.update(Gettext.ExtractorAgent, fn state -> + %{state | messages: %{}, backends: []} + end) + + :ok + end + + @fixture_source """ + defmodule MyApp.Gettext do + use Gettext.Backend, otp_app: %APP% + end + + defmodule MyApp do + use Gettext, backend: MyApp.Gettext + + def singular(), do: gettext("hello") + def with_domain(), do: dgettext("my_domain", "domain hello") + def with_context(), do: pgettext("a context", "ctx hello") + + def plural(n), do: ngettext("one item", "%{count} items", n) + + gettext_comment("a comment for translators") + def with_comment(), do: gettext("commented hello") + + def noop(), do: gettext_noop("noop hello") + end + """ + + defp write_fixture(context) do + source = String.replace(@fixture_source, "%APP%", inspect(context.test)) + write_file(context, "lib/my_app.ex", source) + end + + test "extracts the same POT files as the recompilation path", + %{test: test, tmp_dir: tmp_dir} = context do + create_test_mix_file(context, extraction_environments: [:test]) + write_fixture(context) + + # Reference output: the existing force-recompile path. + capture_io(fn -> + in_project(test, tmp_dir, fn _module -> run([]) end) + end) + + reference_default = read_file(context, "priv/gettext/default.pot") + reference_domain = read_file(context, "priv/gettext/my_domain.pot") + + assert reference_default =~ ~s(msgid "hello") + assert reference_default =~ ~s(msgid "one item") + assert reference_default =~ ~s(msgid_plural "%{count} items") + assert reference_default =~ ~s(msgctxt "a context") + assert reference_default =~ "#. a comment for translators" + assert reference_default =~ ~s(msgid "noop hello") + assert reference_domain =~ ~s(msgid "domain hello") + + # On an up-to-date project, the attribute path must consider the + # POT files written by the recompilation path unchanged. + output = + capture_io(fn -> + in_project(test, tmp_dir, fn _module -> run(["--from-attributes"]) end) + end) + + refute output =~ "Extracted" + assert read_file(context, "priv/gettext/default.pot") == reference_default + assert read_file(context, "priv/gettext/my_domain.pot") == reference_domain + + # From scratch (no POT files), the attribute path must rebuild + # byte-identical POT files without recompiling. + File.rm_rf!(Path.join(tmp_dir, "priv/gettext")) + + output = + capture_io(fn -> + in_project(test, tmp_dir, fn _module -> run(["--from-attributes"]) end) + end) + + assert output =~ "Extracted priv/gettext/default.pot" + assert output =~ "Extracted priv/gettext/my_domain.pot" + assert read_file(context, "priv/gettext/default.pot") == reference_default + assert read_file(context, "priv/gettext/my_domain.pot") == reference_domain + end + + test "persists messages injected by macros into the consuming module", + %{test: test, tmp_dir: tmp_dir} = context do + create_test_mix_file(context, extraction_environments: [:test]) + + write_file(context, "lib/my_app.ex", """ + defmodule MyApp.Gettext do + use Gettext.Backend, otp_app: #{inspect(test)} + end + + defmodule MyApp.Labels do + defmacro def_label(name, msgid) do + quote do + def unquote(name)(), do: gettext(unquote(msgid)) + end + end + end + """) + + # This module does not contain the literal string "gettext\(" for its + # message: the call is injected by the macro at expansion time. + write_file(context, "lib/consumer.ex", """ + defmodule MyApp.Consumer do + use Gettext, backend: MyApp.Gettext + require MyApp.Labels + + MyApp.Labels.def_label(:active, "macro generated label") + end + """) + + capture_io(fn -> + in_project(test, tmp_dir, fn _module -> run(["--from-attributes"]) end) + end) + + pot = read_file(context, "priv/gettext/default.pot") + assert pot =~ ~s(msgid "macro generated label") + assert pot =~ "#: lib/consumer.ex:5" + end + + test "--check-up-to-date passes when fresh and fails after a source change", + %{test: test, tmp_dir: tmp_dir} = context do + create_test_mix_file(context, extraction_environments: [:test]) + write_fixture(context) + + capture_io(fn -> + in_project(test, tmp_dir, fn _module -> + run(["--from-attributes"]) + end) + + in_project(test, tmp_dir, fn _module -> + run(["--from-attributes", "--check-up-to-date"]) + end) + end) + + # Change a message: the incremental compile inside the task must pick up + # the changed file, refresh its attributes, and detect the stale POT. + write_fixture_with_changed_msgid(context) + + capture_io(fn -> + assert_raise Mix.Error, ~r/mix gettext\.extract failed due to --check-up-to-date/, fn -> + in_project(test, tmp_dir, fn _module -> + run(["--from-attributes", "--check-up-to-date"]) + end) + end + end) + end + + test "no attributes are persisted outside extraction environments", + %{test: test, tmp_dir: tmp_dir} = context do + # Default :extraction_environments is [:dev]; tests run in :test, so + # nothing must be persisted, mirroring a :prod release compile. + create_test_mix_file(context) + write_fixture(context) + + # The task compiles the project, finds no persisted attributes, and + # refuses to silently produce empty output. + capture_io(fn -> + assert_raise Mix.Error, ~r/found no persisted Gettext messages/, fn -> + in_project(test, tmp_dir, fn _module -> + run(["--from-attributes"]) + end) + end + end) + + beams = Path.wildcard(Path.join(tmp_dir, "_build/**/Elixir.MyApp*.beam")) + assert beams != [] + + for beam <- beams do + {:ok, {_module, [attributes: attributes]}} = + :beam_lib.chunks(String.to_charlist(beam), [:attributes]) + + refute Keyword.has_key?(attributes, :__gettext_messages__) + refute Keyword.has_key?(attributes, :__gettext_backend_module__) + end + end + + defp write_fixture_with_changed_msgid(context) do + source = + @fixture_source + |> String.replace("%APP%", inspect(context.test)) + |> String.replace(~s{gettext("hello")}, ~s{gettext("hello changed")}) + + write_file(context, "lib/my_app.ex", source) + end + + defp run(args) do + Mix.Tasks.Gettext.Extract.run(args) + end +end From 25e0aee5ec648bb723d0d9fe1f403a019742fd1e Mon Sep 17 00:00:00 2001 From: Oliver Kriska Date: Wed, 10 Jun 2026 15:11:08 +0200 Subject: [PATCH 2/4] Harden attribute extraction and expand its test coverage Address review findings on the --from-attributes prototype: * Validate the shape of persisted message entries when scanning BEAM files, skipping malformed ones instead of crashing the scan with a CaseClauseError (an unrelated module could carry an attribute with the same name, and future format changes need a safe path). * Reenable "compile" and friends in the incremental compile, so that extraction still picks up source changes when the compile tasks already ran in the current VM (for example, through a task alias). * Guard the backend marker registration with Module.has_attribute?/2, matching persist_message/6. Test improvements: assert fixed content on the rebuilt POT (not only equality with the recompilation path), reset the extractor agent's full initial state between tests, make reference line assertions robust to fixture edits, pin the stale-POT filename in the --check-up-to-date assertion, and restore Code.compiler_options/1 after the suite. New coverage: _with_backend macro variants, reference merging when two modules share a msgid, --from-attributes combined with --merge, custom :priv backends, and extraction across umbrella apps (each app gets its own POT through the task's recursion). --- lib/gettext/extractor.ex | 50 ++-- lib/mix/tasks/gettext.extract.ex | 12 +- .../gettext.extract_from_attributes_test.exs | 216 +++++++++++++++++- 3 files changed, 246 insertions(+), 32 deletions(-) diff --git a/lib/gettext/extractor.ex b/lib/gettext/extractor.ex index e1d02f8..0fa24fd 100644 --- a/lib/gettext/extractor.ex +++ b/lib/gettext/extractor.ex @@ -156,7 +156,7 @@ defmodule Gettext.Extractor do ) :: :ok def persist_message(%Macro.Env{module: module} = env, backend, domain, msgctxt, id, comments) when not is_nil(module) do - unless Module.has_attribute?(module, @persisted_messages_attribute) do + if not Module.has_attribute?(module, @persisted_messages_attribute) do Module.register_attribute(module, @persisted_messages_attribute, accumulate: true, persist: true @@ -191,7 +191,10 @@ defmodule Gettext.Extractor do @spec persist_backend_marker(Macro.Env.t()) :: :ok def persist_backend_marker(%Macro.Env{} = env) do if persisting_to_attributes?(env) do - Module.register_attribute(env.module, @persisted_backend_attribute, persist: true) + if not Module.has_attribute?(env.module, @persisted_backend_attribute) do + Module.register_attribute(env.module, @persisted_backend_attribute, persist: true) + end + Module.put_attribute(env.module, @persisted_backend_attribute, true) end @@ -227,31 +230,38 @@ defmodule Gettext.Extractor do end entries = attributes[@persisted_messages_attribute] || [] + messages = Enum.count(entries, &fill_message_from_entry/1) - Enum.each(entries, fn entry -> - id = - case entry do - %{msgid_plural: nil, msgid: msgid} -> msgid - %{msgid_plural: msgid_plural, msgid: msgid} -> {msgid, msgid_plural} - end - - extract( - {entry.file, entry.line}, - entry.backend, - entry.domain, - entry.msgctxt, - id, - entry.comments - ) - end) - - {if(backend?, do: 1, else: 0), length(entries)} + {if(backend?, do: 1, else: 0), messages} {:error, :beam_lib, _reason} -> {0, 0} end end + # Entries are validated structurally so that a malformed attribute (for + # example, one written by an unrelated module that happens to use the same + # attribute name) is skipped instead of crashing the whole scan. + defp fill_message_from_entry(%{ + backend: backend, + domain: domain, + msgctxt: msgctxt, + msgid: msgid, + msgid_plural: msgid_plural, + file: file, + line: line, + comments: comments + }) + when is_atom(backend) and is_binary(msgid) and + (is_binary(msgid_plural) or is_nil(msgid_plural)) and + is_binary(file) and is_integer(line) and is_list(comments) do + id = if msgid_plural, do: {msgid, msgid_plural}, else: msgid + extract({file, line}, backend, domain, msgctxt, id, comments) + true + end + + defp fill_message_from_entry(_malformed), do: false + @doc """ Returns a list of POT files based on the results of the extraction. diff --git a/lib/mix/tasks/gettext.extract.ex b/lib/mix/tasks/gettext.extract.ex index 57c1388..45f57c2 100644 --- a/lib/mix/tasks/gettext.extract.ex +++ b/lib/mix/tasks/gettext.extract.ex @@ -177,10 +177,16 @@ defmodule Mix.Tasks.Gettext.Extract do end defp incremental_compile do - # Same reenabling dance as force_compile/0, but without forcing: if - # "compile" already ran in this VM, reenabling "compile.elixir" lets us - # still pick up source changes incrementally. + # A plain incremental compile, with one wrinkle: "compile" and the + # compilers it runs may already have been invoked in this VM (for + # example, through a task alias), in which case running them again would + # be a no-op unless they are reenabled first. The trailing explicit + # "compile.elixir" run is a no-op when "compile" just ran it, and covers + # the case where a custom "compile" alias does not. + Mix.Task.reenable("compile") + Mix.Task.reenable("compile.all") Mix.Task.reenable("compile.elixir") + Mix.Task.reenable("compile.app") Mix.Task.run("compile", []) Mix.Task.run("compile.elixir", []) end diff --git a/test/mix/tasks/gettext.extract_from_attributes_test.exs b/test/mix/tasks/gettext.extract_from_attributes_test.exs index 835f262..614525e 100644 --- a/test/mix/tasks/gettext.extract_from_attributes_test.exs +++ b/test/mix/tasks/gettext.extract_from_attributes_test.exs @@ -1,4 +1,6 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do + # async: false (default) is required: these tests mutate global Mix task + # invocation state, the code path, and Code.compiler_options/1. use ExUnit.Case import ExUnit.CaptureIO @@ -6,9 +8,20 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do @moduletag :tmp_dir + @fixture_modules [ + MyApp.Gettext, + MyApp, + MyApp.Labels, + MyApp.Consumer, + MyApp.Other, + MyApp.WithBackend + ] + setup_all do # To suppress the `redefining module MyApp` warnings for the test modules + previous = Code.compiler_options()[:ignore_module_conflict] Code.compiler_options(ignore_module_conflict: true) + on_exit(fn -> Code.compiler_options(ignore_module_conflict: previous) end) :ok end @@ -23,7 +36,7 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do # Fully unload fixture modules: with lazy module loading (Elixir 1.19+), # a stale MyApp.Gettext loaded by a previous test would otherwise keep # answering __gettext__/1 with the previous test's configuration. - for module <- [MyApp.Gettext, MyApp, MyApp.Labels, MyApp.Consumer] do + for module <- @fixture_modules do :code.purge(module) :code.delete(module) :code.purge(module) @@ -32,7 +45,8 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do # Also drop previous test projects' ebin dirs from the code path # (in_project is called with prune_code_paths: false), so that the # unloaded fixture modules cannot be lazily re-loaded from a previous - # test's _build. + # test's _build. Must run OUTSIDE in_project: it relies on File.cwd!() + # being the repo root. tmp_root = Path.join(File.cwd!(), "tmp") for path <- :code.get_path(), @@ -43,9 +57,11 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do # Drain extractor state left behind by other test files (their fixture # projects use the same MyApp.Gettext module name, so leftover messages - # would be popped into this test's POT files). - Agent.update(Gettext.ExtractorAgent, fn state -> - %{state | messages: %{}, backends: []} + # would be popped into this test's POT files). Reset the full initial + # state, including extracting?, in case a previous test died between + # enable() and disable(). + Agent.update(Gettext.ExtractorAgent, fn _state -> + %{messages: %{}, backends: [], extracting?: false} end) :ok @@ -120,7 +136,15 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do assert output =~ "Extracted priv/gettext/default.pot" assert output =~ "Extracted priv/gettext/my_domain.pot" - assert read_file(context, "priv/gettext/default.pot") == reference_default + + # Not only equal to the reference, but independently containing the + # expected content (guards against a bug shared by both paths). + rebuilt_default = read_file(context, "priv/gettext/default.pot") + assert rebuilt_default =~ ~s(msgid "hello") + assert rebuilt_default =~ ~s(msgid_plural "%{count} items") + assert rebuilt_default =~ "#. a comment for translators" + + assert rebuilt_default == reference_default assert read_file(context, "priv/gettext/my_domain.pot") == reference_domain end @@ -159,7 +183,115 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do pot = read_file(context, "priv/gettext/default.pot") assert pot =~ ~s(msgid "macro generated label") - assert pot =~ "#: lib/consumer.ex:5" + assert pot =~ ~r{#: lib/consumer\.ex:\d+} + end + + test "extracts messages from the _with_backend macros", + %{test: test, tmp_dir: tmp_dir} = context do + create_test_mix_file(context, extraction_environments: [:test]) + + write_file(context, "lib/my_app.ex", """ + defmodule MyApp.Gettext do + use Gettext.Backend, otp_app: #{inspect(test)} + end + + defmodule MyApp.WithBackend do + require Gettext.Macros + + def singular(), do: Gettext.Macros.gettext_with_backend(MyApp.Gettext, "wb hello") + + def plural(n) do + Gettext.Macros.ngettext_with_backend(MyApp.Gettext, "wb one", "wb many", n) + end + + def noop(), do: Gettext.Macros.dgettext_noop_with_backend(MyApp.Gettext, "wb_domain", "wb noop") + end + """) + + capture_io(fn -> + in_project(test, tmp_dir, fn _module -> run(["--from-attributes"]) end) + end) + + default_pot = read_file(context, "priv/gettext/default.pot") + assert default_pot =~ ~s(msgid "wb hello") + assert default_pot =~ ~s(msgid "wb one") + assert default_pot =~ ~s(msgid_plural "wb many") + assert read_file(context, "priv/gettext/wb_domain.pot") =~ ~s(msgid "wb noop") + end + + test "merges references when two modules use the same msgid", + %{test: test, tmp_dir: tmp_dir} = context do + create_test_mix_file(context, extraction_environments: [:test]) + + write_file(context, "lib/my_app.ex", """ + defmodule MyApp.Gettext do + use Gettext.Backend, otp_app: #{inspect(test)} + end + + defmodule MyApp do + use Gettext, backend: MyApp.Gettext + def shared(), do: gettext("shared message") + end + """) + + write_file(context, "lib/other.ex", """ + defmodule MyApp.Other do + use Gettext, backend: MyApp.Gettext + def shared(), do: gettext("shared message") + end + """) + + capture_io(fn -> + in_project(test, tmp_dir, fn _module -> run(["--from-attributes"]) end) + end) + + pot = read_file(context, "priv/gettext/default.pot") + + # One message, references from both BEAM files merged. + assert length(String.split(pot, ~s(msgid "shared message"))) == 2 + assert pot =~ ~r{#: lib/my_app\.ex:\d+} + assert pot =~ ~r{#: lib/other\.ex:\d+} + end + + test "--from-attributes combined with --merge updates PO files", + %{test: test, tmp_dir: tmp_dir} = context do + create_test_mix_file(context, extraction_environments: [:test]) + write_fixture(context) + + write_file(context, "priv/gettext/it/LC_MESSAGES/default.po", "") + + capture_io(fn -> + in_project(test, tmp_dir, fn _module -> run(["--from-attributes", "--merge"]) end) + end) + + po = read_file(context, "priv/gettext/it/LC_MESSAGES/default.po") + assert po =~ ~s(msgid "hello") + assert po =~ ~s(msgid "one item") + end + + test "respects a custom :priv option on the backend", + %{test: test, tmp_dir: tmp_dir} = context do + create_test_mix_file(context, extraction_environments: [:test]) + + write_file(context, "lib/my_app.ex", """ + defmodule MyApp.Gettext do + use Gettext.Backend, otp_app: #{inspect(test)}, priv: "priv/custom_gettext" + end + + defmodule MyApp do + use Gettext, backend: MyApp.Gettext + def custom(), do: gettext("custom priv hello") + end + """) + + capture_io(fn -> + in_project(test, tmp_dir, fn _module -> run(["--from-attributes"]) end) + end) + + assert read_file(context, "priv/custom_gettext/default.pot") =~ + ~s(msgid "custom priv hello") + + refute File.exists?(Path.join(tmp_dir, "priv/gettext/default.pot")) end test "--check-up-to-date passes when fresh and fails after a source change", @@ -181,8 +313,11 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do # the changed file, refresh its attributes, and detect the stale POT. write_fixture_with_changed_msgid(context) + expected_error = + ~r{failed due to --check-up-to-date.*priv/gettext/default\.pot}s + capture_io(fn -> - assert_raise Mix.Error, ~r/mix gettext\.extract failed due to --check-up-to-date/, fn -> + assert_raise Mix.Error, expected_error, fn -> in_project(test, tmp_dir, fn _module -> run(["--from-attributes", "--check-up-to-date"]) end) @@ -207,7 +342,8 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do end end) - beams = Path.wildcard(Path.join(tmp_dir, "_build/**/Elixir.MyApp*.beam")) + compile_dir = in_project(test, tmp_dir, fn _module -> Mix.Project.compile_path() end) + beams = Path.wildcard(Path.join(compile_dir, "Elixir.MyApp*.beam")) assert beams != [] for beam <- beams do @@ -219,6 +355,68 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do end end + test "extracts each app of an umbrella project", + %{test: test, tmp_dir: tmp_dir} = context do + write_file(context, "mix.exs", """ + defmodule UmbrellaFixture.MixProject do + use Mix.Project + + def project() do + [apps_path: "apps", version: "0.1.0"] + end + end + """) + + for app <- ["app_one", "app_two"] do + camelized = Macro.camelize(app) + + write_file(context, "apps/#{app}/mix.exs", """ + defmodule #{camelized}.MixProject do + use Mix.Project + + def project() do + [ + app: :#{app}, + version: "0.1.0", + gettext: [extraction_environments: [:test]] + ] + end + + def application() do + [extra_applications: [:logger, :gettext]] + end + end + """) + + write_file(context, "apps/#{app}/lib/#{app}.ex", """ + defmodule #{camelized}.Gettext do + use Gettext.Backend, otp_app: :#{app} + end + + defmodule #{camelized} do + use Gettext, backend: #{camelized}.Gettext + def hello(), do: gettext("hello from #{app}") + end + """) + end + + capture_io(fn -> + in_project(test, tmp_dir, fn _module -> + # Mix.Task.run (not a direct module call) so that @recursive true + # recurses into each umbrella app. + Mix.Task.run("gettext.extract", ["--from-attributes"]) + end) + end) + + pot_one = read_file(context, "apps/app_one/priv/gettext/default.pot") + pot_two = read_file(context, "apps/app_two/priv/gettext/default.pot") + + assert pot_one =~ ~s(msgid "hello from app_one") + refute pot_one =~ "app_two" + assert pot_two =~ ~s(msgid "hello from app_two") + refute pot_two =~ "app_one" + end + defp write_fixture_with_changed_msgid(context) do source = @fixture_source From 044f3b1c940f795c1121eeb7d28cdcacf761ac07 Mon Sep 17 00:00:00 2001 From: Oliver Kriska Date: Tue, 16 Jun 2026 14:04:50 +0200 Subject: [PATCH 3/4] Gate attribute extraction on per-backend automatic_extraction config Replace the Mix-environment gate (:extraction_environments) with a per-backend application-environment toggle, read without touching Mix: config :gettext, MyApp.Gettext, automatic_extraction: true Reading from the application environment instead of Mix.Project/Mix.env keeps Mix out of the compile-time macro path, is off by default, and is off outside of Mix (such as runtime compilation in a release), so prod beams stay free of the persisted attributes. The toggle is keyed under the :gettext application (not the backend's own otp_app) so the persist gate never calls the backend at a gettext call site, avoiding a compile-time dependency that would recompile every gettext caller whenever PO files change. The persisted data, BEAM reading, and POT merging are unchanged, so the --from-attributes output is byte-identical to the recompilation path. --- CHANGELOG.md | 9 ++-- lib/gettext/compiler.ex | 4 +- lib/gettext/extractor.ex | 54 ++++++++----------- lib/gettext/macros.ex | 2 +- lib/mix/tasks/gettext.extract.ex | 31 +++++------ .../gettext.extract_from_attributes_test.exs | 35 +++++++----- 6 files changed, 66 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72e2b33..bad8b0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,11 @@ * Parallelize per-locale merging in `mix gettext.merge`. * Add experimental `--from-attributes` flag to `mix gettext.extract`: messages - are persisted as module attributes during normal compilation (in the - environments listed in the `:extraction_environments` gettext configuration, - `[:dev]` by default) and read back from the compiled BEAM files, so - extraction no longer needs to force-recompile the project. + are persisted as module attributes during normal compilation (for backends + with `automatic_extraction: true` in the application environment, for example + `config :gettext, MyApp.Gettext, automatic_extraction: true` in + `config/dev.exs`) and read back from the compiled BEAM files, so extraction + no longer needs to force-recompile the project. ## v1.0.2 diff --git a/lib/gettext/compiler.ex b/lib/gettext/compiler.ex index 24e9204..d474c75 100644 --- a/lib/gettext/compiler.ex +++ b/lib/gettext/compiler.ex @@ -38,8 +38,6 @@ defmodule Gettext.Compiler do interpolation = opts[:interpolation] || Gettext.Interpolation.Default - Gettext.Extractor.persist_backend_marker(env) - quote do @behaviour Gettext.Backend @@ -63,6 +61,8 @@ defmodule Gettext.Compiler do Gettext.ExtractorAgent.add_backend(__MODULE__) end + Gettext.Extractor.persist_backend_marker(__MODULE__) + # These are the two functions we generate inside the backend. @impl Gettext.Backend diff --git a/lib/gettext/extractor.ex b/lib/gettext/extractor.ex index 0fa24fd..48ebd0c 100644 --- a/lib/gettext/extractor.ex +++ b/lib/gettext/extractor.ex @@ -112,33 +112,24 @@ defmodule Gettext.Extractor do module attributes in the module that is currently being compiled. This is true when there is a module being compiled (`env.module` is not - `nil`) and the current Mix environment is included in the - `:extraction_environments` gettext configuration (defaults to `[:dev]`). - Outside of Mix (for example, when modules are compiled at runtime in a - release) this is always false. - """ - @spec persisting_to_attributes?(Macro.Env.t()) :: boolean - def persisting_to_attributes?(%Macro.Env{module: nil}), do: false - def persisting_to_attributes?(%Macro.Env{}), do: extraction_environment?() - - defp extraction_environment? do - cond do - not Code.ensure_loaded?(Mix) -> - false - - is_nil(Mix.Project.get()) -> - false - - true -> - gettext_config = Mix.Project.config()[:gettext] || [] + `nil`) and the `backend` has automatic extraction enabled, that is, when the + application environment holds: - extraction_environments = - gettext_config[:extraction_environments] || - Application.get_env(:gettext, :extraction_environments) || - [:dev] + config :gettext, MyApp.Gettext, automatic_extraction: true - Mix.env() in extraction_environments - end + This is read from the application environment (not `Mix`), so it is `false` + by default and outside of Mix (for example, when modules are compiled at + runtime in a release). + """ + @spec persisting_to_attributes?(Macro.Env.t(), backend :: module) :: boolean + def persisting_to_attributes?(%Macro.Env{module: nil}, _backend), do: false + def persisting_to_attributes?(%Macro.Env{}, backend), do: automatic_extraction?(backend) + + # Tells whether the given backend has automatic extraction enabled in the + # application environment (read from the :gettext app, not Mix). + defp automatic_extraction?(backend) when is_atom(backend) do + Application.get_env(:gettext, backend, []) + |> Keyword.get(:automatic_extraction, false) == true end @doc """ @@ -188,14 +179,11 @@ defmodule Gettext.Extractor do compiled, so that `fill_from_compiled_beams/1` can find all backends without recompiling. """ - @spec persist_backend_marker(Macro.Env.t()) :: :ok - def persist_backend_marker(%Macro.Env{} = env) do - if persisting_to_attributes?(env) do - if not Module.has_attribute?(env.module, @persisted_backend_attribute) do - Module.register_attribute(env.module, @persisted_backend_attribute, persist: true) - end - - Module.put_attribute(env.module, @persisted_backend_attribute, true) + @spec persist_backend_marker(backend :: module) :: :ok + def persist_backend_marker(backend) when is_atom(backend) do + if automatic_extraction?(backend) do + Module.register_attribute(backend, @persisted_backend_attribute, persist: true) + Module.put_attribute(backend, @persisted_backend_attribute, true) end :ok diff --git a/lib/gettext/macros.ex b/lib/gettext/macros.ex index 4e324de..e8a4fea 100644 --- a/lib/gettext/macros.ex +++ b/lib/gettext/macros.ex @@ -618,7 +618,7 @@ defmodule Gettext.Macros do # agent is not running, such as when compiling a dependency that uses # Gettext while the :gettext application is not started. extracting? = Extractor.extracting?() - persisting? = Extractor.persisting_to_attributes?(env) + persisting? = Extractor.persisting_to_attributes?(env, backend) if extracting? || persisting? do comments = get_and_flush_extracted_comments() diff --git a/lib/mix/tasks/gettext.extract.ex b/lib/mix/tasks/gettext.extract.ex index 45f57c2..564eba9 100644 --- a/lib/mix/tasks/gettext.extract.ex +++ b/lib/mix/tasks/gettext.extract.ex @@ -67,22 +67,18 @@ defmodule Mix.Tasks.Gettext.Extract do ``` For this to work, messages must have been persisted as module attributes - during normal compilation. This happens automatically when the current Mix - environment is included in the `:extraction_environments` Gettext - configuration in your `mix.exs`, which defaults to `[:dev]`: + during normal compilation. This happens automatically when the backend has + automatic extraction enabled in the application environment, which you + typically set in `config/dev.exs` so it stays off in `:prod`: - def project do - [ - # ... - gettext: [extraction_environments: [:dev]] - ] - end + # config/dev.exs + config :gettext, MyApp.Gettext, automatic_extraction: true With this flag, the task only runs a normal **incremental** compilation (changed files are recompiled and get fresh attributes; unchanged BEAM - files already carry theirs), then scans the compiled BEAM files. Since - environments outside `:extraction_environments` (such as `:prod`) never - persist these attributes, release artifacts are unaffected. + files already carry theirs), then scans the compiled BEAM files. Since the + attributes are only persisted when `automatic_extraction` is enabled (so not + in `:prod`), release artifacts are unaffected. `--from-attributes` can be combined with `--merge` and `--check-up-to-date`. @@ -162,12 +158,13 @@ defmodule Mix.Tasks.Gettext.Extract do or backends in #{Path.relative_to_cwd(Mix.Project.compile_path())}. Messages are persisted to module attributes during normal compilation only \ - when the current Mix environment is included in the :extraction_environments \ - Gettext configuration (which defaults to [:dev]). The current environment \ - is #{inspect(Mix.env())}. + when the backend has automatic extraction enabled in the application \ + environment, for example in config/dev.exs: - If you just updated Gettext or changed this configuration, force a recompile \ - so that up-to-date modules get their attributes written: + config :gettext, MyApp.Gettext, automatic_extraction: true + + If you just enabled this or updated Gettext, force a recompile so that \ + up-to-date modules get their attributes written: mix compile --force """) diff --git a/test/mix/tasks/gettext.extract_from_attributes_test.exs b/test/mix/tasks/gettext.extract_from_attributes_test.exs index 614525e..b0a19eb 100644 --- a/test/mix/tasks/gettext.extract_from_attributes_test.exs +++ b/test/mix/tasks/gettext.extract_from_attributes_test.exs @@ -64,6 +64,12 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do %{messages: %{}, backends: [], extracting?: false} end) + # The --from-attributes path only persists attributes for backends with + # automatic_extraction enabled in the application environment. Enable it + # for the fixture backend by default; the "disabled" test below clears it. + Application.put_env(:gettext, MyApp.Gettext, automatic_extraction: true) + on_exit(fn -> Application.delete_env(:gettext, MyApp.Gettext) end) + :ok end @@ -95,7 +101,7 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do test "extracts the same POT files as the recompilation path", %{test: test, tmp_dir: tmp_dir} = context do - create_test_mix_file(context, extraction_environments: [:test]) + create_test_mix_file(context) write_fixture(context) # Reference output: the existing force-recompile path. @@ -150,7 +156,7 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do test "persists messages injected by macros into the consuming module", %{test: test, tmp_dir: tmp_dir} = context do - create_test_mix_file(context, extraction_environments: [:test]) + create_test_mix_file(context) write_file(context, "lib/my_app.ex", """ defmodule MyApp.Gettext do @@ -188,7 +194,7 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do test "extracts messages from the _with_backend macros", %{test: test, tmp_dir: tmp_dir} = context do - create_test_mix_file(context, extraction_environments: [:test]) + create_test_mix_file(context) write_file(context, "lib/my_app.ex", """ defmodule MyApp.Gettext do @@ -221,7 +227,7 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do test "merges references when two modules use the same msgid", %{test: test, tmp_dir: tmp_dir} = context do - create_test_mix_file(context, extraction_environments: [:test]) + create_test_mix_file(context) write_file(context, "lib/my_app.ex", """ defmodule MyApp.Gettext do @@ -255,7 +261,7 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do test "--from-attributes combined with --merge updates PO files", %{test: test, tmp_dir: tmp_dir} = context do - create_test_mix_file(context, extraction_environments: [:test]) + create_test_mix_file(context) write_fixture(context) write_file(context, "priv/gettext/it/LC_MESSAGES/default.po", "") @@ -271,7 +277,7 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do test "respects a custom :priv option on the backend", %{test: test, tmp_dir: tmp_dir} = context do - create_test_mix_file(context, extraction_environments: [:test]) + create_test_mix_file(context) write_file(context, "lib/my_app.ex", """ defmodule MyApp.Gettext do @@ -296,7 +302,7 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do test "--check-up-to-date passes when fresh and fails after a source change", %{test: test, tmp_dir: tmp_dir} = context do - create_test_mix_file(context, extraction_environments: [:test]) + create_test_mix_file(context) write_fixture(context) capture_io(fn -> @@ -325,10 +331,11 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do end) end - test "no attributes are persisted outside extraction environments", + test "no attributes are persisted when automatic_extraction is disabled", %{test: test, tmp_dir: tmp_dir} = context do - # Default :extraction_environments is [:dev]; tests run in :test, so - # nothing must be persisted, mirroring a :prod release compile. + # With automatic_extraction off (the default), nothing must be persisted, + # mirroring a :prod release compile. + Application.delete_env(:gettext, MyApp.Gettext) create_test_mix_file(context) write_fixture(context) @@ -357,6 +364,11 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do test "extracts each app of an umbrella project", %{test: test, tmp_dir: tmp_dir} = context do + for backend <- [AppOne.Gettext, AppTwo.Gettext] do + Application.put_env(:gettext, backend, automatic_extraction: true) + on_exit(fn -> Application.delete_env(:gettext, backend) end) + end + write_file(context, "mix.exs", """ defmodule UmbrellaFixture.MixProject do use Mix.Project @@ -377,8 +389,7 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do def project() do [ app: :#{app}, - version: "0.1.0", - gettext: [extraction_environments: [:test]] + version: "0.1.0" ] end From 1b2951dbc8d66ad57fe9a9acfae01c7c39144568 Mon Sep 17 00:00:00 2001 From: Oliver Kriska Date: Tue, 16 Jun 2026 14:23:04 +0200 Subject: [PATCH 4/4] Move recompile-free extraction to its own gettext.generate task --- CHANGELOG.md | 7 +- lib/mix/tasks/gettext.extract.ex | 91 +++---------------- lib/mix/tasks/gettext.generate.ex | 85 +++++++++++++++++ ...tes_test.exs => gettext.generate_test.exs} | 37 +++++--- 4 files changed, 124 insertions(+), 96 deletions(-) create mode 100644 lib/mix/tasks/gettext.generate.ex rename test/mix/tasks/{gettext.extract_from_attributes_test.exs => gettext.generate_test.exs} (92%) diff --git a/CHANGELOG.md b/CHANGELOG.md index bad8b0e..ee79620 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,7 @@ * Parallelize per-locale merging in `mix gettext.merge`. - * Add experimental `--from-attributes` flag to `mix gettext.extract`: messages - are persisted as module attributes during normal compilation (for backends - with `automatic_extraction: true` in the application environment, for example - `config :gettext, MyApp.Gettext, automatic_extraction: true` in - `config/dev.exs`) and read back from the compiled BEAM files, so extraction - no longer needs to force-recompile the project. + * Add experimental `mix gettext.generate` task: messages are persisted as module attributes during normal compilation (for backends with `automatic_extraction: true` in the application environment, for example `config :gettext, MyApp.Gettext, automatic_extraction: true` in `config/dev.exs`) and read back from the compiled BEAM files, so extraction no longer needs to force-recompile the project. ## v1.0.2 diff --git a/lib/mix/tasks/gettext.extract.ex b/lib/mix/tasks/gettext.extract.ex index 564eba9..0a761e3 100644 --- a/lib/mix/tasks/gettext.extract.ex +++ b/lib/mix/tasks/gettext.extract.ex @@ -57,34 +57,14 @@ defmodule Mix.Tasks.Gettext.Extract do ## Extraction Without Recompiling (Experimental) - By default, this task extracts messages by **force-recompiling** the whole - project, because extraction happens during the expansion of the Gettext - macros. As an experimental alternative, the `--from-attributes` flag reads - messages back from the compiled BEAM files instead: - - ```bash - mix gettext.extract --from-attributes - ``` - - For this to work, messages must have been persisted as module attributes - during normal compilation. This happens automatically when the backend has - automatic extraction enabled in the application environment, which you - typically set in `config/dev.exs` so it stays off in `:prod`: - - # config/dev.exs - config :gettext, MyApp.Gettext, automatic_extraction: true - - With this flag, the task only runs a normal **incremental** compilation - (changed files are recompiled and get fresh attributes; unchanged BEAM - files already carry theirs), then scans the compiled BEAM files. Since the - attributes are only persisted when `automatic_extraction` is enabled (so not - in `:prod`), release artifacts are unaffected. - - `--from-attributes` can be combined with `--merge` and `--check-up-to-date`. + This task always **force-recompiles** the whole project, because extraction + happens during the expansion of the Gettext macros. If you would rather + generate the POT files without a force-recompile, see `mix gettext.generate`, + which reads the messages back from the compiled BEAM files instead. """ - @switches [merge: :boolean, check_up_to_date: :boolean, from_attributes: :boolean] + @switches [merge: :boolean, check_up_to_date: :boolean] @impl true def run(args) do @@ -92,8 +72,15 @@ defmodule Mix.Tasks.Gettext.Extract do _ = Mix.Project.get!() mix_config = Mix.Project.config() {opts, _} = OptionParser.parse!(args, switches: @switches) - pot_files = extract(mix_config[:app], mix_config[:gettext] || [], opts) + pot_files = extract(mix_config[:app], mix_config[:gettext] || []) + process(pot_files, opts, args) + end + # Shared by `mix gettext.extract` and `mix gettext.generate`: + # both compute `pot_files` (their only difference) and then write, check, or + # merge them in exactly the same way. + @doc false + def process(pot_files, opts, args) do if opts[:check_up_to_date] do run_up_to_date_check(pot_files) else @@ -130,15 +117,7 @@ defmodule Mix.Tasks.Gettext.Extract do end end - defp extract(app, gettext_config, opts) do - if opts[:from_attributes] do - extract_from_attributes(app, gettext_config) - else - extract_via_recompilation(app, gettext_config) - end - end - - defp extract_via_recompilation(app, gettext_config) do + defp extract(app, gettext_config) do Gettext.Extractor.enable() force_compile() Gettext.Extractor.pot_files(app, gettext_config) @@ -146,48 +125,6 @@ defmodule Mix.Tasks.Gettext.Extract do Gettext.Extractor.disable() end - defp extract_from_attributes(app, gettext_config) do - incremental_compile() - - {backends, messages} = - Gettext.Extractor.fill_from_compiled_beams(Mix.Project.compile_path()) - - if backends == 0 and messages == 0 do - Mix.raise(""" - mix gettext.extract --from-attributes found no persisted Gettext messages \ - or backends in #{Path.relative_to_cwd(Mix.Project.compile_path())}. - - Messages are persisted to module attributes during normal compilation only \ - when the backend has automatic extraction enabled in the application \ - environment, for example in config/dev.exs: - - config :gettext, MyApp.Gettext, automatic_extraction: true - - If you just enabled this or updated Gettext, force a recompile so that \ - up-to-date modules get their attributes written: - - mix compile --force - """) - end - - Gettext.Extractor.pot_files(app, gettext_config) - end - - defp incremental_compile do - # A plain incremental compile, with one wrinkle: "compile" and the - # compilers it runs may already have been invoked in this VM (for - # example, through a task alias), in which case running them again would - # be a no-op unless they are reenabled first. The trailing explicit - # "compile.elixir" run is a no-op when "compile" just ran it, and covers - # the case where a custom "compile" alias does not. - Mix.Task.reenable("compile") - Mix.Task.reenable("compile.all") - Mix.Task.reenable("compile.elixir") - Mix.Task.reenable("compile.app") - Mix.Task.run("compile", []) - Mix.Task.run("compile.elixir", []) - end - defp force_compile do # For old Elixir versions, we have to clean the manifest, # otherwise we are forced to compile all dependencies. diff --git a/lib/mix/tasks/gettext.generate.ex b/lib/mix/tasks/gettext.generate.ex new file mode 100644 index 0000000..606a623 --- /dev/null +++ b/lib/mix/tasks/gettext.generate.ex @@ -0,0 +1,85 @@ +defmodule Mix.Tasks.Gettext.Generate do + use Mix.Task + @recursive true + + @shortdoc "Generates POT files from messages persisted during compilation" + + @moduledoc """ + Generates POT files without force-recompiling the project (experimental). + + ```bash + mix gettext.generate [OPTIONS] + ``` + + Unlike `mix gettext.extract`, which force-recompiles the whole project so + that the Gettext macros run again and re-extract the messages, this task + generates the POT files from messages that were already extracted during the + normal compilation. It compiles the project normally (a no-op when it is + already compiled) and then reads the messages back from the persisted module + attributes in the compiled BEAM files, so it avoids the cost of a + force-recompile. + + For this to work, messages must have been persisted as module attributes + during normal compilation. This happens automatically when the backend has + automatic extraction enabled in the application environment, which you + typically set in `config/dev.exs` so it stays off in `:prod`: + + # config/dev.exs + config :gettext, MyApp.Gettext, automatic_extraction: true + + Since the attributes are only persisted when `automatic_extraction` is + enabled (so not in `:prod`), release artifacts are unaffected. + + This task accepts the same `--merge` and `--check-up-to-date` options as + `mix gettext.extract`, and forwards any other options to + `Mix.Tasks.Gettext.Merge`: + + ```bash + mix gettext.generate --merge --no-fuzzy + mix gettext.generate --check-up-to-date + ``` + + """ + + @switches [merge: :boolean, check_up_to_date: :boolean] + + @impl true + def run(args) do + Application.ensure_all_started(:gettext) + _ = Mix.Project.get!() + mix_config = Mix.Project.config() + {opts, _} = OptionParser.parse!(args, switches: @switches) + pot_files = generate(mix_config[:app], mix_config[:gettext] || []) + Mix.Tasks.Gettext.Extract.process(pot_files, opts, args) + end + + defp generate(app, gettext_config) do + # The messages are extracted and persisted by the normal compilation; here + # we just make sure that has happened. This is a no-op when the project is + # already compiled. + Mix.Task.run("compile", []) + + {backends, messages} = + Gettext.Extractor.fill_from_compiled_beams(Mix.Project.compile_path()) + + if backends == 0 and messages == 0 do + Mix.raise(""" + mix gettext.generate found no persisted Gettext messages \ + or backends in #{Path.relative_to_cwd(Mix.Project.compile_path())}. + + Messages are persisted to module attributes during normal compilation only \ + when the backend has automatic extraction enabled in the application \ + environment, for example in config/dev.exs: + + config :gettext, MyApp.Gettext, automatic_extraction: true + + If you just enabled this or updated Gettext, force a recompile so that \ + up-to-date modules get their attributes written: + + mix compile --force + """) + end + + Gettext.Extractor.pot_files(app, gettext_config) + end +end diff --git a/test/mix/tasks/gettext.extract_from_attributes_test.exs b/test/mix/tasks/gettext.generate_test.exs similarity index 92% rename from test/mix/tasks/gettext.extract_from_attributes_test.exs rename to test/mix/tasks/gettext.generate_test.exs index b0a19eb..685b8ab 100644 --- a/test/mix/tasks/gettext.extract_from_attributes_test.exs +++ b/test/mix/tasks/gettext.generate_test.exs @@ -1,4 +1,4 @@ -defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do +defmodule Mix.Tasks.Gettext.GenerateTest do # async: false (default) is required: these tests mutate global Mix task # invocation state, the code path, and Code.compiler_options/1. use ExUnit.Case @@ -124,7 +124,7 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do # POT files written by the recompilation path unchanged. output = capture_io(fn -> - in_project(test, tmp_dir, fn _module -> run(["--from-attributes"]) end) + in_project(test, tmp_dir, fn _module -> run_from_attributes() end) end) refute output =~ "Extracted" @@ -137,7 +137,7 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do output = capture_io(fn -> - in_project(test, tmp_dir, fn _module -> run(["--from-attributes"]) end) + in_project(test, tmp_dir, fn _module -> run_from_attributes() end) end) assert output =~ "Extracted priv/gettext/default.pot" @@ -184,7 +184,7 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do """) capture_io(fn -> - in_project(test, tmp_dir, fn _module -> run(["--from-attributes"]) end) + in_project(test, tmp_dir, fn _module -> run_from_attributes() end) end) pot = read_file(context, "priv/gettext/default.pot") @@ -215,7 +215,7 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do """) capture_io(fn -> - in_project(test, tmp_dir, fn _module -> run(["--from-attributes"]) end) + in_project(test, tmp_dir, fn _module -> run_from_attributes() end) end) default_pot = read_file(context, "priv/gettext/default.pot") @@ -248,7 +248,7 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do """) capture_io(fn -> - in_project(test, tmp_dir, fn _module -> run(["--from-attributes"]) end) + in_project(test, tmp_dir, fn _module -> run_from_attributes() end) end) pot = read_file(context, "priv/gettext/default.pot") @@ -267,7 +267,7 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do write_file(context, "priv/gettext/it/LC_MESSAGES/default.po", "") capture_io(fn -> - in_project(test, tmp_dir, fn _module -> run(["--from-attributes", "--merge"]) end) + in_project(test, tmp_dir, fn _module -> run_from_attributes(["--merge"]) end) end) po = read_file(context, "priv/gettext/it/LC_MESSAGES/default.po") @@ -291,7 +291,7 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do """) capture_io(fn -> - in_project(test, tmp_dir, fn _module -> run(["--from-attributes"]) end) + in_project(test, tmp_dir, fn _module -> run_from_attributes() end) end) assert read_file(context, "priv/custom_gettext/default.pot") =~ @@ -307,11 +307,11 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do capture_io(fn -> in_project(test, tmp_dir, fn _module -> - run(["--from-attributes"]) + run_from_attributes() end) in_project(test, tmp_dir, fn _module -> - run(["--from-attributes", "--check-up-to-date"]) + run_from_attributes(["--check-up-to-date"]) end) end) @@ -325,7 +325,7 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do capture_io(fn -> assert_raise Mix.Error, expected_error, fn -> in_project(test, tmp_dir, fn _module -> - run(["--from-attributes", "--check-up-to-date"]) + run_from_attributes(["--check-up-to-date"]) end) end end) @@ -344,7 +344,7 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do capture_io(fn -> assert_raise Mix.Error, ~r/found no persisted Gettext messages/, fn -> in_project(test, tmp_dir, fn _module -> - run(["--from-attributes"]) + run_from_attributes() end) end end) @@ -415,7 +415,7 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do in_project(test, tmp_dir, fn _module -> # Mix.Task.run (not a direct module call) so that @recursive true # recurses into each umbrella app. - Mix.Task.run("gettext.extract", ["--from-attributes"]) + Mix.Task.run("gettext.generate", []) end) end) @@ -440,4 +440,15 @@ defmodule Mix.Tasks.Gettext.ExtractFromAttributesTest do defp run(args) do Mix.Tasks.Gettext.Extract.run(args) end + + defp run_from_attributes(args \\ []) do + # Reenable compile so each invocation picks up source changes, mirroring a + # fresh `mix` invocation (the task itself does a plain compile, not a + # force-recompile). + for task <- ~w(compile compile.all compile.elixir compile.app) do + Mix.Task.reenable(task) + end + + Mix.Tasks.Gettext.Generate.run(args) + end end