From a5f7514a264a8fc1aac8a8e9e5c2b2585ba61074 Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 16:57:15 +0400 Subject: [PATCH 1/3] fix: stop inspect printing API keys --- lib/translation_diff/configuration.rb | 6 +- lib/translation_diff/provider.rb | 15 ++++ lib/translation_diff/providers.rb | 1 + lib/translation_diff/redaction.rb | 45 ++++++++++ lib/translation_diff/registry.rb | 2 + test/translation_diff/redaction_test.rb | 115 ++++++++++++++++++++++++ 6 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 lib/translation_diff/redaction.rb create mode 100644 test/translation_diff/redaction_test.rb diff --git a/lib/translation_diff/configuration.rb b/lib/translation_diff/configuration.rb index 944b5bd..66f92fb 100644 --- a/lib/translation_diff/configuration.rb +++ b/lib/translation_diff/configuration.rb @@ -56,6 +56,10 @@ def provider_option_owners = @provider_option_owners ||= ProviderOptionOwners.ne option :open_timeout, 5 option :timeout, 30 option :max_retries, 3 + option :validate_languages, true + + # Credentials are filtered by name; everything else is shown, or an inspect is one nobody reads. + def inspect = "#<#{self.class.name} #{TranslationDiff::Redaction.render(self).join(' ')}>" # Memoised collaborators aren't copied, or a tenant's own cache_namespace leaks its parent's rate limiter. def copy @@ -81,7 +85,7 @@ def segmenter_instance @segmenter_instance ||= resolve(segmenter, TranslationDiff::Segmenters.registry) end - # nil, not a null object: Request checks for nil and skips rate-limiting entirely -- costs nothing normally. + # nil, not a null object: Dispatcher#throttle checks for nil and skips rate-limiting -- costs nothing normally. def rate_limiter_instance return rate_limiter unless rate_limiter.nil? return nil if rate_limit.nil? diff --git a/lib/translation_diff/provider.rb b/lib/translation_diff/provider.rb index 413977b..3f71759 100644 --- a/lib/translation_diff/provider.rb +++ b/lib/translation_diff/provider.rb @@ -34,11 +34,20 @@ def billed_characters(reported) values.empty? ? nil : values.sum end + # Never the default: a provider holds the configuration, so the default renders every key it holds. + def inspect = "#<#{self.class.name} name=#{name.inspect} config=#{config.inspect}>" + def translate(_request) = raise NotImplementedError, "#{self.class} must implement #translate" # Only called when `capabilities.detects_language?`. def detect(_text) = raise NotImplementedError, "#{self.class} must implement #detect" + # Only `rake languages:refresh` calls this; a provider that cannot answer is skipped, not failed. + def languages = raise NotImplementedError, "#{self.class} cannot fetch its languages" + + # The full URL #languages fetches; a provider whose fetch has more than one shape narrows this further. + def languages_endpoint = respond_to?(:api_base) ? api_base.to_s : "" + # Raising when never stamped, rather than falling back to "", is deliberate: "" would merge namespaces silently. def cache_key return name.to_s unless name.nil? @@ -55,6 +64,12 @@ def language_case = :downcase def configuration_options = [] + # Overridable: a provider whose credential is named unusually says so rather than leaking it. + def sensitive_options + configuration_options.flat_map { |o| o.is_a?(Hash) ? o.keys : [o] } + .select { |key| TranslationDiff::Redaction.sensitive?(key) } + end + # Checked once, at build time, so a caller learns what to set before a vendor's own exception does. def configuration_requirements = [] diff --git a/lib/translation_diff/providers.rb b/lib/translation_diff/providers.rb index 4d5c917..2ec23d8 100644 --- a/lib/translation_diff/providers.rb +++ b/lib/translation_diff/providers.rb @@ -36,6 +36,7 @@ def build(name, config) def registered?(name) = registry.registered?(name) def names = registry.names + def classes = registry.classes def registry = @registry ||= TranslationDiff::Registry.new("provider") private diff --git a/lib/translation_diff/redaction.rb b/lib/translation_diff/redaction.rb new file mode 100644 index 0000000..9fa04a8 --- /dev/null +++ b/lib/translation_diff/redaction.rb @@ -0,0 +1,45 @@ +require "uri" + +# Which configuration options must never be printed, decided by name rather than by a list someone maintains. +module TranslationDiff::Redaction + SENSITIVE = /key|secret|token|password|auth|credential/ + FILTERED = "[FILTERED]".freeze + + def self.sensitive?(name) = name.to_s.match?(SENSITIVE) + + # Unioned fresh on every call, never memoised -- a provider can register after the first inspect. + def self.declared_sensitive + TranslationDiff::Providers.classes.flat_map(&:sensitive_options).map(&:to_sym) + end + + # Reads through the public accessor, or an option set only through its ENV-backed default goes unnoticed. + def self.render(config) + declared = declared_sensitive + + TranslationDiff::Configuration.options.filter_map do |key| + value = config.public_send(key) + next if value.nil? + + "#{key}=#{rendered_value(key, value, declared)}" + end + end + + def self.rendered_value(key, value, declared) + return FILTERED if sensitive?(key) || declared.include?(key) + + (redact_userinfo(value) || value).inspect + end + + # redis_url and a provider's *_api_base can carry a credential inline; the host stays, only the userinfo hides. + def self.redact_userinfo(value) + return nil unless value.is_a?(String) + + userinfo = URI.parse(value).userinfo + return nil unless userinfo + + user, separator, = userinfo.partition(":") + value.sub(userinfo, separator.empty? ? FILTERED : "#{user}:#{FILTERED}") + rescue URI::Error + nil + end +end diff --git a/lib/translation_diff/registry.rb b/lib/translation_diff/registry.rb index 9d98fcc..eac92e4 100644 --- a/lib/translation_diff/registry.rb +++ b/lib/translation_diff/registry.rb @@ -18,6 +18,8 @@ def registered?(name) = @entries.key?(name.to_sym) def names = @entries.keys + def classes = @entries.values + private def fetch(name) diff --git a/test/translation_diff/redaction_test.rb b/test/translation_diff/redaction_test.rb new file mode 100644 index 0000000..b7075d5 --- /dev/null +++ b/test/translation_diff/redaction_test.rb @@ -0,0 +1,115 @@ +require "test_helper" + +class RedactionTest < ConfiguredTest + def test_configuration_inspect_hides_every_credential_it_holds + TranslationDiff.configure do |c| + c.deepl_api_key = "SECRET-DEEPL-abc123:fx" + c.amazon_secret_access_key = "SECRET-AWS-xyz789" + c.cache_namespace = "tenant-7" + end + + rendered = TranslationDiff.config.inspect + + refute_includes rendered, "SECRET-DEEPL-abc123:fx" + refute_includes rendered, "SECRET-AWS-xyz789" + assert_includes rendered, "[FILTERED]" + assert_includes rendered, "tenant-7" + end + + def test_a_provider_inspect_hides_the_credentials_of_the_configuration_it_holds + TranslationDiff.configure { |c| c.deepl_api_key = "SECRET-DEEPL-abc123:fx" } + provider = TranslationDiff::Providers.build(:deepl, TranslationDiff.config) + + rendered = provider.inspect + + refute_includes rendered, "SECRET-DEEPL-abc123:fx" + assert_includes rendered, "deepl" + end + + # The default #inspect of any object holding either one calls theirs, so the leak has to stop here. + def test_an_object_holding_the_configuration_leaks_nothing_either + TranslationDiff.configure { |c| c.deepl_api_key = "SECRET-DEEPL-abc123:fx" } + context = TranslationDiff.context { |c| c.cache = :memory } + + refute_includes context.inspect, "SECRET-DEEPL-abc123:fx" + end + + # Derived from the registry, so a provider registered later is covered without anyone remembering. + # The option name must not match SENSITIVE itself, or the test would pass even with the registry union deleted. + def test_the_filtered_list_covers_a_provider_registered_afterwards + klass = Class.new(TranslationDiff::Provider) do + def self.configuration_options = %i[redaction_acme_session redaction_acme_api_base] + def self.sensitive_options = %i[redaction_acme_session] + end + TranslationDiff::Providers.register(:acme_redaction, klass) + TranslationDiff.configure { |c| c.redaction_acme_session = "SECRET-ACME-000" } + + refute_includes TranslationDiff.config.inspect, "SECRET-ACME-000" + end + + def test_a_base_url_is_not_a_credential_and_stays_visible + TranslationDiff.configure { |c| c.libretranslate_api_base = "http://localhost:5000" } + + assert_includes TranslationDiff.config.inspect, "http://localhost:5000" + end + + # No ENV default exists for this one, so it is truly unset everywhere, not just on a laptop without the var. + def test_an_unset_option_is_not_rendered_at_all + refute_includes TranslationDiff.config.inspect, "azure_api_key" + end + + # The regex can't catch this name; the provider has to say so itself, and Redaction has to ask it. + def test_a_provider_overriding_sensitive_options_hides_a_name_the_regex_cannot_match + klass = Class.new(TranslationDiff::Provider) do + def self.configuration_options = %i[acme_cookie] + def self.sensitive_options = %i[acme_cookie] + end + TranslationDiff::Providers.register(:acme_cookie_provider, klass) + TranslationDiff.configure { |c| c.acme_cookie = "SECRET-SESSION-COOKIE-VALUE" } + + refute_includes TranslationDiff.config.inspect, "SECRET-SESSION-COOKIE-VALUE" + end + + # Set only through its ENV-backed default, so no ivar exists; inspect must still filter it, not skip it. + def test_an_option_set_only_through_its_env_default_is_still_filtered + ENV["DEEPL_AUTH_KEY"] = "SECRET-DEEPL-FROM-ENV" + + assert_includes TranslationDiff.config.inspect, "[FILTERED]" + refute_includes TranslationDiff.config.inspect, "SECRET-DEEPL-FROM-ENV" + ensure + ENV.delete("DEEPL_AUTH_KEY") + end + + # Heroku, Upstash, Redis Cloud, Aiven and ElastiCache all put the credential inline in this URL. + def test_a_redis_url_has_its_password_redacted_but_stays_readable + TranslationDiff.configure do |c| + c.redis_url = "rediss://default:AbCdEf-SUPER-SECRET-TOKEN@cache.example.upstash.io:6379" + end + + rendered = TranslationDiff.config.inspect + + refute_includes rendered, "AbCdEf-SUPER-SECRET-TOKEN" + assert_includes rendered, "[FILTERED]" + assert_includes rendered, "default" + assert_includes rendered, "cache.example.upstash.io:6379" + end + + # A provider's *_api_base can carry basic-auth credentials the same way redis_url does. + def test_a_provider_api_base_with_basic_auth_has_its_password_redacted + TranslationDiff.configure { |c| c.libretranslate_api_base = "https://user:hunter2@translate.example.com" } + + rendered = TranslationDiff.config.inspect + + refute_includes rendered, "hunter2" + assert_includes rendered, "translate.example.com" + end + + # An inspect that blows up on a bad value is worse than a verbose one. + def test_a_malformed_url_like_option_does_not_raise_from_inspect + TranslationDiff.configure { |c| c.libretranslate_api_base = "http://[not-a-valid-host" } + + rendered = TranslationDiff.config.inspect + + assert_includes rendered, "libretranslate_api_base" + end +end From 833d96221cdf908e56d6743ee78d30b12e739e8b Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 16:57:27 +0400 Subject: [PATCH 2/3] feat: ship each provider's languages, refuse an unsupported pair, report usage --- .rubocop.yml | 10 +- Rakefile | 26 ++ data/languages/azure.json | 285 ++++++++++++ data/languages/deepl.json | 220 ++++++++++ data/languages/google.json | 399 +++++++++++++++++ data/languages/modernmt.json | 413 ++++++++++++++++++ lib/translation_diff.rb | 13 +- lib/translation_diff/context.rb | 5 +- lib/translation_diff/dispatcher.rb | 54 +++ lib/translation_diff/errors.rb | 3 + lib/translation_diff/http_provider.rb | 9 + lib/translation_diff/languages.rb | 30 ++ lib/translation_diff/languages/refresh.rb | 70 +++ lib/translation_diff/languages/set.rb | 53 +++ lib/translation_diff/providers/amazon.rb | 22 +- lib/translation_diff/providers/azure.rb | 9 + lib/translation_diff/providers/deepl.rb | 10 + lib/translation_diff/providers/google.rb | 11 + .../providers/libretranslate.rb | 10 + lib/translation_diff/providers/modernmt.rb | 8 + lib/translation_diff/translator.rb | 60 +-- test/translation_diff/dispatcher_test.rb | 161 +++++++ test/translation_diff/http_provider_test.rb | 15 + test/translation_diff/instrumentation_test.rb | 54 ++- .../languages/refresh_test.rb | 123 ++++++ test/translation_diff/languages/set_test.rb | 38 ++ test/translation_diff/languages_test.rb | 85 ++++ .../providers/languages_test.rb | 133 ++++++ test/translation_diff/translator_test.rb | 113 ++++- 29 files changed, 2395 insertions(+), 47 deletions(-) create mode 100644 data/languages/azure.json create mode 100644 data/languages/deepl.json create mode 100644 data/languages/google.json create mode 100644 data/languages/modernmt.json create mode 100644 lib/translation_diff/dispatcher.rb create mode 100644 lib/translation_diff/languages.rb create mode 100644 lib/translation_diff/languages/refresh.rb create mode 100644 lib/translation_diff/languages/set.rb create mode 100644 test/translation_diff/dispatcher_test.rb create mode 100644 test/translation_diff/languages/refresh_test.rb create mode 100644 test/translation_diff/languages/set_test.rb create mode 100644 test/translation_diff/languages_test.rb create mode 100644 test/translation_diff/providers/languages_test.rb diff --git a/.rubocop.yml b/.rubocop.yml index dea43ba..578f546 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -26,17 +26,13 @@ Gemspec/DevelopmentDependencies: Style/FrozenStringLiteralComment: Enabled: false -# Request takes its collaborators as keyword arguments -- values, from:, to:, -# provider:, config: and the provider's own options. The confusion this cop -# guards against is positional: six named arguments at a call site read -# perfectly well, and the alternative is pulling them out of an options hash -# where nothing documents them. +# Translator#initialize takes its collaborators as keyword arguments -- values, from:, to:, provider:, config:, +# assume_supported: and the provider's own options -- which read perfectly well named at a call site, unlike +# pulling them out of an undocumented options hash, which is the confusion this cop guards against. Metrics/ParameterLists: CountKeywordArgs: false Metrics/ClassLength: Exclude: - - lib/translation_diff/tokenizer.rb - - lib/translation_diff/request.rb # Test classes are mostly tables of cases. - test/**/* diff --git a/Rakefile b/Rakefile index d4ffd43..ef05ed5 100644 --- a/Rakefile +++ b/Rakefile @@ -10,3 +10,29 @@ Rake::TestTask.new(:test) do |t| end task default: :test + +namespace :languages do + desc "Re-fetch every provider's language lists from its vendor" + task :refresh do + require "translation_diff" + + not_shipped = TranslationDiff::Languages::NOT_SHIPPED + unless not_shipped.empty? + puts "not shipping data for #{not_shipped.join(', ')}: a vendor credential or a private instance " \ + "would make the file unshareable" + end + + skip = [:null, *not_shipped] + providers = TranslationDiff::Providers.names.reject { |name| skip.include?(name) }.map do |name| + TranslationDiff::Providers.build(name, TranslationDiff.config) + rescue TranslationDiff::ConfigurationError => e + warn "skipping #{name}: #{e.message}" + nil + end.compact + + report = TranslationDiff::Languages::Refresh.call(providers: providers) + puts "updated: #{report[:updated].join(', ')}" unless report[:updated].empty? + puts "skipped: #{report[:skipped].join(', ')}" unless report[:skipped].empty? + report[:failed].each { |name, message| warn "failed: #{name}: #{message}" } + end +end diff --git a/data/languages/azure.json b/data/languages/azure.json new file mode 100644 index 0000000..396f244 --- /dev/null +++ b/data/languages/azure.json @@ -0,0 +1,285 @@ +{ + "provider": "azure", + "captured_at": "2026-09-10", + "endpoint": "https://api.cognitive.microsofttranslator.com/languages?api-version=3.0&scope=translation", + "source": [ + "af", + "am", + "ar", + "as", + "az", + "ba", + "be", + "bg", + "bho", + "bn", + "bo", + "brx", + "bs", + "ca", + "cs", + "cy", + "da", + "de", + "doi", + "dsb", + "dv", + "el", + "en", + "es", + "es-mx", + "et", + "eu", + "fa", + "fi", + "fil", + "fj", + "fo", + "fr", + "fr-ca", + "ga", + "gl", + "gom", + "gu", + "ha", + "he", + "hi", + "hne", + "hr", + "hsb", + "ht", + "hu", + "hy", + "id", + "ig", + "ikt", + "is", + "it", + "iu", + "iu-latn", + "ja", + "ka", + "kk", + "km", + "kmr", + "kn", + "ko", + "ks", + "ku", + "ky", + "lb", + "ln", + "lo", + "lt", + "lug", + "lv", + "lzh", + "mai", + "mg", + "mi", + "mk", + "ml", + "mn-cyrl", + "mn-mong", + "mni", + "mr", + "ms", + "mt", + "mww", + "my", + "nb", + "ne", + "nl", + "nso", + "nya", + "or", + "otq", + "pa", + "pl", + "prs", + "ps", + "pt", + "pt-pt", + "ro", + "ru", + "run", + "rw", + "sd", + "si", + "sk", + "sl", + "sm", + "sn", + "so", + "sq", + "sr-cyrl", + "sr-latn", + "st", + "sv", + "sw", + "ta", + "te", + "th", + "ti", + "tk", + "tlh-latn", + "tlh-piqd", + "tn", + "to", + "tr", + "tt", + "ty", + "ug", + "uk", + "ur", + "uz", + "vi", + "xh", + "yo", + "yua", + "yue", + "zh-hans", + "zh-hant", + "zu" + ], + "target": [ + "af", + "am", + "ar", + "as", + "az", + "ba", + "be", + "bg", + "bho", + "bn", + "bo", + "brx", + "bs", + "ca", + "cs", + "cy", + "da", + "de", + "doi", + "dsb", + "dv", + "el", + "en", + "es", + "es-mx", + "et", + "eu", + "fa", + "fi", + "fil", + "fj", + "fo", + "fr", + "fr-ca", + "ga", + "gl", + "gom", + "gu", + "ha", + "he", + "hi", + "hne", + "hr", + "hsb", + "ht", + "hu", + "hy", + "id", + "ig", + "ikt", + "is", + "it", + "iu", + "iu-latn", + "ja", + "ka", + "kk", + "km", + "kmr", + "kn", + "ko", + "ks", + "ku", + "ky", + "lb", + "ln", + "lo", + "lt", + "lug", + "lv", + "lzh", + "mai", + "mg", + "mi", + "mk", + "ml", + "mn-cyrl", + "mn-mong", + "mni", + "mr", + "ms", + "mt", + "mww", + "my", + "nb", + "ne", + "nl", + "nso", + "nya", + "or", + "otq", + "pa", + "pl", + "prs", + "ps", + "pt", + "pt-pt", + "ro", + "ru", + "run", + "rw", + "sd", + "si", + "sk", + "sl", + "sm", + "sn", + "so", + "sq", + "sr-cyrl", + "sr-latn", + "st", + "sv", + "sw", + "ta", + "te", + "th", + "ti", + "tk", + "tlh-latn", + "tlh-piqd", + "tn", + "to", + "tr", + "tt", + "ty", + "ug", + "uk", + "ur", + "uz", + "vi", + "xh", + "yo", + "yua", + "yue", + "zh-hans", + "zh-hant", + "zu" + ] +} diff --git a/data/languages/deepl.json b/data/languages/deepl.json new file mode 100644 index 0000000..62419ad --- /dev/null +++ b/data/languages/deepl.json @@ -0,0 +1,220 @@ +{ + "provider": "deepl", + "captured_at": "2026-09-10", + "endpoint": "https://api.deepl.com/v2/languages", + "source": [ + "af", + "an", + "ar", + "as", + "ay", + "az", + "ba", + "be", + "bg", + "bn", + "br", + "bs", + "ca", + "cs", + "cy", + "da", + "de", + "el", + "en", + "eo", + "es", + "et", + "eu", + "fa", + "fi", + "fr", + "ga", + "gl", + "gn", + "gu", + "ha", + "he", + "hi", + "hr", + "ht", + "hu", + "hy", + "id", + "ig", + "is", + "it", + "ja", + "jv", + "ka", + "kk", + "ko", + "ky", + "la", + "lb", + "ln", + "lt", + "lv", + "mg", + "mi", + "mk", + "ml", + "mn", + "mr", + "ms", + "mt", + "my", + "nb", + "ne", + "nl", + "oc", + "om", + "pa", + "pl", + "ps", + "pt", + "qu", + "ro", + "ru", + "sa", + "sk", + "sl", + "sq", + "sr", + "st", + "su", + "sv", + "sw", + "ta", + "te", + "tg", + "th", + "tk", + "tl", + "tn", + "tr", + "ts", + "tt", + "uk", + "ur", + "uz", + "vi", + "wo", + "xh", + "yi", + "zh", + "zu" + ], + "target": [ + "af", + "an", + "ar", + "as", + "ay", + "az", + "ba", + "be", + "bg", + "bn", + "br", + "bs", + "ca", + "cs", + "cy", + "da", + "de", + "de-ch", + "de-de", + "el", + "en-gb", + "en-us", + "eo", + "es", + "es-419", + "et", + "eu", + "fa", + "fi", + "fr", + "fr-ca", + "fr-fr", + "ga", + "gl", + "gn", + "gu", + "ha", + "he", + "hi", + "hr", + "ht", + "hu", + "hy", + "id", + "ig", + "is", + "it", + "ja", + "jv", + "ka", + "kk", + "ko", + "ky", + "la", + "lb", + "ln", + "lt", + "lv", + "mg", + "mi", + "mk", + "ml", + "mn", + "mr", + "ms", + "mt", + "my", + "nb", + "ne", + "nl", + "oc", + "om", + "pa", + "pl", + "ps", + "pt-br", + "pt-pt", + "qu", + "ro", + "ru", + "sa", + "sk", + "sl", + "sq", + "sr", + "st", + "su", + "sv", + "sw", + "ta", + "te", + "tg", + "th", + "tk", + "tl", + "tn", + "tr", + "ts", + "tt", + "uk", + "ur", + "uz", + "vi", + "wo", + "xh", + "yi", + "zh", + "zh-hans", + "zh-hant", + "zu" + ] +} diff --git a/data/languages/google.json b/data/languages/google.json new file mode 100644 index 0000000..61f465d --- /dev/null +++ b/data/languages/google.json @@ -0,0 +1,399 @@ +{ + "provider": "google", + "captured_at": "2026-09-10", + "endpoint": "https://translation.googleapis.com/language/translate/v2/languages", + "source": [ + "ab", + "ace", + "ach", + "af", + "ak", + "alz", + "am", + "ar", + "as", + "awa", + "ay", + "az", + "ba", + "ban", + "bbc", + "be", + "bem", + "bew", + "bg", + "bho", + "bik", + "bm", + "bn", + "br", + "bs", + "bts", + "btx", + "bua", + "ca", + "ceb", + "cgg", + "chm", + "ckb", + "cnh", + "co", + "crh", + "crs", + "cs", + "cv", + "cy", + "da", + "de", + "din", + "doi", + "dov", + "dv", + "dz", + "ee", + "el", + "en", + "eo", + "es", + "et", + "eu", + "fa", + "ff", + "fi", + "fj", + "fr", + "fr-ca", + "fy", + "ga", + "gaa", + "gd", + "gl", + "gn", + "gom", + "gu", + "ha", + "haw", + "he", + "hi", + "hil", + "hmn", + "hr", + "hrx", + "ht", + "hu", + "hy", + "id", + "ig", + "ilo", + "is", + "it", + "iw", + "ja", + "jv", + "jw", + "ka", + "kk", + "km", + "kn", + "ko", + "kri", + "ktu", + "ku", + "ky", + "la", + "lb", + "lg", + "li", + "lij", + "lmo", + "ln", + "lo", + "lt", + "ltg", + "luo", + "lus", + "lv", + "mai", + "mak", + "mg", + "mi", + "min", + "mk", + "ml", + "mn", + "mni-mtei", + "mr", + "ms", + "ms-arab", + "mt", + "my", + "ne", + "new", + "nl", + "no", + "nr", + "nso", + "nus", + "ny", + "oc", + "om", + "or", + "pa", + "pa-arab", + "pag", + "pam", + "pap", + "pl", + "ps", + "pt", + "pt-pt", + "qu", + "rn", + "ro", + "rom", + "ru", + "rw", + "sa", + "scn", + "sd", + "sg", + "shn", + "si", + "sk", + "sl", + "sm", + "sn", + "so", + "sq", + "sr", + "ss", + "st", + "su", + "sv", + "sw", + "szl", + "ta", + "te", + "tet", + "tg", + "th", + "ti", + "tk", + "tl", + "tn", + "tr", + "ts", + "tt", + "ug", + "uk", + "ur", + "uz", + "vi", + "xh", + "yi", + "yo", + "yua", + "yue", + "zh", + "zh-cn", + "zh-tw", + "zu" + ], + "target": [ + "ab", + "ace", + "ach", + "af", + "ak", + "alz", + "am", + "ar", + "as", + "awa", + "ay", + "az", + "ba", + "ban", + "bbc", + "be", + "bem", + "bew", + "bg", + "bho", + "bik", + "bm", + "bn", + "br", + "bs", + "bts", + "btx", + "bua", + "ca", + "ceb", + "cgg", + "chm", + "ckb", + "cnh", + "co", + "crh", + "crs", + "cs", + "cv", + "cy", + "da", + "de", + "din", + "doi", + "dov", + "dv", + "dz", + "ee", + "el", + "en", + "eo", + "es", + "et", + "eu", + "fa", + "ff", + "fi", + "fj", + "fr", + "fr-ca", + "fy", + "ga", + "gaa", + "gd", + "gl", + "gn", + "gom", + "gu", + "ha", + "haw", + "he", + "hi", + "hil", + "hmn", + "hr", + "hrx", + "ht", + "hu", + "hy", + "id", + "ig", + "ilo", + "is", + "it", + "iw", + "ja", + "jv", + "jw", + "ka", + "kk", + "km", + "kn", + "ko", + "kri", + "ktu", + "ku", + "ky", + "la", + "lb", + "lg", + "li", + "lij", + "lmo", + "ln", + "lo", + "lt", + "ltg", + "luo", + "lus", + "lv", + "mai", + "mak", + "mg", + "mi", + "min", + "mk", + "ml", + "mn", + "mni-mtei", + "mr", + "ms", + "ms-arab", + "mt", + "my", + "ne", + "new", + "nl", + "no", + "nr", + "nso", + "nus", + "ny", + "oc", + "om", + "or", + "pa", + "pa-arab", + "pag", + "pam", + "pap", + "pl", + "ps", + "pt", + "pt-pt", + "qu", + "rn", + "ro", + "rom", + "ru", + "rw", + "sa", + "scn", + "sd", + "sg", + "shn", + "si", + "sk", + "sl", + "sm", + "sn", + "so", + "sq", + "sr", + "ss", + "st", + "su", + "sv", + "sw", + "szl", + "ta", + "te", + "tet", + "tg", + "th", + "ti", + "tk", + "tl", + "tn", + "tr", + "ts", + "tt", + "ug", + "uk", + "ur", + "uz", + "vi", + "xh", + "yi", + "yo", + "yua", + "yue", + "zh", + "zh-cn", + "zh-tw", + "zu" + ] +} diff --git a/data/languages/modernmt.json b/data/languages/modernmt.json new file mode 100644 index 0000000..19b4ff8 --- /dev/null +++ b/data/languages/modernmt.json @@ -0,0 +1,413 @@ +{ + "provider": "modernmt", + "captured_at": "2026-09-10", + "endpoint": "https://api.modernmt.com/translate/languages", + "source": [ + "ace", + "af", + "ak", + "als", + "am", + "ar", + "as", + "ast", + "awa", + "ayr", + "az", + "azb", + "azj", + "ba", + "ban", + "be", + "bem", + "bg", + "bho", + "bjn", + "bm", + "bn", + "bo", + "bs", + "bug", + "ca", + "ceb", + "cjk", + "ckb", + "crh", + "cs", + "cy", + "da", + "de", + "dik", + "diq", + "dyu", + "dz", + "ee", + "el", + "en", + "eo", + "es", + "es-419", + "es-es", + "et", + "fi", + "fj", + "fo", + "fon", + "fr", + "fur", + "fuv", + "ga", + "gaz", + "gd", + "gl", + "gn", + "gu", + "ha", + "he", + "hi", + "hne", + "hr", + "ht", + "hu", + "hy", + "id", + "ig", + "ilo", + "is", + "it", + "ja", + "jv", + "ka", + "kab", + "kac", + "kam", + "kas", + "kbp", + "kea", + "kg", + "khk", + "ki", + "kk", + "km", + "kmb", + "kmr", + "kn", + "knc", + "ko", + "ks", + "ky", + "la", + "lb", + "lg", + "li", + "lij", + "lmo", + "ln", + "lo", + "lt", + "ltg", + "lua", + "luo", + "lus", + "lv", + "lvs", + "mag", + "mai", + "mg", + "mi", + "min", + "mk", + "ml", + "mn", + "mni", + "mos", + "mr", + "ms", + "mt", + "my", + "nb", + "ne", + "nl", + "nn", + "nso", + "nus", + "ny", + "oc", + "or", + "pa", + "pag", + "pap", + "pbt", + "pes", + "pl", + "plt", + "prs", + "ps", + "pt", + "pt-br", + "pt-pt", + "quy", + "rn", + "ro", + "ru", + "rw", + "sa", + "sat", + "sc", + "scn", + "sd", + "sg", + "shn", + "si", + "sk", + "sl", + "sm", + "sn", + "so", + "sq", + "sr", + "ss", + "st", + "su", + "sv", + "sw", + "szl", + "ta", + "taq", + "te", + "tg", + "th", + "ti", + "tk", + "tl", + "tn", + "tpi", + "tr", + "ts", + "tt", + "tum", + "tw", + "tzm", + "ug", + "uk", + "umb", + "ur", + "uzn", + "vec", + "vi", + "war", + "wo", + "xh", + "ydd", + "yo", + "zh", + "zh-cn", + "zh-tw", + "zsm", + "zu" + ], + "target": [ + "ace", + "af", + "ak", + "als", + "am", + "ar", + "as", + "ast", + "awa", + "ayr", + "az", + "azb", + "azj", + "ba", + "ban", + "be", + "bem", + "bg", + "bho", + "bjn", + "bm", + "bn", + "bo", + "bs", + "bug", + "ca", + "ceb", + "cjk", + "ckb", + "crh", + "cs", + "cy", + "da", + "de", + "dik", + "diq", + "dyu", + "dz", + "ee", + "el", + "en", + "eo", + "es", + "es-419", + "es-es", + "et", + "fi", + "fj", + "fo", + "fon", + "fr", + "fur", + "fuv", + "ga", + "gaz", + "gd", + "gl", + "gn", + "gu", + "ha", + "he", + "hi", + "hne", + "hr", + "ht", + "hu", + "hy", + "id", + "ig", + "ilo", + "is", + "it", + "ja", + "jv", + "ka", + "kab", + "kac", + "kam", + "kas", + "kbp", + "kea", + "kg", + "khk", + "ki", + "kk", + "km", + "kmb", + "kmr", + "kn", + "knc", + "ko", + "ks", + "ky", + "la", + "lb", + "lg", + "li", + "lij", + "lmo", + "ln", + "lo", + "lt", + "ltg", + "lua", + "luo", + "lus", + "lv", + "lvs", + "mag", + "mai", + "mg", + "mi", + "min", + "mk", + "ml", + "mn", + "mni", + "mos", + "mr", + "ms", + "mt", + "my", + "nb", + "ne", + "nl", + "nn", + "nso", + "nus", + "ny", + "oc", + "or", + "pa", + "pag", + "pap", + "pbt", + "pes", + "pl", + "plt", + "prs", + "ps", + "pt", + "pt-br", + "pt-pt", + "quy", + "rn", + "ro", + "ru", + "rw", + "sa", + "sat", + "sc", + "scn", + "sd", + "sg", + "shn", + "si", + "sk", + "sl", + "sm", + "sn", + "so", + "sq", + "sr", + "ss", + "st", + "su", + "sv", + "sw", + "szl", + "ta", + "taq", + "te", + "tg", + "th", + "ti", + "tk", + "tl", + "tn", + "tpi", + "tr", + "ts", + "tt", + "tum", + "tw", + "tzm", + "ug", + "uk", + "umb", + "ur", + "uzn", + "vec", + "vi", + "war", + "wo", + "xh", + "ydd", + "yo", + "zh", + "zh-cn", + "zh-tw", + "zsm", + "zu" + ] +} diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index fa1639d..4918118 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -8,7 +8,12 @@ require "translation_diff/version" require "translation_diff/error" require "translation_diff/errors" +require "translation_diff/redaction" require "translation_diff/capabilities" +require "json" +require "translation_diff/languages" +require "translation_diff/languages/set" +require "translation_diff/languages/refresh" require "translation_diff/translation/usage" require "translation_diff/translation/request" require "translation_diff/translation/response" @@ -44,6 +49,7 @@ require "translation_diff/redis_cache_store" require "translation_diff/redis_rate_limiter" require "translation_diff/instrumentation" +require "translation_diff/dispatcher" require "translation_diff/translator" require "translation_diff/context" @@ -59,9 +65,10 @@ def reset! = @config = nil # An isolated copy of the configuration with the same entry point, for per-tenant settings. def context(&) = Context.new(config.copy.tap(&)) - # `provider:` and `config:` are reserved; every other keyword is forwarded to the provider. - def translate(values, from: nil, to: nil, provider: nil, **) - Translator.new(values, from: from, to: to, provider: provider, config: config, **).call + # `provider:`, `config:` and `assume_supported:` are reserved; every other keyword is forwarded to the provider. + def translate(values, from: nil, to: nil, provider: nil, assume_supported: false, **) + Translator.new(values, from: from, to: to, provider: provider, config: config, + assume_supported: assume_supported, **).call end end end diff --git a/lib/translation_diff/context.rb b/lib/translation_diff/context.rb index 44f9a12..5631413 100644 --- a/lib/translation_diff/context.rb +++ b/lib/translation_diff/context.rb @@ -6,9 +6,10 @@ def initialize(config) @config = config end - def translate(values, from: nil, to: nil, provider: nil, **) + def translate(values, from: nil, to: nil, provider: nil, assume_supported: false, **) TranslationDiff::Translator.new( - values, from: from, to: to, provider: provider, config: config, ** + values, from: from, to: to, provider: provider, config: config, + assume_supported: assume_supported, ** ).call end end diff --git a/lib/translation_diff/dispatcher.rb b/lib/translation_diff/dispatcher.rb new file mode 100644 index 0000000..2f6c6b9 --- /dev/null +++ b/lib/translation_diff/dispatcher.rb @@ -0,0 +1,54 @@ +# Sends batches to the provider, throttles each one, and reports what it cost -- the seam between cache and wire. +class TranslationDiff::Dispatcher + include TranslationDiff::Instrumentation + + attr_reader :config + + def initialize(provider:, from:, to:, options: {}, config: nil) + @provider = provider + @from = from + @to = to + @options = options + @config = config || TranslationDiff.config + end + + def dispatch(segments) + batches = TranslationDiff::Batch.pack(segments, capabilities: @provider.class.capabilities) + batches.each { |batch| send_batch(batch) } + end + + private + + # The batch applies the reply to the segments that produced it, so no step ever correlates by position again. + def send_batch(batch) + texts = batch.texts + payload = { provider: @provider.cache_key, batch: texts.size, characters: texts.sum(&:size) } + throttle(payload[:characters]) + response = instrument("request", payload) { @provider.translate(request(texts)) } + report_usage(response, payload[:characters]) + batch.apply(response.texts) + end + + def request(texts) + TranslationDiff::Translation::Request.new(texts: texts, from: @from, to: @to, options: @options) + end + + # Consulted with what is about to be sent, before it is sent; nil means no rate limiting was configured at all. + def throttle(characters) + limiter = config.rate_limiter_instance + return if limiter.nil? + + instrument("rate_limit", provider: @provider.cache_key, characters: characters) { limiter.check(characters) } + end + + # A point event: what this request cost, as this library counted it -- a provider's own count never overrides it. + def report_usage(response, characters) + usage = response.usage + + instrument("usage", provider: @provider.cache_key, + characters: characters, + billed_characters: usage&.billed_characters, + reported: @provider.class.capabilities.reports_billing?, + model: usage&.model) + end +end diff --git a/lib/translation_diff/errors.rb b/lib/translation_diff/errors.rb index 974d48d..12854d1 100644 --- a/lib/translation_diff/errors.rb +++ b/lib/translation_diff/errors.rb @@ -2,6 +2,9 @@ module TranslationDiff class ConfigurationError < Error; end + # Raised before any request: the pair is checked against data captured from the vendor, not by asking it. + class UnsupportedLanguageError < Error; end + class ProviderError < Error attr_reader :provider, :status diff --git a/lib/translation_diff/http_provider.rb b/lib/translation_diff/http_provider.rb index b591578..e45260d 100644 --- a/lib/translation_diff/http_provider.rb +++ b/lib/translation_diff/http_provider.rb @@ -45,6 +45,15 @@ def post(url, payload) raise TranslationDiff::TransportError, "#{self.class}: #{e.class}: #{e.message}" end + def get(url) + raw = connection.get(url) + response = Decoded.new(status: raw.status, headers: raw.headers, body: decode(raw)) + raise_for_status!(response) + response + rescue *TRANSPORT_FAILURES => e + raise TranslationDiff::TransportError, "#{self.class}: #{e.class}: #{e.message}" + end + # Faraday's JSON middleware passes parser options positionally, which json 3 (default on Ruby 4.x) removed. def decode(response) body = response.body diff --git a/lib/translation_diff/languages.rb b/lib/translation_diff/languages.rb new file mode 100644 index 0000000..7143537 --- /dev/null +++ b/lib/translation_diff/languages.rb @@ -0,0 +1,30 @@ +# What each provider translates, captured from the vendor and shipped; unknown means no opinion, never a refusal. +module TranslationDiff::Languages + DIRECTORY = File.expand_path("../../data/languages", __dir__).freeze + # Derived from the directory itself: shipping a new file makes it load without editing this list too. + SHIPPED = Dir.children(DIRECTORY).grep(/\.json\z/).map { |file| File.basename(file, ".json").to_sym }.sort.freeze + # Amazon needs AWS credentials most maintainers lack; LibreTranslate's list is one private instance's own. + NOT_SHIPPED = %i[amazon libretranslate].freeze + + class << self + # nil, not false: a provider we ship no data for must never refuse a pair. + def supports?(provider, from:, to:) + set = self.for(provider) + return nil if set.nil? # rubocop:disable Style/ReturnNilInPredicateMethodDefinition + + set.supports_source?(from) && set.supports_target?(to) + end + + def for(provider) + name = provider.to_s + return nil unless SHIPPED.include?(name.to_sym) + + sets[name] ||= Set.load(File.join(DIRECTORY, "#{name}.json")) + end + + private + + # Loaded on first use: six JSON files read at require time would slow every application that never validates. + def sets = @sets ||= {} + end +end diff --git a/lib/translation_diff/languages/refresh.rb b/lib/translation_diff/languages/refresh.rb new file mode 100644 index 0000000..8bbe2c2 --- /dev/null +++ b/lib/translation_diff/languages/refresh.rb @@ -0,0 +1,70 @@ +# A maintainer's tool: re-fetches every provider's lists and rewrites the shipped files. +class TranslationDiff::Languages::Refresh + def self.call(providers:, directory: TranslationDiff::Languages::DIRECTORY, on: Time.now.strftime("%Y-%m-%d")) + new(providers: providers, directory: directory, on: on).call + end + + def initialize(providers:, directory:, on:) + @providers = providers + @directory = directory + @on = on + end + + # A provider whose fetch fails keeps its previous file: an emptied list is worse than a stale one. + def call + report = { updated: [], failed: {}, skipped: [] } + + @providers.each do |provider| + name = provider.cache_key + fetched = fetch(provider, name, report) + next if fetched.nil? + + persist(name, provider, fetched, report) + end + + report + end + + private + + def fetch(provider, name, report) + fetched = provider.languages + return report_empty(name, report) if empty?(fetched) + + fetched + rescue NotImplementedError + report[:skipped] << name + nil + rescue StandardError => e + report[:failed][name] = "#{e.class}: #{e.message}" + nil + end + + # A 200 with an empty or malformed body degrades to [] through Array(...); that is not data, it's a failure. + def empty?(fetched) = Array(fetched[:source]).empty? || Array(fetched[:target]).empty? + + def report_empty(name, report) + report[:failed][name] = "the vendor answered with an empty source or target list" + nil + end + + # A write that cannot land -- a read-only checkout, a full disk -- must not cost the rest of the run. + def persist(name, provider, fetched, report) + write(name, provider, fetched) + report[:updated] << name + rescue StandardError => e + report[:failed][name] = "#{e.class}: #{e.message}" + end + + def write(name, provider, fetched) + document = { "provider" => name, "captured_at" => @on, + "endpoint" => endpoint(provider), + "source" => normalise(fetched[:source]), "target" => normalise(fetched[:target]) } + + File.write(File.join(@directory, "#{name}.json"), "#{JSON.pretty_generate(document)}\n") + end + + def normalise(codes) = Array(codes).map { |code| code.to_s.downcase }.uniq.sort + + def endpoint(provider) = provider.languages_endpoint.to_s +end diff --git a/lib/translation_diff/languages/set.rb b/lib/translation_diff/languages/set.rb new file mode 100644 index 0000000..cb528da --- /dev/null +++ b/lib/translation_diff/languages/set.rb @@ -0,0 +1,53 @@ +# One provider's languages as captured on a date; source and target differ, so they are kept apart. +class TranslationDiff::Languages::Set + attr_reader :provider, :captured_at, :endpoint, :source, :target + + # ISO 639-1 macrolanguage codes callers write, mapped to the ISO 639-3 individual some vendors ship instead. + # One-way only: a vendor listing the individual has, by definition, covered its macro; the reverse is not true. + MACRO_ALIASES = { + "fa" => "pes", "uz" => "uzn", "yi" => "ydd", "om" => "gaz", "qu" => "quy", "ay" => "ayr", + "mn" => "khk", "ms" => "zsm", "lv" => "lvs", "mg" => "plt", "az" => "azj", "ps" => "pbt", + "sw" => "swh", "ku" => "kmr", "zh" => "cmn", "ne" => "npi", "or" => "ory", "sq" => "als" + }.freeze + + def self.load(path) + document = parse(path) + + new(provider: document["provider"], captured_at: document["captured_at"], + endpoint: document["endpoint"], source: document["source"], target: document["target"]) + end + + # A maintainer sees a path and the parser's own complaint instead of guessing which of the shipped files broke. + def self.parse(path) + JSON.parse(File.read(path)) + rescue JSON::ParserError => e + raise TranslationDiff::Error, "#{path} is not valid JSON: #{e.message}" + end + private_class_method :parse + + def initialize(provider:, captured_at:, endpoint:, source:, target:) + @provider = provider + @captured_at = captured_at + @endpoint = endpoint + @source = normalise(source) + @target = normalise(target) + freeze + end + + def supports_source?(code) = matches?(@source, code) + def supports_target?(code) = matches?(@target, code) + + private + + def normalise(codes) = Array(codes).map { |code| code.to_s.downcase }.freeze + + # Primary subtag in both directions, plus a macro wanted against the individual code this provider lists for it. + def matches?(codes, code) + wanted = code.to_s.downcase + return true if wanted.empty? + + primary = wanted.split("-").first + accepted = [primary, MACRO_ALIASES[primary]].compact + codes.any? { |known| known == wanted || accepted.include?(known.split("-").first) } + end +end diff --git a/lib/translation_diff/providers/amazon.rb b/lib/translation_diff/providers/amazon.rb index 8278655..99f353f 100644 --- a/lib/translation_diff/providers/amazon.rb +++ b/lib/translation_diff/providers/amazon.rb @@ -2,6 +2,7 @@ class TranslationDiff::Providers::Amazon < TranslationDiff::HTTPProvider SERVICE = "translate".freeze TARGET = "AWSShineFrontendService_20170701.TranslateText".freeze + LIST_LANGUAGES = "AWSShineFrontendService_20170701.ListLanguages".freeze CONTENT_TYPE = "application/x-amz-json-1.1".freeze # Amazon's own way of asking for detection; reaches Comprehend under the hood, in regions that have it. @@ -46,6 +47,17 @@ def detect(text) .fetch("SourceLanguageCode", nil)&.downcase end + # ListLanguages is a signed POST like every other Amazon call; the list serves both directions. + def languages + body = post_signed(JSON.generate({}), target: LIST_LANGUAGES).body + codes = Array(body["Languages"]).map { |entry| entry["LanguageCode"] } + + { source: codes, target: codes } + end + + # There is no distinct languages URL: every Amazon call, ListLanguages included, is a signed POST to the root. + def languages_endpoint = "#{api_base}/" + private def call(text, request) @@ -58,8 +70,8 @@ def call(text, request) post_signed(JSON.generate(payload)).body end - def post_signed(body) - raw = connection.post("/", body, signed_headers(body)) + def post_signed(body, target: TARGET) + raw = connection.post("/", body, signed_headers(body, target: target)) response = decoded_response(raw) raise_for_status!(response) response @@ -92,13 +104,13 @@ def require_sigv4 'Add `gem "aws-sigv4"` to your Gemfile.' end - def signed_headers(body) + def signed_headers(body, target: TARGET) signature = signer.sign_request( http_method: "POST", url: "#{api_base}/", body: body, - headers: { "Content-Type" => CONTENT_TYPE, "X-Amz-Target" => TARGET } + headers: { "Content-Type" => CONTENT_TYPE, "X-Amz-Target" => target } ) - signature.headers.merge("Content-Type" => CONTENT_TYPE, "X-Amz-Target" => TARGET) + signature.headers.merge("Content-Type" => CONTENT_TYPE, "X-Amz-Target" => target) end # The signature covers the body exactly as sent, so this omits `faraday.request :json` unlike the base class. diff --git a/lib/translation_diff/providers/azure.rb b/lib/translation_diff/providers/azure.rb index 9cc4d04..dde26e4 100644 --- a/lib/translation_diff/providers/azure.rb +++ b/lib/translation_diff/providers/azure.rb @@ -53,6 +53,15 @@ def detect(text) response.body.dig(0, "language")&.downcase end + # Azure's own docs mark this endpoint public, but this provider still requires a key to be built at all. + def languages + codes = Array(get("languages?api-version=#{API_VERSION}&scope=translation").body["translation"]&.keys) + + { source: codes, target: codes } + end + + def languages_endpoint = "#{api_base}/languages?api-version=#{API_VERSION}&scope=translation" + private # Defaults, then caller options, then mandatory fields: a caller must not displace the language pair. diff --git a/lib/translation_diff/providers/deepl.rb b/lib/translation_diff/providers/deepl.rb index 6283a3e..4e0db57 100644 --- a/lib/translation_diff/providers/deepl.rb +++ b/lib/translation_diff/providers/deepl.rb @@ -63,8 +63,18 @@ def detect(text) translate(request).detected_source end + def languages + { source: codes(get("v2/languages?type=source").body), + target: codes(get("v2/languages?type=target").body) } + end + + # The source-list URL is enough to document what #languages fetches; the target one differs only by query. + def languages_endpoint = "#{api_base}/v2/languages" + private + def codes(body) = Array(body).map { |entry| entry["language"] } + def free_key? = config.deepl_api_key.to_s.end_with?(FREE_KEY_SUFFIX) def usage_for(request, translations) diff --git a/lib/translation_diff/providers/google.rb b/lib/translation_diff/providers/google.rb index 020e4a8..5cf7650 100644 --- a/lib/translation_diff/providers/google.rb +++ b/lib/translation_diff/providers/google.rb @@ -48,6 +48,17 @@ def detect(text) response = post(detect_url, { q: [text] }) response.body.dig("data", "detections", 0, 0, "language")&.downcase end + + # One list, used in both directions. + def languages + codes = Array(get("language/translate/v2/languages?key=#{CGI.escape(config.google_api_key.to_s)}") + .body.dig("data", "languages")).map { |entry| entry["language"] } + + { source: codes, target: codes } + end + + # The key is left out: it is a credential, not part of what documents where this list comes from. + def languages_endpoint = "#{api_base}/language/translate/v2/languages" end TranslationDiff::Providers.register(:google, TranslationDiff::Providers::Google) diff --git a/lib/translation_diff/providers/libretranslate.rb b/lib/translation_diff/providers/libretranslate.rb index 5a2728a..ec6ccbb 100644 --- a/lib/translation_diff/providers/libretranslate.rb +++ b/lib/translation_diff/providers/libretranslate.rb @@ -50,6 +50,16 @@ def detect(text) post("detect", payload).body.dig(0, "language")&.downcase end + + # Each entry lists its own targets; a self-hosted instance answers for itself, which is the point. + def languages + entries = Array(get("languages").body) + + { source: entries.map { |entry| entry["code"] }, + target: entries.flat_map { |entry| Array(entry["targets"]) }.uniq } + end + + def languages_endpoint = "#{api_base}/languages" end TranslationDiff::Providers.register(:libretranslate, TranslationDiff::Providers::LibreTranslate) diff --git a/lib/translation_diff/providers/modernmt.rb b/lib/translation_diff/providers/modernmt.rb index b3bc68d..0231cb8 100644 --- a/lib/translation_diff/providers/modernmt.rb +++ b/lib/translation_diff/providers/modernmt.rb @@ -48,6 +48,14 @@ def detect(text) translate(request).detected_source end + def languages + codes = Array(get("translate/languages").body["data"]) + + { source: codes, target: codes } + end + + def languages_endpoint = "#{api_base}/translate/languages" + private def results_from(body) diff --git a/lib/translation_diff/translator.rb b/lib/translation_diff/translator.rb index 5d7bdb7..b51c531 100644 --- a/lib/translation_diff/translator.rb +++ b/lib/translation_diff/translator.rb @@ -7,8 +7,8 @@ class Error < TranslationDiff::Error; end attr_reader :config - # `provider:` and `config:` are reserved; every other keyword is forwarded to the provider untouched. - def initialize(values, from: nil, to: nil, provider: nil, config: nil, **options) + # `provider:`, `config:` and `assume_supported:` are reserved; every other keyword reaches the provider untouched. + def initialize(values, from: nil, to: nil, provider: nil, config: nil, assume_supported: false, **options) raise ArgumentError, "a translation needs a target language: pass `to:` a language code." if to.nil? @values = values @@ -17,6 +17,7 @@ def initialize(values, from: nil, to: nil, provider: nil, config: nil, **options @options = options @config = config || TranslationDiff.config @requested_provider = provider + @assume_supported = assume_supported end # Hands back the caller's value untouched unless something in it was actually translated. @@ -28,7 +29,7 @@ def call return @values if same_language?(@from) provider = resolve_provider - from = source_language(provider, segments) + from = resolve_source_language(provider, segments) return @values if same_language?(from) translated(document, passages, segments, provider, from) @@ -36,6 +37,31 @@ def call private + # `from:` given means the whole pair is already known, so it is validated once, up front. `from:` nil means + # the target alone is checked before a possibly billed #detect runs, and the full pair only once it answers. + def resolve_source_language(provider, segments) + return @from.tap { |from| ensure_supported!(provider, from) } unless @from.nil? + + ensure_supported!(provider, nil) + source_language(provider, segments).tap { |from| ensure_supported!(provider, from) } + end + + # nil means we ship no data for this provider, and silence is not evidence of absence. + # `from` nil (source not known yet) checks the target alone: a nil source is never itself refused. + def ensure_supported!(provider, from) + return if @assume_supported || !config.validate_languages + + supported = TranslationDiff::Languages.supports?(provider.cache_key, from: from, to: @to) + return if supported.nil? || supported + + raise TranslationDiff::UnsupportedLanguageError, + "Provider #{provider.cache_key} does not translate #{pair_description(from)}. If it does " \ + "now, pass `assume_supported: true` for this call, or set " \ + "`config.validate_languages = false`, and run `rake languages:refresh`." + end + + def pair_description(from) = from.nil? ? "to #{@to}" : "#{from} to #{@to}" + # The `translate` event wraps everything a call that reaches a provider does, and nothing an early return does. def translated(document, passages, segments, provider, from) values = TranslationDiff::Leaves.count(@values) @@ -105,7 +131,7 @@ def fill(provider, segments, from) cache = sentence_cache(provider, from) misses = cache.fill(segments) instrument("cache", provider: provider.cache_key, hits: segments.size - misses.size, misses: misses.size) - dispatch(provider, misses, from) + dispatcher(provider, from).dispatch(misses) cache.store(misses) end @@ -114,29 +140,7 @@ def sentence_cache(provider, from) from: from, to: @to, options: @options) end - def dispatch(provider, segments, from) - batches = TranslationDiff::Batch.pack(segments, capabilities: provider.class.capabilities) - batches.each { |batch| send_batch(provider, batch, from) } - end - - # The batch applies the reply to the segments that produced it, so no step ever correlates by position again. - def send_batch(provider, batch, from) - texts = batch.texts - payload = { provider: provider.cache_key, batch: texts.size, characters: texts.sum(&:size) } - throttle(provider, payload[:characters]) - response = instrument("request", payload) { provider.translate(request(texts, from)) } - batch.apply(response.texts) - end - - def request(texts, from) - TranslationDiff::Translation::Request.new(texts: texts, from: from, to: @to, options: @options) - end - - # Consulted with what is about to be sent, before it is sent; nil means no rate limiting was configured at all. - def throttle(provider, characters) - limiter = config.rate_limiter_instance - return if limiter.nil? - - instrument("rate_limit", provider: provider.cache_key, characters: characters) { limiter.check(characters) } + def dispatcher(provider, from) + TranslationDiff::Dispatcher.new(provider: provider, from: from, to: @to, options: @options, config: config) end end diff --git a/test/translation_diff/dispatcher_test.rb b/test/translation_diff/dispatcher_test.rb new file mode 100644 index 0000000..9d4d0cb --- /dev/null +++ b/test/translation_diff/dispatcher_test.rb @@ -0,0 +1,161 @@ +require "test_helper" + +class DispatcherTest < ConfiguredTest + class RecordingProvider < TranslationDiff::Provider + def self.capabilities + TranslationDiff::Capabilities.new( + max_request_size: 1_000, max_batch_size: 10, max_text_size: nil, + html: :none, notranslate: false, detects_language: false, reports_billing: false + ) + end + + attr_reader :requests + + def initialize(config) + super + @requests = [] + end + + def translate(request) + @requests << request + TranslationDiff::Translation::Response.build(request: request, texts: request.texts.map(&:upcase)) + end + + def cache_key = "recording" + end + + # Its Usage claims a wrong character count -- the library's own tally must win in the event anyway. + class MisreportingProvider < RecordingProvider + def self.capabilities + TranslationDiff::Capabilities.new( + max_request_size: 1_000, max_batch_size: 10, max_text_size: nil, + html: :none, notranslate: false, detects_language: false, reports_billing: true + ) + end + + def translate(request) + @requests << request + usage = TranslationDiff::Translation::Usage.new(characters: 999_999, billed_characters: 3, model: "x-model") + TranslationDiff::Translation::Response.build(request: request, texts: request.texts.map(&:upcase), usage: usage) + end + + def cache_key = "misreporting" + end + + class Recorder + attr_reader :events + + def initialize = @events = [] + + def instrument(name, payload) + @events << [name, payload] + yield if block_given? + end + end + + # Always lets the call through, so the `rate_limit` event fires without a real Redis connection. + class FakeRateLimiter + attr_reader :sizes + + def initialize = @sizes = [] + + def check(size) = @sizes << size + end + + def segments(*sources) = sources.map { |s| TranslationDiff::Segment.new(s) } + + def dispatcher(provider, **) + TranslationDiff::Dispatcher.new(provider: provider, from: "en", to: "ru", **) + end + + def configured(**settings) + config = TranslationDiff::Configuration.new + settings.each { |key, value| config.public_send("#{key}=", value) } + config + end + + # Dispatches the given texts through the given provider, and hands back everything it instrumented. + def instrumented(provider, *sources, **settings) + recorder = Recorder.new + config = configured(instrumenter: recorder, **settings) + dispatcher(provider, config: config).dispatch(segments(*sources)) + recorder + end + + def payload_for(recorder, event) + recorder.events.find { |name, _| name == "#{event}.translation_diff" }.last + end + + def test_dispatch_applies_the_reply_to_the_segments_that_produced_it + provider = RecordingProvider.new(TranslationDiff::Configuration.new) + segs = segments("one", "two") + + dispatcher(provider).dispatch(segs) + + assert_equal %w[ONE TWO], segs.map(&:translation) + end + + def test_the_request_carries_the_from_and_to_languages + provider = RecordingProvider.new(TranslationDiff::Configuration.new) + + dispatcher(provider).dispatch(segments("one")) + + request = provider.requests.first + assert_equal "en", request.from + assert_equal "ru", request.to + end + + def test_the_request_event_carries_the_provider_a_batch_size_and_a_character_count + recorder = instrumented(RecordingProvider.new(TranslationDiff::Configuration.new), "Hello there.") + payload = payload_for(recorder, "request") + + assert_equal "recording", payload[:provider] + assert_equal 1, payload[:batch] + assert_equal "Hello there.".size, payload[:characters] + end + + # The design says `characters` is what this library sent, always known, so a provider's own count never wins. + def test_the_usage_event_reports_the_locally_counted_characters_even_when_the_provider_misreports_them + recorder = instrumented(MisreportingProvider.new(TranslationDiff::Configuration.new), "one two") + payload = payload_for(recorder, "usage") + + assert_equal "one two".size, payload[:characters] + refute_equal 999_999, payload[:characters] + end + + def test_the_usage_event_still_reads_billed_characters_and_model_from_the_provider + recorder = instrumented(MisreportingProvider.new(TranslationDiff::Configuration.new), "one") + payload = payload_for(recorder, "usage") + + assert_equal 3, payload[:billed_characters] + assert_equal "x-model", payload[:model] + assert_equal true, payload[:reported] + end + + def test_no_rate_limit_event_without_a_rate_limiter + recorder = instrumented(RecordingProvider.new(TranslationDiff::Configuration.new), "one") + + refute_includes recorder.events.map(&:first), "rate_limit.translation_diff" + end + + def test_the_rate_limiter_is_consulted_before_the_request_with_the_characters_about_to_be_sent + limiter = FakeRateLimiter.new + provider = RecordingProvider.new(TranslationDiff::Configuration.new) + recorder = instrumented(provider, "one two", rate_limiter: limiter) + + names = recorder.events.map(&:first).select { |name| name.start_with?("rate_limit", "request") } + assert_equal %w[rate_limit.translation_diff request.translation_diff], names + assert_equal ["one two".size], limiter.sizes + end + + def test_no_payload_ever_contains_the_text_being_translated + secret = "Zaphod Beeblebrox is president." + provider = RecordingProvider.new(TranslationDiff::Configuration.new) + recorder = instrumented(provider, secret, rate_limiter: FakeRateLimiter.new) + + serialised = recorder.events.map { |name, payload| "#{name}#{payload}" }.join + refute_includes serialised, "Zaphod" + refute_includes serialised, secret + refute_includes serialised, secret.upcase + end +end diff --git a/test/translation_diff/http_provider_test.rb b/test/translation_diff/http_provider_test.rb index 51dad96..993087d 100644 --- a/test/translation_diff/http_provider_test.rb +++ b/test/translation_diff/http_provider_test.rb @@ -92,6 +92,21 @@ def test_a_connection_failure_becomes_a_transport_error assert_raises(TranslationDiff::TransportError) { provider.translate(request) } end + # #get is what every provider's #languages calls; a connection failure there must become the same StandardError + # #post's does, or Refresh's "keeps its previous file on any StandardError" discipline would not cover it. + def test_a_connection_failure_on_get_becomes_a_transport_error + @config.max_retries = 0 + stubs = Faraday::Adapter::Test::Stubs.new do |stub| + stub.get("/v1/languages") { raise Faraday::ConnectionFailed, "no route" } + end + provider = Echo.new(@config) + provider.instance_variable_set(:@connection, provider.send(:build_connection) do |faraday| + faraday.adapter :test, stubs + end) + + assert_raises(TranslationDiff::TransportError) { provider.send(:get, "v1/languages") } + end + # No line this library writes may carry source text or a credential. def test_no_logging_middleware_is_installed_even_when_a_logger_is_configured @config.logger = Logger.new(StringIO.new) diff --git a/test/translation_diff/instrumentation_test.rb b/test/translation_diff/instrumentation_test.rb index c6bd826..4874f8b 100644 --- a/test/translation_diff/instrumentation_test.rb +++ b/test/translation_diff/instrumentation_test.rb @@ -69,8 +69,54 @@ def test_the_rate_limit_event_carries_the_provider_and_a_character_count assert_equal "Hello there.".size, payload[:characters] end + # Reports billing, so `reported` is true and the count is the provider's own claim. + class Billing < TranslationDiff::Provider + def self.capabilities + TranslationDiff::Capabilities.new( + max_request_size: 1_000_000, max_batch_size: 1_000_000, max_text_size: nil, + html: :none, notranslate: false, detects_language: false, reports_billing: true + ) + end + + def translate(request) + TranslationDiff::Translation::Response.build( + request: request, texts: request.texts.map(&:to_s), + usage: TranslationDiff::Translation::Usage.new( + characters: request.texts.sum(&:size), billed_characters: 42, model: "billing-v1" + ) + ) + end + + def cache_key = "billing" + end + + def test_the_usage_event_carries_what_the_provider_reported + TranslationDiff.translate("Hello there.", from: "en", to: "ru", provider: Billing.new(TranslationDiff.config)) + + payload = usage_event_payload + + assert_equal "billing", payload[:provider] + assert_equal "Hello there.".size, payload[:characters] + assert_equal 42, payload[:billed_characters] + assert payload[:reported] + assert_equal "billing-v1", payload[:model] + end + + # nil billed_characters alone cannot distinguish "never says" from "did not say this time". + def test_a_provider_that_does_not_report_billing_says_so + TranslationDiff.translate("Hello there.", from: "en", to: "ru") + + payload = usage_event_payload + + assert_equal "null", payload[:provider] + assert_nil payload[:billed_characters] + refute payload[:reported] + assert_equal "Hello there.".size, payload[:characters] + end + ALL_EVENT_NAMES = %w[translate.translation_diff cache.translation_diff - request.translation_diff rate_limit.translation_diff].sort.freeze + request.translation_diff rate_limit.translation_diff + usage.translation_diff].sort.freeze # A guard that only checked payload content would pass even if an event quietly stopped firing. def test_no_payload_ever_contains_the_text_being_translated @@ -89,4 +135,10 @@ def test_translating_without_an_instrumenter_still_works assert_equal "Hello.", TranslationDiff.translate("Hello.", from: "en", to: "ru") end + + private + + def usage_event_payload + @recorder.events.find { |name, _| name == "usage.translation_diff" }.last + end end diff --git a/test/translation_diff/languages/refresh_test.rb b/test/translation_diff/languages/refresh_test.rb new file mode 100644 index 0000000..d393a66 --- /dev/null +++ b/test/translation_diff/languages/refresh_test.rb @@ -0,0 +1,123 @@ +require "test_helper" +require "tmpdir" + +class LanguagesRefreshTest < Minitest::Test + class Answering < TranslationDiff::Provider + def cache_key = "answering" + def languages = { source: %w[en de], target: %w[en de fr] } + end + + class Failing < TranslationDiff::Provider + def cache_key = "failing" + def languages = raise(TranslationDiff::TransportError, "nobody answered") + end + + class Silent < TranslationDiff::Provider + def cache_key = "silent" + end + + class Empty < TranslationDiff::Provider + def cache_key = "empty" + def languages = { source: [], target: [] } + end + + class Unwritable < TranslationDiff::Provider + def cache_key = "unwritable" + def languages = { source: %w[en], target: %w[en] } + end + + # api_base and languages_endpoint deliberately differ, so a regression that reads the former is caught. + class Detailed < TranslationDiff::Provider + def cache_key = "detailed" + def api_base = "https://api.detailed.test" + def languages_endpoint = "https://api.detailed.test/v9/languages?scope=translation" + def languages = { source: %w[en], target: %w[en] } + end + + def test_a_provider_that_answers_is_written_with_todays_date + Dir.mktmpdir do |dir| + report = TranslationDiff::Languages::Refresh.call( + providers: [Answering.new(config)], directory: dir, on: "2026-09-11" + ) + + document = JSON.parse(File.read(File.join(dir, "answering.json"))) + + assert_equal %w[answering], report[:updated] + assert_equal "2026-09-11", document["captured_at"] + assert_equal %w[de en], document["source"] + assert_equal %w[de en fr], document["target"] + end + end + + # A refresh that emptied a list because a network call timed out would be worse than never running. + def test_a_provider_whose_fetch_fails_keeps_its_previous_file_and_is_reported + Dir.mktmpdir do |dir| + path = File.join(dir, "failing.json") + File.write(path, JSON.generate({ "provider" => "failing", "source" => %w[en], "target" => %w[en] })) + + report = TranslationDiff::Languages::Refresh.call(providers: [Failing.new(config)], directory: dir) + + assert_equal %w[en], JSON.parse(File.read(path))["source"] + assert_empty report[:updated] + assert_match(/nobody answered/, report[:failed]["failing"]) + end + end + + def test_a_provider_that_cannot_fetch_its_languages_is_skipped_not_failed + Dir.mktmpdir do |dir| + report = TranslationDiff::Languages::Refresh.call(providers: [Silent.new(config)], directory: dir) + + assert_equal %w[silent], report[:skipped] + assert_empty report[:failed] + refute_path_exists File.join(dir, "silent.json") + end + end + + # A vendor answering 200 with an empty body must not blank a shipped file with ~190 codes in it. + def test_a_provider_whose_languages_come_back_empty_keeps_its_previous_file_and_is_reported + Dir.mktmpdir do |dir| + path = File.join(dir, "empty.json") + File.write(path, JSON.generate({ "provider" => "empty", "source" => %w[en], "target" => %w[en] })) + before = File.read(path) + + report = TranslationDiff::Languages::Refresh.call(providers: [Empty.new(config)], directory: dir) + + assert_equal before, File.read(path) + assert_empty report[:updated] + assert_match(/empty/i, report[:failed]["empty"]) + end + end + + # One provider's unwritable file must not cost the other providers in the same run their refresh. + def test_a_provider_whose_file_cannot_be_written_still_lets_the_rest_of_the_run_complete + Dir.mktmpdir do |dir| + Dir.mkdir(File.join(dir, "unwritable.json")) + + report = TranslationDiff::Languages::Refresh.call( + providers: [Unwritable.new(config), Answering.new(config)], directory: dir + ) + + assert_equal %w[answering], report[:updated] + assert report[:failed].key?("unwritable") + assert_equal %w[de en], source_for(dir, "answering") + end + end + + # DeepL on a free-plan key writes api-free.deepl.com through #api_base; the shipped file documents the metadata + # URL #languages itself fetches, which #languages_endpoint names precisely and #api_base alone cannot. + def test_the_written_endpoint_is_the_providers_own_languages_endpoint_not_its_api_base + Dir.mktmpdir do |dir| + TranslationDiff::Languages::Refresh.call(providers: [Detailed.new(config)], directory: dir) + + document = JSON.parse(File.read(File.join(dir, "detailed.json"))) + + assert_equal "https://api.detailed.test/v9/languages?scope=translation", document["endpoint"] + end + end + + private + + def config = TranslationDiff::Configuration.new + + def source_for(dir, name) = JSON.parse(File.read(File.join(dir, "#{name}.json")))["source"] +end diff --git a/test/translation_diff/languages/set_test.rb b/test/translation_diff/languages/set_test.rb new file mode 100644 index 0000000..c7bf8f1 --- /dev/null +++ b/test/translation_diff/languages/set_test.rb @@ -0,0 +1,38 @@ +require "test_helper" +require "tmpdir" + +class LanguagesSetTest < Minitest::Test + def test_a_corrupt_file_raises_naming_the_path_and_the_parsers_own_message + Dir.mktmpdir do |dir| + path = File.join(dir, "broken.json") + File.write(path, "not json at all") + parser_message = parser_message_for("not json at all") + + error = assert_raises(TranslationDiff::Error) { TranslationDiff::Languages::Set.load(path) } + + assert_includes error.message, path + assert_includes error.message, parser_message + end + end + + def parser_message_for(garbage) + JSON.parse(garbage) + rescue JSON::ParserError => e + e.message + end + + def set(codes) + TranslationDiff::Languages::Set.new(provider: "x", captured_at: "2026-01-01", endpoint: "", + source: codes, target: codes) + end + + # A vendor publishing the individual code has, by definition, covered the macro it belongs to. + def test_a_macro_code_matches_a_vendor_that_only_publishes_the_individual_code + assert set(%w[en pes]).supports_target?("fa") + end + + # The reverse does not hold: a vendor publishing only the macro has not promised the individual. + def test_an_individual_code_does_not_match_a_vendor_that_only_publishes_the_macro + refute set(%w[en zh]).supports_target?("cmn") + end +end diff --git a/test/translation_diff/languages_test.rb b/test/translation_diff/languages_test.rb new file mode 100644 index 0000000..7c72b7e --- /dev/null +++ b/test/translation_diff/languages_test.rb @@ -0,0 +1,85 @@ +require "test_helper" + +class LanguagesTest < Minitest::Test + def test_a_shipped_provider_answers_for_a_pair_it_supports + assert TranslationDiff::Languages.supports?(:deepl, from: "en", to: "ru") + assert TranslationDiff::Languages.supports?("google", from: "en", to: "ru") + end + + def test_a_pair_the_provider_does_not_do_is_refused + refute TranslationDiff::Languages.supports?(:deepl, from: "en", to: "klingon") + end + + # Three states, not two: an unknown provider has no opinion, and no opinion never refuses. + def test_an_unknown_provider_answers_nil + assert_nil TranslationDiff::Languages.supports?(:amazon, from: "en", to: "ru") + assert_nil TranslationDiff::Languages.supports?(:libretranslate, from: "en", to: "ru") + assert_nil TranslationDiff::Languages.supports?(:whatever, from: "en", to: "ru") + assert_nil TranslationDiff::Languages.for(:whatever) + end + + # Matching is on the primary subtag in both directions: a bare entry takes a regional code and back. + def test_a_regional_code_matches_a_bare_entry_and_the_other_way_round + assert TranslationDiff::Languages.supports?(:deepl, from: "en-GB", to: "ru") + assert TranslationDiff::Languages.supports?(:deepl, from: "en", to: "pt") + assert TranslationDiff::Languages.supports?(:deepl, from: "EN", to: "PT-BR") + end + + def test_source_and_target_are_kept_apart + set = TranslationDiff::Languages.for(:deepl) + + assert_includes set.target, "en-gb" + refute_includes set.source, "en-gb" + assert_includes set.source, "en" + end + + def test_a_set_records_when_and_where_it_was_captured + set = TranslationDiff::Languages.for(:azure) + + assert_equal "2026-09-10", set.captured_at + assert_includes set.endpoint, "microsofttranslator.com" + end + + # A nil code cannot be validated, so it is not refused -- detection has not run yet. + def test_a_missing_code_is_not_refused + assert TranslationDiff::Languages.supports?(:deepl, from: nil, to: "ru") + end + + # ModernMT ships ISO 639-3 individual codes where callers write the 639-1 macrolanguage code. + def test_a_macrolanguage_code_matches_the_vendors_iso_639_3_individual_code + assert TranslationDiff::Languages.supports?(:modernmt, from: "en", to: "fa") + assert TranslationDiff::Languages.supports?(:modernmt, from: "fa", to: "en") + end + + # The alias only runs macro -> individual: a vendor publishing only "zh" has not promised to accept "cmn". + def test_the_macro_alias_does_not_run_backwards + refute TranslationDiff::Languages.supports?(:deepl, from: "en", to: "cmn") + assert TranslationDiff::Languages.supports?(:deepl, from: "en", to: "zh") + end + + # The live leaks: none of these vendors publish the individual code, so none should accept it. + def test_deepl_does_not_accept_individual_codes_it_never_published + refute TranslationDiff::Languages.supports?(:deepl, from: "en", to: "pes") + refute TranslationDiff::Languages.supports?(:deepl, from: "en", to: "swh") + end + + # Derived from the directory, not hardcoded, so a maintainer who ships amazon.json needs no second place to say so. + def test_shipped_is_derived_from_the_data_files_actually_present + assert_equal %i[azure deepl google modernmt], TranslationDiff::Languages::SHIPPED + end + + # Amazon needs AWS credentials most maintainers lack; LibreTranslate's list is one private instance's own. + def test_not_shipped_names_amazon_and_libretranslate + assert_equal %i[amazon libretranslate], TranslationDiff::Languages::NOT_SHIPPED + end + + def test_every_shipped_file_parses_and_carries_both_lists + TranslationDiff::Languages::SHIPPED.each do |name| + set = TranslationDiff::Languages.for(name) + + refute_empty set.source, name.to_s + refute_empty set.target, name.to_s + assert_match(/\A\d{4}-\d{2}-\d{2}\z/, set.captured_at) + end + end +end diff --git a/test/translation_diff/providers/languages_test.rb b/test/translation_diff/providers/languages_test.rb new file mode 100644 index 0000000..52540aa --- /dev/null +++ b/test/translation_diff/providers/languages_test.rb @@ -0,0 +1,133 @@ +require "test_helper" +require "faraday" +require "aws-sigv4" + +class ProviderLanguagesTest < Minitest::Test + JSON_HEADERS = { "Content-Type" => "application/json" }.freeze + + def setup = TranslationDiff.reset! + + def test_deepl_asks_for_both_directions + provider = build(TranslationDiff::Providers::DeepL, deepl_api_key: "test-key:fx") do |stub| + stub.get("/v2/languages?type=source") { [200, JSON_HEADERS, JSON.generate([{ "language" => "EN" }])] } + stub.get("/v2/languages?type=target") { [200, JSON_HEADERS, JSON.generate([{ "language" => "EN-GB" }])] } + end + + assert_equal({ source: %w[EN], target: %w[EN-GB] }, provider.languages) + end + + # The source-list URL is enough to document what #languages fetches; DeepL also asks a target one. + def test_deepls_languages_endpoint_is_the_source_list_url + provider = build(TranslationDiff::Providers::DeepL, deepl_api_key: "test-key") + + assert_equal "https://api.deepl.com/v2/languages", provider.languages_endpoint + end + + def test_google_asks_once_and_uses_the_same_list_both_ways + provider = build(TranslationDiff::Providers::Google, google_api_key: "test-key") do |stub| + stub.get("/language/translate/v2/languages?key=test-key") do + [200, JSON_HEADERS, + JSON.generate({ "data" => { "languages" => [{ "language" => "en" }, { "language" => "ru" }] } })] + end + end + + assert_equal({ source: %w[en ru], target: %w[en ru] }, provider.languages) + end + + def test_googles_languages_endpoint_omits_the_credential + provider = build(TranslationDiff::Providers::Google, google_api_key: "test-key") + + assert_equal "https://translation.googleapis.com/language/translate/v2/languages", provider.languages_endpoint + end + + def test_azure_reads_the_translation_hashs_keys + provider = build(TranslationDiff::Providers::Azure, azure_api_key: "test-key") do |stub| + stub.get("/languages?api-version=3.0&scope=translation") do + [200, JSON_HEADERS, JSON.generate({ "translation" => { "en" => {}, "ru" => {} } })] + end + end + + assert_equal({ source: %w[en ru], target: %w[en ru] }, provider.languages) + end + + def test_azures_languages_endpoint_is_the_full_url_it_fetches + provider = build(TranslationDiff::Providers::Azure, azure_api_key: "test-key") + + assert_equal "https://api.cognitive.microsofttranslator.com/languages?api-version=3.0&scope=translation", + provider.languages_endpoint + end + + def test_modernmt_reads_the_data_array + provider = build(TranslationDiff::Providers::ModernMT, modernmt_api_key: "test-key") do |stub| + stub.get("/translate/languages") { [200, JSON_HEADERS, JSON.generate({ "data" => %w[en ru] })] } + end + + assert_equal({ source: %w[en ru], target: %w[en ru] }, provider.languages) + end + + def test_modernmts_languages_endpoint_is_the_full_url_it_fetches + provider = build(TranslationDiff::Providers::ModernMT, modernmt_api_key: "test-key") + + assert_equal "https://api.modernmt.com/translate/languages", provider.languages_endpoint + end + + def test_libretranslate_reads_each_entrys_own_targets + provider = build(TranslationDiff::Providers::LibreTranslate, + libretranslate_api_base: "https://libretranslate.test") do |stub| + stub.get("/languages") do + [200, JSON_HEADERS, + JSON.generate([{ "code" => "en", "targets" => %w[ru fr] }, { "code" => "fr", "targets" => %w[en] }])] + end + end + + assert_equal({ source: %w[en fr], target: %w[ru fr en] }, provider.languages) + end + + def test_libretranslates_languages_endpoint_is_the_full_url_it_fetches + provider = build(TranslationDiff::Providers::LibreTranslate, + libretranslate_api_base: "https://libretranslate.test") + + assert_equal "https://libretranslate.test/languages", provider.languages_endpoint + end + + def test_amazon_sends_the_list_languages_target_and_reads_language_codes + requests = [] + provider = build(TranslationDiff::Providers::Amazon, + amazon_access_key_id: "AKIAEXAMPLE", amazon_secret_access_key: "secret", + amazon_region: "eu-central-1") { |stub| stub.post("/", &record(requests)) } + + assert_equal({ source: %w[en ru], target: %w[en ru] }, provider.languages) + assert_equal "AWSShineFrontendService_20170701.ListLanguages", requests.first.request_headers["X-Amz-Target"] + end + + # Amazon has no distinct languages URL: every call, including ListLanguages, is a signed POST to the root. + def test_amazons_languages_endpoint_is_its_signed_root + provider = build(TranslationDiff::Providers::Amazon, + amazon_access_key_id: "AKIAEXAMPLE", amazon_secret_access_key: "secret", + amazon_region: "eu-central-1") + + assert_equal "https://translate.eu-central-1.amazonaws.com/", provider.languages_endpoint + end + + private + + # Records the raw request so the header assertion can inspect what was actually signed and sent. + def record(requests) + lambda do |env| + requests << env.dup + [200, JSON_HEADERS, JSON.generate({ "Languages" => [{ "LanguageCode" => "en" }, { "LanguageCode" => "ru" }] })] + end + end + + def build(provider_class, **config_values, &) + config = TranslationDiff::Configuration.new + config_values.each { |key, value| config.public_send(:"#{key}=", value) } + stubs = Faraday::Adapter::Test::Stubs.new(&) + + provider_class.new(config).tap do |built| + built.name = :"languages-test" + built.instance_variable_set(:@connection, + built.send(:build_connection) { |faraday| faraday.adapter :test, stubs }) + end + end +end diff --git a/test/translation_diff/translator_test.rb b/test/translation_diff/translator_test.rb index e84fbab..2409956 100644 --- a/test/translation_diff/translator_test.rb +++ b/test/translation_diff/translator_test.rb @@ -251,7 +251,8 @@ def test_a_call_that_returns_early_emits_no_events end ALL_EVENT_NAMES = %w[translate.translation_diff cache.translation_diff - request.translation_diff rate_limit.translation_diff].sort.freeze + request.translation_diff rate_limit.translation_diff + usage.translation_diff].sort.freeze # A guard that only checked payload content would pass even if an event quietly stopped firing. def test_no_payload_ever_contains_the_text_being_translated @@ -340,6 +341,116 @@ def test_a_missing_target_language_is_refused_by_name assert_match(/to:/, error.message) end + # A provider whose languages we ship, driven with a pair it does not do. + class DeepLDouble < TranslationDiff::Providers::DeepL + attr_reader :requests + + def translate(request) + (@requests ||= []) << request + TranslationDiff::Translation::Response.build(request: request, texts: request.texts.map(&:to_s)) + end + + def cache_key = "deepl" + end + + def deepl_double + TranslationDiff.configure { |c| c.deepl_api_key = "test-key" } + DeepLDouble.new(TranslationDiff.config) + end + + def test_an_unsupported_pair_raises_before_a_request_is_made + provider = deepl_double + + error = assert_raises(TranslationDiff::UnsupportedLanguageError) do + TranslationDiff.translate("Hello there.", from: "en", to: "klingon", provider: provider) + end + + assert_nil provider.requests + assert_match(/klingon/, error.message) + assert_match(/deepl/, error.message) + assert_match(/assume_supported/, error.message) + end + + def test_a_supported_pair_goes_through + assert_equal "Hello there.", + TranslationDiff.translate("Hello there.", from: "en", to: "ru", provider: deepl_double) + end + + def test_the_per_call_escape_lets_an_unlisted_pair_through + assert_equal "Hello there.", + TranslationDiff.translate("Hello there.", from: "en", to: "klingon", + provider: deepl_double, assume_supported: true) + end + + def test_the_global_switch_turns_validation_off + TranslationDiff.configure { |c| c.validate_languages = false } + + assert_equal "Hello there.", + TranslationDiff.translate("Hello there.", from: "en", to: "klingon", provider: deepl_double) + end + + # Reserved, like provider: and config: -- every other keyword is forwarded to the vendor untouched. + def test_assume_supported_never_reaches_the_provider + provider = deepl_double + TranslationDiff.translate("Hello there.", from: "en", to: "ru", provider: provider, assume_supported: true) + + refute_includes provider.requests.first.options.keys, :assume_supported + end + + def test_a_provider_we_ship_no_data_for_refuses_nothing + assert_equal "Hello there.", + TranslationDiff.translate("Hello there.", from: "en", to: "klingon", provider: :null) + end + + # ModernMT genuinely translates Persian; it just publishes it under its ISO 639-3 individual code "pes". + class ModernMTDouble < TranslationDiff::Providers::ModernMT + def translate(request) + TranslationDiff::Translation::Response.build(request: request, texts: request.texts.map(&:to_s)) + end + + def cache_key = "modernmt" + end + + def test_modernmt_translates_a_macrolanguage_it_only_lists_under_its_individual_code + TranslationDiff.configure { |c| c.modernmt_api_key = "test-key" } + provider = ModernMTDouble.new(TranslationDiff.config) + + assert_equal "Hello there.", + TranslationDiff.translate("Hello there.", from: "en", to: "fa", provider: provider) + end + + # A provider whose #detect would cost a real request if it were ever reached. + class ExplodingOnDetectDouble < DeepLDouble + def detect(_text) = raise "detect must never be called when the target alone is already unsupported" + end + + def test_an_unsupported_target_is_refused_before_a_provider_is_asked_to_detect + TranslationDiff.configure { |c| c.deepl_api_key = "test-key" } + provider = ExplodingOnDetectDouble.new(TranslationDiff.config) + + error = assert_raises(TranslationDiff::UnsupportedLanguageError) do + TranslationDiff.translate("Hello there.", to: "klingon", provider: provider) + end + + assert_match(/klingon/, error.message) + end + + # When `from:` is given the whole pair is already known, so there is nothing to defer: validate it once. + def test_a_pair_given_up_front_is_validated_exactly_once + original = TranslationDiff::Languages.method(:supports?) + calls = 0 + TranslationDiff::Languages.define_singleton_method(:supports?) do |*args, **kwargs| + calls += 1 + original.call(*args, **kwargs) + end + + translate("one.", from: "en", to: "ru") + + assert_equal 1, calls + ensure + TranslationDiff::Languages.define_singleton_method(:supports?, &original) + end + private def instrumented From d77f8ca33a3297461b9d99e56afadc223a0e91fc Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 16:57:36 +0400 Subject: [PATCH 3/3] docs: document languages, the usage event and redacted inspect --- CHANGELOG.md | 39 +++++++++++++++++ README.md | 6 ++- docs/configuration.md | 1 + docs/contracts.md | 6 +-- docs/errors.md | 3 ++ docs/how-it-works.md | 23 ++++++---- docs/instrumentation.md | 15 ++++++- docs/languages.md | 94 +++++++++++++++++++++++++++++++++++++++++ docs/providers.md | 58 +++++++++++++++++++++---- 9 files changed, 222 insertions(+), 23 deletions(-) create mode 100644 docs/languages.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 53d7ff4..f5692ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,45 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## [Unreleased] + +### Breaking + +- **Language validation is on by default.** `TranslationDiff.translate` now + refuses, before making a request, any source/target pair the shipped data + doesn't list for that provider -- raising + `TranslationDiff::UnsupportedLanguageError`. Data ships for DeepL, Google, + Azure and ModernMT only; Amazon and LibreTranslate ship none, and a + provider with no shipped data refuses nothing. Two escapes: pass + `assume_supported: true` for one call, or set + `config.validate_languages = false` globally. See + [Languages](docs/languages.md). + +### Added + +- A `usage` instrumentation event, firing once per provider request, beside + `translate`, `cache`, `request` and `rate_limit`. Its payload carries + `provider`, `characters` (what this library sent, counted locally), + `billed_characters` (what the provider said it charged, or `nil`), + `reported` (whether the provider reports billing **at all** -- not that + this response was billed) and `model`. Summing `billed_characters` across + providers without checking `reported` first produces a total that is + quietly too low: only three of the six built-in providers report billing + at all. See [Instrumentation](docs/instrumentation.md). + +### Security + +- `Configuration#inspect` and `Provider#inspect` print `[FILTERED]` in place + of every credential option's value, instead of the credential itself. The + filtered set is derived, not hand-maintained: option names matching a + sensitive pattern, plus whatever each registered provider declares in + `sensitive_options`. A non-credential option -- a base URL, a region, + `cache_namespace` -- stays visible in full. A URL-valued option that + carries a credential in its userinfo, `redis_url` included, has just that + part redacted (`rediss://default:[FILTERED]@cache.example.upstash.io:6379`); + the scheme, host, port and path stay visible. See + [Providers](docs/providers.md). + ## [3.1.0] - 2026-09-08 First release under the name **translation_diff**. This gem was published as diff --git a/README.md b/README.md index 8b9d6a2..bfc9981 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,9 @@ TranslationDiff.translate(blog_post, from: "en", to: "de", provider: :google) ``` ```ruby -# Any keyword other than from:, to:, provider: and config: is forwarded to the provider +# Any keyword other than from:, to:, provider:, config: and assume_supported: is forwarded to +# the provider. assume_supported: is this library's own decision, not a vendor's, so it's +# reserved rather than forwarded -- it must never reach a payload. TranslationDiff.translate(contract, from: "en", to: "de", formality: :more) ``` @@ -138,7 +140,7 @@ This gem loads `ox`, `pragmatic_segmenter`, `faraday`, and `faraday-retry` at re ## Documentation -[Configuration](docs/configuration.md) · [Providers](docs/providers.md) · [Caching](docs/caching.md) · [Contracts](docs/contracts.md) · [Instrumentation](docs/instrumentation.md) · [Errors](docs/errors.md) · [How it works](docs/how-it-works.md) · [Upgrading & development](docs/development.md) +[Configuration](docs/configuration.md) · [Providers](docs/providers.md) · [Languages](docs/languages.md) · [Caching](docs/caching.md) · [Contracts](docs/contracts.md) · [Instrumentation](docs/instrumentation.md) · [Errors](docs/errors.md) · [How it works](docs/how-it-works.md) · [Upgrading & development](docs/development.md) ## Contributing diff --git a/docs/configuration.md b/docs/configuration.md index 1bcea61..e341b8f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -70,6 +70,7 @@ at all, so an unset environment variable never has to be special-cased. | `open_timeout` | `5` | Seconds an HTTP-backed provider waits to open a connection before raising `TranslationDiff::TransportError`. | | `timeout` | `30` | Seconds an HTTP-backed provider waits for a response before raising `TranslationDiff::TransportError`. | | `max_retries` | `3` | Retries `faraday-retry` attempts on a transport failure or a `429`/`500`/`502`/`503`/`504` response, with exponential backoff. `faraday-retry` honours a `Retry-After` header itself, so a `429` usually exhausts its retries before `TranslationDiff::RateLimitError` is ever raised. | +| `validate_languages` | `true` | Whether `translate` refuses a source/target pair the shipped data doesn't list, before making a request. See [Languages](languages.md). | ## Provider options diff --git a/docs/contracts.md b/docs/contracts.md index 353a0cd..24c88d6 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -7,9 +7,9 @@ symbol through a registry -- there is only one built-in implementation. `config.rate_limiter_instance` is: - the object assigned to `config.rate_limiter`, if any; -- otherwise `nil` if `rate_limit` was never set -- and `Request` checks for - that `nil` and skips rate limiting entirely, so the common case costs - nothing; +- otherwise `nil` if `rate_limit` was never set -- and `Dispatcher#throttle` + checks for that `nil` and skips rate limiting entirely, so the common case + costs nothing; - otherwise a `TranslationDiff::RedisRateLimiter` built from `rate_limit`, `rate_interval`, `redis_url` and `cache_namespace`. diff --git a/docs/errors.md b/docs/errors.md index 34b0756..37fc254 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -23,6 +23,9 @@ TranslationDiff::Error │ # that returned no translation for an input ├── TranslationDiff::InvalidProviderError # a class registered without inheriting │ # TranslationDiff::Provider +├── TranslationDiff::UnsupportedLanguageError # the shipped language data doesn't list +│ # this source/target pair for this +│ # provider -- see docs/languages.md ├── TranslationDiff::Translator::Error # from: missing and the provider cannot │ # detect, cache_key missing on an │ # assigned provider object diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 2e9adb3..6be46c7 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -4,8 +4,12 @@ A call to `TranslationDiff.translate` walks a value, cuts the prose in it into sentences, translates only the sentences no cache already holds, and puts the value back together in the shape it arrived in. -`TranslationDiff::Translator` is the one class that coordinates all of it. -Everything below is a collaborator it drives. +`TranslationDiff::Translator` coordinates document assembly, provider +resolution, settling the source language, language validation and the +sentence cache. `TranslationDiff::Dispatcher` takes over once there are cache +misses to send: it packs them into batches, throttles each one, makes the +wire call, and fires the `request`, `rate_limit` and `usage` events. +Everything below is a collaborator one of the two drives. ## The steps @@ -36,13 +40,14 @@ Everything below is a collaborator it drives. them all in a single `read_multi`. Sentences it answers for are already done; the rest are misses. See [Caching](caching.md). -5. **The misses are packed into requests.** - `TranslationDiff::Batch.pack` groups the missing sentences into batches - that fit inside the provider's declared `max_batch_size` and - `max_request_size` -- its `TranslationDiff::Capabilities` (see - [Providers](providers.md)). Each batch is sent, and the reply is - applied back onto the very segments that produced it -- no step ever - correlates a translation to a sentence by position after the fact. +5. **The misses are handed to `TranslationDiff::Dispatcher`.** + `TranslationDiff::Batch.pack` groups them into batches that fit inside the + provider's declared `max_batch_size` and `max_request_size` -- its + `TranslationDiff::Capabilities` (see [Providers](providers.md)). + `Dispatcher` throttles each batch through `config.rate_limiter_instance` + when one is configured, sends it to the provider, and applies the reply + back onto the very segments that produced it -- no step ever correlates a + translation to a sentence by position after the fact. 6. **What came back is written home, and the value is rebuilt.** Only sentences that actually got a translation are cached. Then each diff --git a/docs/instrumentation.md b/docs/instrumentation.md index f26f285..9e407f9 100644 --- a/docs/instrumentation.md +++ b/docs/instrumentation.md @@ -6,7 +6,7 @@ and `config.logger` accepts a standard `Logger`. Neither is required: with both unset, `TranslationDiff.translate` runs exactly the same, at no extra cost. -A translation emits up to four events, each named `.translation_diff`: +A translation emits up to five events, each named `.translation_diff`: | Event | Fired | Payload | | --- | --- | --- | @@ -14,6 +14,19 @@ A translation emits up to four events, each named `.translation_diff`: | `cache` | Once per `translate` call that reaches the provider, after checking the cache for every sentence at once. | `provider`, `hits`, `misses` | | `request` | Once per batch actually sent to the provider (skipped entirely on a full cache hit). | `provider`, `batch` (values sent), `characters` | | `rate_limit` | Once per batch sent to the provider, only when a rate limiter is configured. | `provider`, `characters` | +| `usage` | Once per batch actually sent to the provider, right after `request`. | `provider`, `characters`, `billed_characters`, `reported`, `model` | + +`usage`'s `characters` is what this library sent, counted locally -- the same +number `request` carries. `billed_characters` is what the provider said it +charged for, or `nil` when it said nothing. **`reported` means the provider +reports billing at all -- not that this particular response was billed.** +`billed_characters: nil` alone cannot tell "this provider never says" apart +from "this response omitted it"; `reported` is what makes the `nil` honest. +Summing `billed_characters` across providers without checking `reported` +first produces a total that is quietly too low, since only three of the six +built-in providers (DeepL, Azure, ModernMT) report billing at all -- the +other three always answer `nil`. `model` is the model the provider used, +when it names one, and `nil` otherwise. **`cache` fires once per call as of 3.1.0, not once per chunk.** The cache is now consulted for every sentence in one `read_multi` before anything is diff --git a/docs/languages.md b/docs/languages.md new file mode 100644 index 0000000..3c60003 --- /dev/null +++ b/docs/languages.md @@ -0,0 +1,94 @@ +# Languages + +`TranslationDiff::Languages` answers, without making a request, whether a +provider translates a given source into a given target. `Translator` calls it +before every translation and refuses a pair it says no to. + +```ruby +TranslationDiff::Languages.supports?(:deepl, from: "en", to: "ru") # => true +TranslationDiff::Languages.supports?(:amazon, from: "en", to: "ru") # => nil +``` + +Three answers, not two: `true`, `false`, and `nil` for "no opinion". `nil` is +not a refusal -- a provider this registry knows nothing about is never +blocked from translating anything. + +## What ships + +One JSON file per provider under `data/languages/`, each naming the date it +was captured and the endpoint it came from: + +```json +{ + "provider": "deepl", + "captured_at": "2026-09-10", + "endpoint": "https://api.deepl.com/v2/languages", + "source": ["ar", "bg", "cs", "..."], + "target": ["ar", "bg", "cs", "en-gb", "en-us", "..."] +} +``` + +`captured_at` is not decoration: it is the difference between "these are the +languages" and "these were the languages on 10 September 2026", and only the +second is true. + +Data ships for **DeepL, Google, Azure and ModernMT** only. **Amazon** ships +none -- its language list needs signed credentials, so nothing can be fetched +without them. **LibreTranslate** ships none either, for a different reason: +it is self-hosted, so the language set belongs to whichever instance you +point this gem at, not to a vendor this gem can capture once and ship. + +A provider with no shipped data -- Amazon, LibreTranslate, or any provider of +your own -- refuses nothing. `Languages.supports?` returns `nil` for it, +every time, and `Translator` treats `nil` the same as `true`. + +## Matching a code + +A code is matched downcased, on its primary subtag, so `en-GB` and `en` +match each other in both directions. + +This is deliberately permissive. The registry exists to catch a wrong +language -- `to: "klingon"`, a swapped pair, a typo -- not to adjudicate a +wrong regional variant. `Languages.supports?(:azure, from: "en", to: "zh")` +answers `true`, even though Azure itself wants `zh-Hans` or `zh-Hant` as a +target and would reject a bare `zh`. The rule can over-allow; it must never +wrongly refuse a pair the provider would actually have accepted. + +One more rule, for a mismatch the vendors themselves create. ModernMT +publishes ISO 639-3 individual codes where applications write the ISO 639-1 +macrolanguage: it lists `pes`, not `fa`, and `uzn`, not `uz`. A vendor that +translates the individual language translates the macrolanguage it belongs +to, so `to: "fa"` is accepted against a list carrying `pes`. + +That holds in one direction only. A vendor publishing `zh` has not promised +to accept `cmn`, so `Languages.supports?(:deepl, from: "en", to: "cmn")` +answers `false` -- DeepL would reject the code it was sent. + +## The two escapes + +Shipped data goes stale, and a vendor adding a language must not make this +gem refuse work that would now succeed. Two escapes exist for that: + +```ruby +# Per call +TranslationDiff.translate(text, from: "en", to: "yue", assume_supported: true) + +# Globally +TranslationDiff.configure { |config| config.validate_languages = false } +``` + +`assume_supported:` is a reserved keyword, like `provider:` and `config:` -- +it is read by `Translator` and never forwarded to the provider itself. + +## `rake languages:refresh` + +A maintainer's tool, not something an application runs: it re-fetches every +provider's language lists from its vendor and rewrites the shipped files, +which is why it needs each vendor's own credentials configured to do +anything. + +A provider whose fetch fails keeps its previous file -- an emptied list from +a timed-out request would be worse than a stale one -- and the task reports +which providers failed and why, rather than failing silently. Its output is +meant to be reviewed as a diff, the way any change to shipped data should be, +before it is committed. diff --git a/docs/providers.md b/docs/providers.md index 24bf308..abbc139 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -7,6 +7,31 @@ translation service can be plugged in by subclassing a small base class. See the provider table in the [README](../README.md#providers). +Language pairs are checked against shipped vendor data before every call, and +a provider with no shipped data refuses nothing. Data ships for **DeepL, +Google, Azure and ModernMT**; **Amazon** and **LibreTranslate** ship none -- +see [Languages](languages.md). + +`Configuration#inspect` and `Provider#inspect` print `[FILTERED]` in place of +every credential option below -- `deepl_api_key`, `azure_api_key`, +`amazon_secret_access_key`, and so on -- so a Rails error page and a stray +`p config` in a console cannot leak one; `pp` and `p` both go through the +overridden `inspect`. That guarantee stops at `inspect`, though: an error +reporter that serialises object state instead of calling `inspect` is not +covered, and neither is `p provider.connection`, which prints Faraday's own +headers, `Authorization` included, untouched. + +Options that carry no credential at all, like `azure_region` or +`cache_namespace`, stay visible in full. Any option whose value parses as a +URI carrying userinfo -- `redis_url` among them -- has just that userinfo +redacted: `rediss://default:AbCdEf-TOKEN@cache.example.upstash.io:6379` +prints as `rediss://default:[FILTERED]@cache.example.upstash.io:6379`. The +scheme, host, port and path stay visible, since that's what you need to +debug against -- only the credential Heroku, Upstash, Redis Cloud and Aiven +all put in the userinfo is hidden. A value that isn't a URI, or a URI with no +userinfo, is left unchanged, and a malformed value never raises out of +`inspect`. + ## Configuring each provider Every example below is complete: set the options shown and @@ -125,16 +150,20 @@ never serves you the other's translations. ## Capabilities, in full -**Every keyword other than `from:`, `to:`, `provider:` and `config:` is -forwarded to the provider, and every provider applies them the same way: -its own defaults first, then your options, then the fields the request cannot -do without.** So `formality: :less` overrides a default, and a keyword -colliding with the language pair or the texts themselves is overridden rather -than obeyed. +**Every keyword other than `from:`, `to:`, `provider:`, `config:` and +`assume_supported:` is forwarded to the provider, and every provider applies +them the same way: its own defaults first, then your options, then the +fields the request cannot do without.** So `formality: :less` overrides a +default, and a keyword colliding with the language pair or the texts +themselves is overridden rather than obeyed. `assume_supported:` is reserved +because it is this library's own decision -- whether to skip language +validation for this call -- not a vendor's, and it must never reach a +payload. **`usage.billed_characters` is `nil` when the provider said nothing about -billing and a number -- `0` included -- when it said something.** All three -providers that report billing follow that rule; the other four always answer +billing and a number -- `0` included -- when it said something.** Three of +the six built-in providers report billing that way -- DeepL, Azure and +ModernMT; the other three, Google, LibreTranslate and Amazon, always answer `nil`. **Language codes are normalised per vendor, so switching provider needs no @@ -269,6 +298,19 @@ entirely optional -- omit it (and leave `capabilities.detects_language: false`) if the provider has no detection endpoint, or if callers of this gem always pass `from:` explicitly. +**`self.sensitive_options` decides which of this provider's +`configuration_options` `Configuration#inspect` and `Provider#inspect` +filter.** By default it is every declared option whose name matches +`TranslationDiff::Redaction::SENSITIVE` (`key`, `secret`, `token`, +`password`, `auth`, `credential`) -- `acme_api_key` above is caught by that +pattern for free. Override it when a credential's name doesn't match: a +provider reading `config.acme_handshake` for its credential leaks it on +every `inspect` unless it says so itself: + +```ruby +def self.sensitive_options = %i[acme_handshake] +``` + **Provider names must be unique.** `TranslationDiff::Providers.register` overwrites whatever was previously registered under that name, silently -- there is no error for registering `:deepl` twice. This is deliberate: a