From 0af12af90c1797a6c0962164c93153ed9f5b595e Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 00:13:28 +0400 Subject: [PATCH 01/21] test: pin the pipeline's behaviour before replacing it --- test/support/pipeline_corpus.rb | 51 +++++++++++++++++++ test/translation_diff/pipeline_corpus_test.rb | 36 +++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 test/support/pipeline_corpus.rb create mode 100644 test/translation_diff/pipeline_corpus_test.rb diff --git a/test/support/pipeline_corpus.rb b/test/support/pipeline_corpus.rb new file mode 100644 index 0000000..1497749 --- /dev/null +++ b/test/support/pipeline_corpus.rb @@ -0,0 +1,51 @@ +# The inputs the pipeline rewrite is judged against. Every one of these is a +# case some earlier bug or review turned up; none is invented. +module PipelineCorpus + INPUTS = { + "plain sentence" => "Hello there.", + "two sentences" => "Hello there. Second sentence!", + "nested hash" => { title: "One. Two.", body: "Third." }, + "nested array" => ["A. B.", ["C."], "D."], + "hash with non-strings" => { title: "One.", count: 42, missing: nil, flag: true }, + "empty string" => "", + "nil" => nil, + "not a string" => 42, + "bold markup" => "Bold text here. Second sentence.", + "attributes preserved" => %(Link text. After.), + "void element" => "One line.
Two lines.", + "unclosed paragraph" => "

First para.

Second para.", + "uppercase tags" => "Bold text.", + "script and style" => "альбракил", + "processing instruction" => %(Hey!
Look!), + "comment" => " Visible text here.", + "doctype" => "

Body text.

", + "cdata" => "Before.After.", + "notranslate span" => %(Bold Mountain is a good place.), + "nested notranslate" => "foobarbaz", + "notranslate inside span" => "foobar
baz", + "br before closing tag" => "Смеркалось.
", + "blank line between sentences" => "Первое предложение.\n\nВторое предложение.", + "single newline" => "test\nphrase", + "leading and trailing space" => " Padded sentence. ", + "many sentences" => (1..40).map { |i| "Sentence number #{i}." }.join(" "), + "non-ascii" => "Привет. Как дела? Всё хорошо.", + "entity ampersand" => "Salt & pepper. Fine.", + "entity nbsp" => "Hard space here. Fine.", + "bare less-than" => "if a < b then stop. Fine.", + "bare less-than and greater" => "5 < 6 and 7 > 6. True." + }.freeze + + # Written BEFORE the rewrite, on purpose. A list assembled after seeing the + # new output would be a report of what happened, not a prediction that can + # fail. Everything not named here must come out byte-identical. + # + # Both entries change because the rewrite fixes them: today the entity + # reaches the provider raw, and everything after a bare "<" is treated as + # markup and never translated at all. + EXPECTED_TO_CHANGE = [ + "entity ampersand", + "entity nbsp", + "bare less-than", + "bare less-than and greater" + ].freeze +end diff --git a/test/translation_diff/pipeline_corpus_test.rb b/test/translation_diff/pipeline_corpus_test.rb new file mode 100644 index 0000000..6cad0df --- /dev/null +++ b/test/translation_diff/pipeline_corpus_test.rb @@ -0,0 +1,36 @@ +require "test_helper" +require "support/pipeline_corpus" + +# Judges the pipeline rewrite against the baseline captured in ~/JetRockets/.deepl_diff-specs/pipeline-baseline.txt +# before any of the pipeline changed, translating every input the same way the baseline script did: through the +# :null provider, from "en" to "ru". +class PipelineCorpusTest < ConfiguredTest + BASELINE_PATH = File.expand_path("~/JetRockets/.deepl_diff-specs/pipeline-baseline.txt") + + def self.baseline_outputs + @baseline_outputs ||= File.read(BASELINE_PATH).scan(/^=== (.+) ===\nOUTPUT: (.*)\n/).to_h + end + + def translated(name) + TranslationDiff.translate(PipelineCorpus::INPUTS.fetch(name), from: "en", to: "ru", provider: :null).inspect + end + + def self.method_name_for(name) = :"test_#{name.gsub(/[^a-zA-Z0-9]+/, '_')}" + + (PipelineCorpus::INPUTS.keys - PipelineCorpus::EXPECTED_TO_CHANGE).each do |name| + define_method(method_name_for(name)) do + assert_equal self.class.baseline_outputs.fetch(name), translated(name) + end + end + + # These four are named in EXPECTED_TO_CHANGE: the rewrite is scheduled to fix them, not to leave them alone. + # Skipped, not deleted -- a later task removes the skip and then this asserts the fixed output differs from + # today's recorded baseline. + PipelineCorpus::EXPECTED_TO_CHANGE.each do |name| + define_method(method_name_for(name)) do + skip "scheduled to change: a later task removes this skip once the rewrite fixes this input " \ + "(see PipelineCorpus::EXPECTED_TO_CHANGE)" + refute_equal self.class.baseline_outputs.fetch(name), translated(name) + end + end +end From 27bce12ae031a69b02e2be6bf49a28884d90f507 Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 00:17:17 +0400 Subject: [PATCH 02/21] test: trim EXPECTED_TO_CHANGE comment to one line --- test/support/pipeline_corpus.rb | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/test/support/pipeline_corpus.rb b/test/support/pipeline_corpus.rb index 1497749..47488d1 100644 --- a/test/support/pipeline_corpus.rb +++ b/test/support/pipeline_corpus.rb @@ -35,13 +35,7 @@ module PipelineCorpus "bare less-than and greater" => "5 < 6 and 7 > 6. True." }.freeze - # Written BEFORE the rewrite, on purpose. A list assembled after seeing the - # new output would be a report of what happened, not a prediction that can - # fail. Everything not named here must come out byte-identical. - # - # Both entries change because the rewrite fixes them: today the entity - # reaches the provider raw, and everything after a bare "<" is treated as - # markup and never translated at all. + # Written before the rewrite, on purpose, so it can fail -- unlike a list assembled after seeing what changed. EXPECTED_TO_CHANGE = [ "entity ampersand", "entity nbsp", From 5c4411a8358c7564cfbdc0740612e199b4a2e2ed Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 00:19:01 +0400 Subject: [PATCH 03/21] feat: walk a caller's structure without flattening it --- lib/translation_diff.rb | 1 + lib/translation_diff/document.rb | 23 ++++++++ test/translation_diff/document_test.rb | 76 ++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) create mode 100644 lib/translation_diff/document.rb create mode 100644 test/translation_diff/document_test.rb diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index 2011962..fef9c2a 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -13,6 +13,7 @@ require "translation_diff/translation/request" require "translation_diff/translation/response" require "translation_diff/registry" +require "translation_diff/document" require "translation_diff/configuration" require "translation_diff/configuration/provider_option_owners" diff --git a/lib/translation_diff/document.rb b/lib/translation_diff/document.rb new file mode 100644 index 0000000..18a1f16 --- /dev/null +++ b/lib/translation_diff/document.rb @@ -0,0 +1,23 @@ +# A caller's value together with its Hash/Array shape, walked without ever flattening it into an array. +class TranslationDiff::Document + def initialize(value) = @value = value + + # Returns a new structure of the same shape with every leaf String replaced by the block's result. + def map(&) = walk(@value, &) + + # Returns the leaf strings in document order, without touching the original value. + def strings + [].tap { |acc| walk(@value) { |string| acc << string } } + end + + private + + def walk(node, &block) + case node + when Hash then node.to_h { |key, value| [key, walk(value, &block)] } + when Array then node.map { |value| walk(value, &block) } + when String then block.call(node) + else node + end + end +end diff --git a/test/translation_diff/document_test.rb b/test/translation_diff/document_test.rb new file mode 100644 index 0000000..e66f925 --- /dev/null +++ b/test/translation_diff/document_test.rb @@ -0,0 +1,76 @@ +require "test_helper" + +class DocumentTest < Minitest::Test + def map(value, &) = TranslationDiff::Document.new(value).map(&) + + def test_a_bare_string_is_mapped + assert_equal "HELLO", map("hello", &:upcase) + end + + def test_a_hash_keeps_its_keys_and_their_order + result = map({ title: "one", body: "two" }, &:upcase) + + assert_equal({ title: "ONE", body: "TWO" }, result) + assert_equal %i[title body], result.keys + end + + def test_an_array_keeps_its_order + assert_equal %w[A B C], map(%w[a b c], &:upcase) + end + + def test_nesting_of_both_kinds_survives + value = { a: ["one", { b: "two" }], c: "three" } + + assert_equal({ a: ["ONE", { b: "TWO" }], c: "THREE" }, map(value, &:upcase)) + end + + # Anything that is not a String is not translatable and must arrive on the + # other side as the same object, in the same place. + def test_non_strings_pass_through_untouched + value = { text: "one", count: 42, missing: nil, flag: true, at: :symbol } + + assert_equal({ text: "ONE", count: 42, missing: nil, flag: true, at: :symbol }, + map(value, &:upcase)) + end + + def test_the_block_is_not_called_for_non_strings + seen = [] + map({ text: "one", count: 42, missing: nil }) do |s| + seen << s + s + end + + assert_equal ["one"], seen + end + + def test_an_empty_string_is_still_a_string + assert_equal [""], TranslationDiff::Document.new([""]).strings + end + + def test_strings_are_returned_in_document_order + value = { a: ["one", { b: "two" }], c: "three" } + + assert_equal %w[one two three], TranslationDiff::Document.new(value).strings + end + + def test_strings_does_not_modify_the_value + value = { a: ["one"] } + TranslationDiff::Document.new(value).strings + + assert_equal({ a: ["one"] }, value) + end + + # The caller's structure is theirs. Mapping returns a new one. + def test_map_does_not_mutate_the_original + value = { a: ["one"] } + map(value, &:upcase) + + assert_equal({ a: ["one"] }, value) + end + + def test_deep_nesting_does_not_lose_its_shape + value = [[[["deep"]]]] + + assert_equal [[[["DEEP"]]]], map(value, &:upcase) + end +end From 1096a9609bddedb50eb144e943962ec98f25baa8 Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 00:22:03 +0400 Subject: [PATCH 04/21] feat: keep a sentence and its whitespace in one object --- lib/translation_diff.rb | 1 + lib/translation_diff/segment.rb | 16 ++++++++ test/translation_diff/segment_test.rb | 55 +++++++++++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 lib/translation_diff/segment.rb create mode 100644 test/translation_diff/segment_test.rb diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index fef9c2a..9b74fb5 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -14,6 +14,7 @@ require "translation_diff/translation/response" require "translation_diff/registry" require "translation_diff/document" +require "translation_diff/segment" require "translation_diff/configuration" require "translation_diff/configuration/provider_option_owners" diff --git a/lib/translation_diff/segment.rb b/lib/translation_diff/segment.rb new file mode 100644 index 0000000..7a833ee --- /dev/null +++ b/lib/translation_diff/segment.rb @@ -0,0 +1,16 @@ +# A sentence with the whitespace it was found in, so rendering never needs a second string to restore it. +class TranslationDiff::Segment + attr_reader :source, :core + attr_accessor :translation + + def initialize(source) + @source = source + @leading, @core, @trailing = source.partition(/\S.*\S|\S/m) + end + + def translated? = !translation.nil? + + def empty? = core.empty? + + def render = "#{@leading}#{translated? ? translation : core}#{@trailing}" +end diff --git a/test/translation_diff/segment_test.rb b/test/translation_diff/segment_test.rb new file mode 100644 index 0000000..a27b807 --- /dev/null +++ b/test/translation_diff/segment_test.rb @@ -0,0 +1,55 @@ +require "test_helper" + +class SegmentTest < Minitest::Test + def test_core_is_the_sentence_without_its_padding + assert_equal "It was getting dark.", TranslationDiff::Segment.new(" It was getting dark. ").core + end + + def test_source_is_kept_exactly + assert_equal " It was getting dark. ", TranslationDiff::Segment.new(" It was getting dark. ").source + end + + def test_render_puts_the_translation_back_inside_the_original_padding + segment = TranslationDiff::Segment.new(" It was getting dark. ") + segment.translation = "Смеркалось." + + assert_equal " Смеркалось. ", segment.render + end + + # The padding is whatever was there, not a normalised guess at it. + def test_padding_is_reproduced_character_for_character + segment = TranslationDiff::Segment.new("\n\t One. \n") + segment.translation = "Один." + + assert_equal "\n\t Один. \n", segment.render + end + + def test_an_untranslated_segment_renders_its_source + assert_equal " One. ", TranslationDiff::Segment.new(" One. ").render + end + + def test_translated_reports_whether_a_translation_was_set + segment = TranslationDiff::Segment.new("One.") + + refute_predicate segment, :translated? + segment.translation = "Один." + assert_predicate segment, :translated? + end + + # A run of whitespace between two sentences is a segment with nothing to + # translate. It must render unchanged and never reach a provider. + def test_a_segment_of_only_whitespace_is_empty + segment = TranslationDiff::Segment.new(" \n ") + + assert_predicate segment, :empty? + assert_equal " \n ", segment.render + end + + def test_a_segment_with_words_is_not_empty + refute_predicate TranslationDiff::Segment.new(" One. "), :empty? + end + + def test_an_empty_source_is_empty + assert_predicate TranslationDiff::Segment.new(""), :empty? + end +end From 3e8f479345e617e18e24758144ef85d9e4ca857c Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 00:26:34 +0400 Subject: [PATCH 05/21] fix: copy source and use unicode-aware whitespace in Segment --- lib/translation_diff/segment.rb | 5 +++-- test/translation_diff/segment_test.rb | 27 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/lib/translation_diff/segment.rb b/lib/translation_diff/segment.rb index 7a833ee..ad82c59 100644 --- a/lib/translation_diff/segment.rb +++ b/lib/translation_diff/segment.rb @@ -4,10 +4,11 @@ class TranslationDiff::Segment attr_accessor :translation def initialize(source) - @source = source - @leading, @core, @trailing = source.partition(/\S.*\S|\S/m) + @source = source.dup + @leading, @core, @trailing = @source.partition(/[^[:space:]].*[^[:space:]]|[^[:space:]]/m) end + # Reflects whether a translation is set right now, not history -- clearing it to nil flips this back to false. def translated? = !translation.nil? def empty? = core.empty? diff --git a/test/translation_diff/segment_test.rb b/test/translation_diff/segment_test.rb index a27b807..94182a6 100644 --- a/test/translation_diff/segment_test.rb +++ b/test/translation_diff/segment_test.rb @@ -52,4 +52,31 @@ def test_a_segment_with_words_is_not_empty def test_an_empty_source_is_empty assert_predicate TranslationDiff::Segment.new(""), :empty? end + + # String literals are mutable in this project -- the magic comment was + # removed everywhere -- so a segment must not alias the string it was given. + def test_mutating_the_source_afterwards_does_not_change_the_segment + source = +" One. " + segment = TranslationDiff::Segment.new(source) + source << "trailing junk" + + assert_equal " One. ", segment.source + assert_equal "One.", segment.core + assert_equal " One. ", segment.render + end + + def test_a_segment_of_only_a_non_breaking_space_is_empty + segment = TranslationDiff::Segment.new(" ") + + assert_predicate segment, :empty? + assert_equal " ", segment.render + end + + def test_a_non_breaking_space_around_a_sentence_is_padding + segment = TranslationDiff::Segment.new(" One. ") + segment.translation = "Один." + + assert_equal "One.", segment.core + assert_equal " Один. ", segment.render + end end From a2c9885d5db052848509d8fb114de9e11c006dfb Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 00:32:38 +0400 Subject: [PATCH 06/21] feat: separate markup from prose without rebuilding either --- lib/translation_diff.rb | 2 + lib/translation_diff/fragment.rb | 32 +++++++ lib/translation_diff/passage.rb | 131 +++++++++++++++++++++++++ test/translation_diff/passage_test.rb | 132 ++++++++++++++++++++++++++ 4 files changed, 297 insertions(+) create mode 100644 lib/translation_diff/fragment.rb create mode 100644 lib/translation_diff/passage.rb create mode 100644 test/translation_diff/passage_test.rb diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index 9b74fb5..cc68a67 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -15,6 +15,8 @@ require "translation_diff/registry" require "translation_diff/document" require "translation_diff/segment" +require "translation_diff/fragment" +require "translation_diff/passage" require "translation_diff/configuration" require "translation_diff/configuration/provider_option_owners" diff --git a/lib/translation_diff/fragment.rb b/lib/translation_diff/fragment.rb new file mode 100644 index 0000000..bb134b7 --- /dev/null +++ b/lib/translation_diff/fragment.rb @@ -0,0 +1,32 @@ +# A run of the source that is either markup, handed back as found, or prose, handed back through its segments. +class TranslationDiff::Fragment + EMPTY = [].freeze + + attr_reader :source + + # Markup has nothing a provider should see, so it carries no segments and renders the bytes it was cut from. + def self.markup(source) = new(source, nil) + + # Prose is cut where the segmenter says sentences begin, so every segment keeps the whitespace it was found in. + def self.prose(source, segmenter:, language: nil) + new(source, cut(source, segmenter.split_offsets(source, language: language))) + end + + # The offsets start at 0 and strictly increase, so slicing between them and from the last to the end is exact. + def self.cut(source, offsets) + sentences = offsets.each_cons(2).map { |from, to| source[from...to] } << source[offsets.last..] + sentences.map { |sentence| TranslationDiff::Segment.new(sentence) } + end + private_class_method :cut + + def initialize(source, segments) + @source = source + @segments = segments + end + + def markup? = @segments.nil? + + def segments = @segments || EMPTY + + def render = markup? ? source : @segments.map(&:render).join +end diff --git a/lib/translation_diff/passage.rb b/lib/translation_diff/passage.rb new file mode 100644 index 0000000..fedcb38 --- /dev/null +++ b/lib/translation_diff/passage.rb @@ -0,0 +1,131 @@ +# A source document as markup and prose: the markup kept as found, the prose cut into segments a provider can take. +class TranslationDiff::Passage + attr_reader :fragments + + def initialize(source, segmenter:, language: nil) + @source = source + @segmenter = segmenter + @language = language + @fragments = Scanner.new(source).runs.map { |run| fragment(run) } + end + + # The translatable sentences, in document order; the empty ones are whitespace a provider has no use for. + def segments = fragments.flat_map(&:segments) + + def render = fragments.map(&:render).join + + private + + # Every fragment is a slice of the source, never a rebuilt string; that is what makes an untranslated render exact. + def fragment(run) + from, to, prose = run + slice = @source.byteslice(from, to - from) + return TranslationDiff::Fragment.markup(slice) unless prose + + TranslationDiff::Fragment.prose(slice, segmenter: @segmenter, language: @language) + end + + # Ox reports a byte position for every construct it sees; recording those is what lets rendering slice the source. + class Scanner < Ox::Sax + # Content nobody wants translated, however much of it looks like prose. + OPAQUE = %i[script style].freeze + + # Providers honour this class themselves under the HTML mode this gem sends, so the element must reach them whole. + PROTECTED = "notranslate".freeze + + OPENING_ANGLE = "<".ord + + # Where a run begins and whether it is prose; prose is set after the fact when an element claims protection. + Mark = Struct.new(:offset, :prose) + + # Ox reports positions only to a handler that already has the ivar, so @pos exists before parsing starts. + def initialize(source) + super() + @source = source + @pos = 0 + @marks = [] + @protected_depth = 0 + @opaque_depth = 0 + @pending = nil + end + + # Triples of [first byte, last byte + 1, prose?], contiguous, covering the source exactly once. + def runs + Ox.sax_html(self, StringIO.new(@source)) + merge(bounds) + end + + def start_element(name) + return @protected_depth += 1 if @protected_depth.positive? + + @opaque_depth += 1 if @opaque_depth.positive? || OPAQUE.include?(name) + @pending = mark(prose: false) + end + + # Attributes arrive straight after their own start element, so @pending is that element and never another. + def attr(name, value) + return unless @pending && name == :class && value.split.include?(PROTECTED) + + @pending.prose = true + @protected_depth = 1 + @pending = nil + end + + def end_element(_name) + return @protected_depth -= 1 if @protected_depth.positive? + + @opaque_depth -= 1 if @opaque_depth.positive? + record(prose: false) + end + + def value(_value) = record(prose: @opaque_depth.zero?) + + def comment(_content) = record(prose: false) + + def cdata(_content) = record(prose: false) + + def doctype(_content) = record(prose: false) + + def instruct(_target) = record(prose: false) + + private + + # Everything inside a protected element belongs to the run that element opened, so it records nothing of its own. + def record(prose:) + @pending = nil + return if @protected_depth.positive? + + mark(prose: prose) + end + + # Ox counts from one. A markup mark that does not land on a "<" is a tag Ox implied, not one the source holds. + def mark(prose:) + offset = @pos - 1 + return nil if offset.negative? || (!prose && @source.getbyte(offset) != OPENING_ANGLE) + + Mark.new(offset, prose).tap { |recorded| @marks << recorded } + end + + # Each mark owns the source as far as the next one begins, and the last one owns whatever is left. + def bounds + marks = ordered + finishes = marks.drop(1).map(&:offset) << @source.bytesize + marks.zip(finishes).map { |mark, finish| [mark.offset, finish, mark.prose] } + end + + # Ox skips the whitespace ahead of the first construct it reports, so a prose run is seeded where it starts. + def ordered + sorted = @marks.sort_by.with_index { |mark, index| [mark.offset, index] } + return sorted if sorted.first&.offset&.zero? + + sorted.unshift(Mark.new(0, true)) + end + + # Adjacent prose has to become one run, or a notranslate element would be cut off from the sentence it sits in. + def merge(spans) + spans.reject { |from, to, _prose| from == to } + .chunk_while { |before, after| before.last == after.last } + .map { |chunk| [chunk.first[0], chunk.last[1], chunk.first[2]] } + end + end +end diff --git a/test/translation_diff/passage_test.rb b/test/translation_diff/passage_test.rb new file mode 100644 index 0000000..7c7f4fd --- /dev/null +++ b/test/translation_diff/passage_test.rb @@ -0,0 +1,132 @@ +require "test_helper" + +class PassageTest < Minitest::Test + def passage(source) + TranslationDiff::Passage.new(source, segmenter: TranslationDiff::Segmenters::Pragmatic.new) + end + + # What a provider would be asked to translate, in order. + def cores(source) = passage(source).segments.reject(&:empty?).map(&:core) + + def assert_round_trips(source) + assert_equal source, passage(source).render, "render must return the source byte for byte" + end + + # -- byte-exact reconstruction ------------------------------------------ + + # rubocop:disable-next Metrics/MethodLength + def test_every_corpus_input_round_trips_untranslated + [ + "Hello there.", + "Bold text here. Second sentence.", + %(Link text. After.), + "One line.
Two lines.", + "

First para.

Second para.", + "Bold text.", + "альбракил", + %(Hey!
Look!), + " Visible text here.", + "

Body text.

", + "Before.After.", + "Смеркалось.
", + " Padded sentence. ", + "Первое предложение.\n\nВторое предложение.", + "" + ].each { |source| assert_round_trips(source) } + end + + # -- what counts as prose ---------------------------------------------- + + def test_text_around_markup_is_prose + assert_equal ["Bold", "text here.", "Second sentence."], + cores("Bold text here. Second sentence.") + end + + def test_script_and_style_contents_are_not_prose + assert_equal %w[аль бра кил], cores("альбракил") + end + + def test_a_comment_is_not_prose + assert_equal ["Visible text here."], cores(" Visible text here.") + end + + def test_a_doctype_is_not_prose + assert_equal ["Body text."], cores("

Body text.

") + end + + def test_a_processing_instruction_is_not_prose + assert_equal %w[Hey! Look!], cores(%(Hey!
Look!)) + end + + def test_attribute_values_are_not_prose + assert_equal ["Link text.", "After."], cores(%(Link text. After.)) + end + + # -- notranslate, which is prose ON PURPOSE ------------------------------ + + # The provider honours class="notranslate" itself, under the HTML mode all + # six providers send. Holding the span back as markup would deprive it of + # the protection it exists to request. + def test_a_notranslate_span_reaches_the_provider_with_its_tags + source = %(Bold Mountain is a good place.) + + assert_equal [%(Bold Mountain is a good place.)], cores(source) + end + + def test_a_notranslate_span_nested_in_another_is_one_unit + source = "foobarbaz" + + assert_equal [source], cores(source) + end + + def test_a_notranslate_span_inside_an_ordinary_span_keeps_the_outer_span_as_markup + source = "foobar
baz" + + assert_equal ["foobar
baz"], cores(source) + end + + # -- sentence boundaries ------------------------------------------------- + + def test_prose_is_cut_into_sentences + assert_equal ["! Киловольт.", "Смеркалось.", "Ворчало.", "Кричало."], + cores("! Киловольт. Смеркалось. Ворчало. Кричало.") + end + + def test_a_blank_line_separates_sentences + assert_equal ["Первое предложение.", "Второе предложение."], + cores("Первое предложение.\n\nВторое предложение.") + end + + # A lone newline is not a sentence boundary -- this was a real regression. + def test_a_single_newline_does_not_split_a_sentence + assert_equal ["test\nphrase"], cores("test\nphrase") + end + + def test_markup_holding_nothing_but_a_line_break_offers_nothing_to_translate + assert_empty cores("
\n
") + assert_round_trips("
\n
") + end + + # -- translation and rendering ------------------------------------------- + + def test_translating_every_segment_rebuilds_the_document + subject = passage("Bold text here. Second sentence.") + subject.segments.reject(&:empty?).each { |s| s.translation = s.core.upcase } + + assert_equal "BOLD TEXT HERE. SECOND SENTENCE.", subject.render + end + + def test_padding_between_sentences_survives_translation + subject = passage(" One. Two. ") + subject.segments.reject(&:empty?).each { |s| s.translation = s.core.upcase } + + assert_equal " ONE. TWO. ", subject.render + end + + def test_an_untranslated_segment_renders_its_source + subject = passage("One. Two.") + subject.segments.reject(&:empty?).first.translation = "ОДИН." + + assert_equal "ОДИН. Two.", subject.render + end +end From 565571774ef0d63db8fe91fb52686140c833bd62 Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 00:40:32 +0400 Subject: [PATCH 07/21] fix: keep notranslate protection when the class attribute is uppercase --- lib/translation_diff/fragment.rb | 3 ++- lib/translation_diff/passage.rb | 8 +++++++- test/translation_diff/fragment_test.rb | 12 ++++++++++++ test/translation_diff/passage_test.rb | 20 ++++++++++++++++++++ 4 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 test/translation_diff/fragment_test.rb diff --git a/lib/translation_diff/fragment.rb b/lib/translation_diff/fragment.rb index bb134b7..1dcdd33 100644 --- a/lib/translation_diff/fragment.rb +++ b/lib/translation_diff/fragment.rb @@ -28,5 +28,6 @@ def markup? = @segments.nil? def segments = @segments || EMPTY - def render = markup? ? source : @segments.map(&:render).join + # The slice is copied on the way out, as Segment copies its own: rendering must not hand a caller the passage. + def render = markup? ? source.dup : @segments.map(&:render).join end diff --git a/lib/translation_diff/passage.rb b/lib/translation_diff/passage.rb index fedcb38..9a8d3b8 100644 --- a/lib/translation_diff/passage.rb +++ b/lib/translation_diff/passage.rb @@ -55,6 +55,7 @@ def runs merge(bounds) end + # Protection beats opacity on purpose: a caller wrapping a subtree asked for it to be passed through as it is. def start_element(name) return @protected_depth += 1 if @protected_depth.positive? @@ -64,7 +65,7 @@ def start_element(name) # Attributes arrive straight after their own start element, so @pending is that element and never another. def attr(name, value) - return unless @pending && name == :class && value.split.include?(PROTECTED) + return unless @pending && protection?(name, value) @pending.prose = true @protected_depth = 1 @@ -90,6 +91,11 @@ def instruct(_target) = record(prose: false) private + # Ox lowercases element names but not attribute names; the value stays exact because HTML class tokens are. + def protection?(name, value) + name.to_s.casecmp?("class") && value.split.include?(PROTECTED) + end + # Everything inside a protected element belongs to the run that element opened, so it records nothing of its own. def record(prose:) @pending = nil diff --git a/test/translation_diff/fragment_test.rb b/test/translation_diff/fragment_test.rb new file mode 100644 index 0000000..19bf34e --- /dev/null +++ b/test/translation_diff/fragment_test.rb @@ -0,0 +1,12 @@ +require "test_helper" + +class FragmentTest < Minitest::Test + # Rendering used to hand back the slice itself, so appending to it edited the document it came from. + def test_rendering_markup_does_not_hand_out_the_fragment_source + fragment = TranslationDiff::Fragment.markup("") + + fragment.render << "XXX" + + assert_equal "", fragment.render + end +end diff --git a/test/translation_diff/passage_test.rb b/test/translation_diff/passage_test.rb index 7c7f4fd..808f062 100644 --- a/test/translation_diff/passage_test.rb +++ b/test/translation_diff/passage_test.rb @@ -73,6 +73,19 @@ def test_a_notranslate_span_reaches_the_provider_with_its_tags assert_equal [%(Bold Mountain is a good place.)], cores(source) end + # Ox lowercases element names but not attribute names, and HTML attribute names are case-insensitive. + def test_an_uppercase_class_attribute_still_protects + source = %(Bold Mountain is a good place.) + + assert_equal [source], cores(source) + end + + # Class token values are case-sensitive in HTML and providers look for the lowercase word, so this asks for nothing. + def test_an_uppercase_notranslate_value_does_not_protect + assert_equal ["Bold Mountain", "is a good place."], + cores(%(Bold Mountain is a good place.)) + end + def test_a_notranslate_span_nested_in_another_is_one_unit source = "foobarbaz" @@ -85,6 +98,13 @@ def test_a_notranslate_span_inside_an_ordinary_span_keeps_the_outer_span_as_mark assert_equal ["foobar
baz"], cores(source) end + # Protection beats opacity: the caller asked for this subtree to be passed through, script and all. + def test_a_script_inside_a_notranslate_element_stays_inside_the_protected_unit + source = %() + + assert_equal [source], cores(source) + end + # -- sentence boundaries ------------------------------------------------- def test_prose_is_cut_into_sentences From cf7c08b1b49580de0a10ffed4593c605d3768888 Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 00:55:28 +0400 Subject: [PATCH 08/21] fix: stop entities and bare angle brackets from eating prose --- lib/translation_diff.rb | 1 + lib/translation_diff/markup.rb | 42 ++++++ lib/translation_diff/passage.rb | 20 ++- test/translation_diff/markup_test.rb | 133 ++++++++++++++++++ test/translation_diff/pipeline_corpus_test.rb | 28 +++- 5 files changed, 213 insertions(+), 11 deletions(-) create mode 100644 lib/translation_diff/markup.rb create mode 100644 test/translation_diff/markup_test.rb diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index cc68a67..3801d76 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -14,6 +14,7 @@ require "translation_diff/translation/response" require "translation_diff/registry" require "translation_diff/document" +require "translation_diff/markup" require "translation_diff/segment" require "translation_diff/fragment" require "translation_diff/passage" diff --git a/lib/translation_diff/markup.rb b/lib/translation_diff/markup.rb new file mode 100644 index 0000000..f071303 --- /dev/null +++ b/lib/translation_diff/markup.rb @@ -0,0 +1,42 @@ +# Entity references, and a `<` that opens no tag: the two places a document's text is not the text `ox` reports. +module TranslationDiff::Markup + # Escaping a lone `<` is a workaround around `ox`, not a fix of it: the real fix is our own lexer, and out of scope. + + # A `<` opens a tag only when an element name, a closing name, a declaration or an instruction follows it. + TAG_OPENER = %r{[A-Za-z!?]|/[A-Za-z]} + + # The two characters escaping has to move: a lone `<`, and an `&` that would read as an escape this module wrote. + AMBIGUOUS = /&(?=(?:amp;)*lt;)|<(?!#{TAG_OPENER})/ + + # `<` was a lone `<`; every further `amp;` is a level the source itself wrote and escaping pushed up by one. + ESCAPED_ANGLE = /&((?:amp;)*)lt;/ + + # `&` and ` ` reach Google raw today and it stops translating at them; `<` is this module's own escape. + DECODED = { "&" => "&", "<" => "<", " " => "\u00A0" }.freeze + + DECODABLE = /&(?:amp|lt|nbsp);/ + + # `<` is missing on purpose: restoring a lone angle is the escape's job, and a real tag in prose must stay a tag. + ENCODED = { "&" => "&", "\u00A0" => " " }.freeze + + ENCODABLE = /[&\u00A0]/ + + # Hands back markup `ox` can parse: same document, with every lone `<` written as the entity it should have been. + def self.escape_bare_angles(source) + source.gsub(AMBIGUOUS) { |ambiguous| ambiguous == "&" ? "&" : "<" } + end + + # The exact inverse: one `amp;` off every escaped angle, and the angles with none left were the lone ones. + def self.restore_bare_angles(rendered) + rendered.gsub(ESCAPED_ANGLE) do + levels = Regexp.last_match(1) + levels.empty? ? "<" : "&#{levels.delete_prefix('amp;')}lt;" + end + end + + # What a provider is sent is text, so it gets the characters; a document arriving without entities gains none. + def self.decode_entities(text) = text.gsub(DECODABLE, DECODED) + + # What a document renders is markup, so a decoded character goes back to its entity -- a lone `&` gains one. + def self.encode_entities(text) = text.gsub(ENCODABLE, ENCODED) +end diff --git a/lib/translation_diff/passage.rb b/lib/translation_diff/passage.rb index 9a8d3b8..d957065 100644 --- a/lib/translation_diff/passage.rb +++ b/lib/translation_diff/passage.rb @@ -2,27 +2,39 @@ class TranslationDiff::Passage attr_reader :fragments + # The source is scanned with every lone `<` escaped, so the offsets, the slices and the render all agree on it. def initialize(source, segmenter:, language: nil) - @source = source + @source = TranslationDiff::Markup.escape_bare_angles(source) @segmenter = segmenter @language = language - @fragments = Scanner.new(source).runs.map { |run| fragment(run) } + @fragments = Scanner.new(@source).runs.map { |run| fragment(run) } end # The translatable sentences, in document order; the empty ones are whitespace a provider has no use for. def segments = fragments.flat_map(&:segments) - def render = fragments.map(&:render).join + # Prose is handed back as the markup it came from; markup was never decoded, so only the escaped angles undo. + def render + TranslationDiff::Markup.restore_bare_angles(fragments.map { |fragment| rendered(fragment) }.join) + end private + # Only prose was decoded, so only prose is encoded again; markup still holds the entities the document arrived with. + def rendered(fragment) + return fragment.render if fragment.markup? + + TranslationDiff::Markup.encode_entities(fragment.render) + end + # Every fragment is a slice of the source, never a rebuilt string; that is what makes an untranslated render exact. def fragment(run) from, to, prose = run slice = @source.byteslice(from, to - from) return TranslationDiff::Fragment.markup(slice) unless prose - TranslationDiff::Fragment.prose(slice, segmenter: @segmenter, language: @language) + TranslationDiff::Fragment.prose(TranslationDiff::Markup.decode_entities(slice), + segmenter: @segmenter, language: @language) end # Ox reports a byte position for every construct it sees; recording those is what lets rendering slice the source. diff --git a/test/translation_diff/markup_test.rb b/test/translation_diff/markup_test.rb new file mode 100644 index 0000000..b6e24cf --- /dev/null +++ b/test/translation_diff/markup_test.rb @@ -0,0 +1,133 @@ +require "test_helper" + +class MarkupTest < Minitest::Test + def passage(source) + TranslationDiff::Passage.new(source, segmenter: TranslationDiff::Segmenters::Pragmatic.new) + end + + def cores(source) = passage(source).segments.reject(&:empty?).map(&:core) + + def translated(source) + subject = passage(source) + subject.segments.reject(&:empty?).each { |s| s.translation = s.core.upcase } + subject.render + end + + def assert_round_trips(source) + assert_equal source, passage(source).render, "render must return the source byte for byte" + end + + # -- entities ------------------------------------------------------------ + + # What reaches a provider is text, so it gets the character, not the entity. + def test_an_entity_is_decoded_before_translation + assert_equal ["Salt & pepper.", "Fine."], cores("Salt & pepper. Fine.") + end + + def test_a_non_breaking_space_is_decoded + assert_equal ["Hard\u00A0space here.", "Fine."], cores("Hard space here. Fine.") + end + + # The document keeps the entities it arrived with. + def test_entities_are_restored_on_render + assert_equal "SALT & PEPPER. FINE.", translated("Salt & pepper. Fine.") + end + + def test_an_untranslated_document_with_entities_round_trips + source = "Salt & pepper. Hard space. Fine." + + assert_equal source, passage(source).render + end + + # -- the bare < ---------------------------------------------------------- + + def test_a_bare_less_than_stays_in_the_sentence + assert_equal ["if a < b then stop.", "Fine."], cores("if a < b then stop. Fine.") + end + + def test_bare_angles_on_both_sides_stay_prose + assert_equal ["5 < 6 and 7 > 6.", "True."], cores("5 < 6 and 7 > 6. True.") + end + + def test_a_bare_less_than_survives_rendering_untranslated + source = "if a < b then stop. Fine." + + assert_equal source, passage(source).render + end + + # The distinction that makes this hard: a real tag must still be a tag when + # the same string also contains a bare <. + def test_a_real_tag_beside_a_bare_less_than_is_still_markup + assert_equal ["if a < b then", "stop."], cores("if a < b then stop.") + assert_equal "IF A < B THEN STOP.", translated("if a < b then stop.") + end + + def test_a_less_than_immediately_before_a_letter_is_a_tag + assert_equal ["Bold", "text."], cores("Bold text.") + end + + # -- the escape and its inverse ------------------------------------------ + + # Every mixture of real tags, bare angles and already-escaped angles the + # workaround has to survive; each must come back exactly as it went in. + MIXED = [ + "if a < b then stop.", + "5 < 6 and 7 > 6.", + "Bold and a < b.", + "a < b and c < d.", + "< is already an entity.", + "&lt; is an entity for an entity.", + "a < b, < c, &lt; d, e.", + " Visible.", + %(Link. After.), + "trailing angle <", + "<", + "<3 is not a tag.", + "" + ].freeze + + def test_restoring_undoes_escaping + MIXED.each do |source| + escaped = TranslationDiff::Markup.escape_bare_angles(source) + + assert_equal source, TranslationDiff::Markup.restore_bare_angles(escaped), + "escape and restore must be a matched pair for #{source.inspect}" + end + end + + def test_escaping_leaves_no_bare_angle_for_ox_to_swallow + MIXED.each do |source| + escaped = TranslationDiff::Markup.escape_bare_angles(source) + + refute_match(%r{<(?![A-Za-z!?]|/[A-Za-z])}, escaped, "#{escaped.inspect} still holds a bare <") + end + end + + def test_every_mixed_input_round_trips_through_a_passage + MIXED.each { |source| assert_round_trips(source) } + end + + # -- decoding and encoding ----------------------------------------------- + + def test_decoding_resolves_only_the_entities_encoding_can_put_back + assert_equal "& \u00A0 < >", TranslationDiff::Markup.decode_entities("&   < >") + end + + # < is left alone on purpose: restoring a bare angle is the escape's job, and + # a real tag inside a protected element must stay a real tag. + def test_encoding_puts_back_the_characters_decoding_took + assert_equal "&   ", TranslationDiff::Markup.encode_entities("& \u00A0 ") + end + + # -- what still has to hold ---------------------------------------------- + + # A notranslate element is prose that contains real tags; encoding must not eat them. + def test_a_notranslate_element_keeps_its_tags_through_a_render + assert_round_trips(%(Bold Mountain is a good place.)) + end + + def test_an_entity_inside_markup_is_left_for_the_browser + assert_round_trips(%(Link text. After.)) + assert_round_trips("Before.After.") + end +end diff --git a/test/translation_diff/pipeline_corpus_test.rb b/test/translation_diff/pipeline_corpus_test.rb index 6cad0df..5339eca 100644 --- a/test/translation_diff/pipeline_corpus_test.rb +++ b/test/translation_diff/pipeline_corpus_test.rb @@ -15,6 +15,13 @@ def translated(name) TranslationDiff.translate(PipelineCorpus::INPUTS.fetch(name), from: "en", to: "ru", provider: :null).inspect end + # What Passage now hands a provider, which is where the four fixed inputs actually differ. + def provider_texts(name) + passage = TranslationDiff::Passage.new(PipelineCorpus::INPUTS.fetch(name), + segmenter: TranslationDiff::Segmenters::Pragmatic.new) + passage.segments.reject(&:empty?).map(&:core) + end + def self.method_name_for(name) = :"test_#{name.gsub(/[^a-zA-Z0-9]+/, '_')}" (PipelineCorpus::INPUTS.keys - PipelineCorpus::EXPECTED_TO_CHANGE).each do |name| @@ -23,14 +30,21 @@ def self.method_name_for(name) = :"test_#{name.gsub(/[^a-zA-Z0-9]+/, '_')}" end end - # These four are named in EXPECTED_TO_CHANGE: the rewrite is scheduled to fix them, not to leave them alone. - # Skipped, not deleted -- a later task removes the skip and then this asserts the fixed output differs from - # today's recorded baseline. - PipelineCorpus::EXPECTED_TO_CHANGE.each do |name| + # The four named in EXPECTED_TO_CHANGE, written out: the document each produces, and the texts a provider is sent. + # The document is unchanged and has to stay so -- :null echoes, so an echoed document proves the round trip only. + # What the rewrite fixes is the second half. The old path sends ["Salt & pepper.", "Fine."] for the first, + # ["Hard space here.", "Fine."] for the second, ["if a"] for the third and ["5", "6.", "True."] for the fourth. + CHANGED = { + "entity ampersand" => ["Salt & pepper. Fine.", ["Salt & pepper.", "Fine."]], + "entity nbsp" => ["Hard space here. Fine.", ["Hard\u00A0space here.", "Fine."]], + "bare less-than" => ["if a < b then stop. Fine.", ["if a < b then stop.", "Fine."]], + "bare less-than and greater" => ["5 < 6 and 7 > 6. True.", ["5 < 6 and 7 > 6.", "True."]] + }.freeze + + CHANGED.each do |name, (document, texts)| define_method(method_name_for(name)) do - skip "scheduled to change: a later task removes this skip once the rewrite fixes this input " \ - "(see PipelineCorpus::EXPECTED_TO_CHANGE)" - refute_equal self.class.baseline_outputs.fetch(name), translated(name) + assert_equal document.inspect, translated(name) + assert_equal texts, provider_texts(name) end end end From 192c01e24245d7e510ee4fd99606ae7d344efac8 Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 01:37:50 +0400 Subject: [PATCH 09/21] fix: decode every entity so the ones we never touched stop being escaped twice encode_entities escaped every &, so an entity outside the three decode resolved came back escaped again -- > rendered as &gt;, and an & inside a notranslate element was respelled the caller never asked for. Decode named and numeric references, encode only & and <, and move both to Segment, which is what knows whether it was translated: untranslated renders the bytes it was cut from, translated renders equivalent markup. --- lib/translation_diff/markup.rb | 28 ++++--- lib/translation_diff/passage.rb | 14 +--- lib/translation_diff/segment.rb | 15 +++- test/support/pipeline_corpus.rb | 7 +- test/translation_diff/markup_test.rb | 62 ++++++++++++++-- test/translation_diff/pipeline_corpus_test.rb | 74 +++++++++++++++---- 6 files changed, 152 insertions(+), 48 deletions(-) diff --git a/lib/translation_diff/markup.rb b/lib/translation_diff/markup.rb index f071303..4e6f924 100644 --- a/lib/translation_diff/markup.rb +++ b/lib/translation_diff/markup.rb @@ -11,15 +11,17 @@ module TranslationDiff::Markup # `<` was a lone `<`; every further `amp;` is a level the source itself wrote and escaping pushed up by one. ESCAPED_ANGLE = /&((?:amp;)*)lt;/ - # `&` and ` ` reach Google raw today and it stops translating at them; `<` is this module's own escape. - DECODED = { "&" => "&", "<" => "<", " " => "\u00A0" }.freeze + # The bargain: an untranslated segment renders byte-exact, a translated one renders equivalent HTML, not equal bytes. - DECODABLE = /&(?:amp|lt|nbsp);/ + # Named and numeric alike, so nothing an `&` opens survives to be escaped again; ` ` because Google breaks on it. + DECODABLE = /&(?:nbsp|amp|lt|gt|quot|apos|#\d+|#[xX]\h+);/ - # `<` is missing on purpose: restoring a lone angle is the escape's job, and a real tag in prose must stay a tag. - ENCODED = { "&" => "&", "\u00A0" => " " }.freeze + NBSP = "\u00A0".freeze - ENCODABLE = /[&\u00A0]/ + # Only the two characters that are unsafe in HTML text; every other decoded character is left as the character it is. + ENCODED = { "&" => "&", "<" => "<" }.freeze + + ENCODABLE = /[&<]/ # Hands back markup `ox` can parse: same document, with every lone `<` written as the entity it should have been. def self.escape_bare_angles(source) @@ -34,9 +36,17 @@ def self.restore_bare_angles(rendered) end end - # What a provider is sent is text, so it gets the characters; a document arriving without entities gains none. - def self.decode_entities(text) = text.gsub(DECODABLE, DECODED) + # What a provider is sent is text, so it gets the characters; one left-to-right pass, so nothing is decoded twice. + def self.decode_entities(text) = text.gsub(DECODABLE) { |entity| decoded(entity) } + + # An entity CGI cannot decode stays as it arrived, and so does a surrogate: that decodes to invalid UTF-8. + def self.decoded(entity) + return NBSP if entity == " " + + plain = CGI.unescapeHTML(entity) + plain.valid_encoding? ? plain : entity + end - # What a document renders is markup, so a decoded character goes back to its entity -- a lone `&` gains one. + # What a document renders is markup, so text that changed is made safe again -- and only where it is unsafe. def self.encode_entities(text) = text.gsub(ENCODABLE, ENCODED) end diff --git a/lib/translation_diff/passage.rb b/lib/translation_diff/passage.rb index d957065..fa1117d 100644 --- a/lib/translation_diff/passage.rb +++ b/lib/translation_diff/passage.rb @@ -13,28 +13,20 @@ def initialize(source, segmenter:, language: nil) # The translatable sentences, in document order; the empty ones are whitespace a provider has no use for. def segments = fragments.flat_map(&:segments) - # Prose is handed back as the markup it came from; markup was never decoded, so only the escaped angles undo. + # Entities are a segment's business, so the only thing left to undo here is the escape this class put in. def render - TranslationDiff::Markup.restore_bare_angles(fragments.map { |fragment| rendered(fragment) }.join) + TranslationDiff::Markup.restore_bare_angles(fragments.map(&:render).join) end private - # Only prose was decoded, so only prose is encoded again; markup still holds the entities the document arrived with. - def rendered(fragment) - return fragment.render if fragment.markup? - - TranslationDiff::Markup.encode_entities(fragment.render) - end - # Every fragment is a slice of the source, never a rebuilt string; that is what makes an untranslated render exact. def fragment(run) from, to, prose = run slice = @source.byteslice(from, to - from) return TranslationDiff::Fragment.markup(slice) unless prose - TranslationDiff::Fragment.prose(TranslationDiff::Markup.decode_entities(slice), - segmenter: @segmenter, language: @language) + TranslationDiff::Fragment.prose(slice, segmenter: @segmenter, language: @language) end # Ox reports a byte position for every construct it sees; recording those is what lets rendering slice the source. diff --git a/lib/translation_diff/segment.rb b/lib/translation_diff/segment.rb index ad82c59..9810e35 100644 --- a/lib/translation_diff/segment.rb +++ b/lib/translation_diff/segment.rb @@ -1,17 +1,24 @@ -# A sentence with the whitespace it was found in, so rendering never needs a second string to restore it. +# A sentence with the whitespace it was found in: its core is the text a provider sees, its render is markup again. class TranslationDiff::Segment attr_reader :source, :core attr_accessor :translation + # A core is compared decoded, so a sentence that was only ` ` counts as the padding it is and is never sent. + BLANK = /\A[[:space:]]*\z/ + def initialize(source) @source = source.dup - @leading, @core, @trailing = @source.partition(/[^[:space:]].*[^[:space:]]|[^[:space:]]/m) + @leading, @body, @trailing = @source.partition(/[^[:space:]].*[^[:space:]]|[^[:space:]]/m) + @core = TranslationDiff::Markup.decode_entities(@body) end # Reflects whether a translation is set right now, not history -- clearing it to nil flips this back to false. def translated? = !translation.nil? - def empty? = core.empty? + def empty? = core.match?(BLANK) - def render = "#{@leading}#{translated? ? translation : core}#{@trailing}" + # Untranslated hands back the bytes it was cut from; a translation is text, so it is encoded as markup on the way out. + def render + "#{@leading}#{translated? ? TranslationDiff::Markup.encode_entities(translation) : @body}#{@trailing}" + end end diff --git a/test/support/pipeline_corpus.rb b/test/support/pipeline_corpus.rb index 47488d1..2da7416 100644 --- a/test/support/pipeline_corpus.rb +++ b/test/support/pipeline_corpus.rb @@ -32,7 +32,9 @@ module PipelineCorpus "entity ampersand" => "Salt & pepper. Fine.", "entity nbsp" => "Hard space here. Fine.", "bare less-than" => "if a < b then stop. Fine.", - "bare less-than and greater" => "5 < 6 and 7 > 6. True." + "bare less-than and greater" => "5 < 6 and 7 > 6. True.", + # A recorded limit, not a defect: `Bold" without a lexer of our own. + "bare less-than before a letter" => "a \" ' \u00A0", TranslationDiff::Markup.decode_entities("& < > " '  ") + assert_equal "& & \u00A0 \u00A0", TranslationDiff::Markup.decode_entities("& &    ") + end + + # Sane rather than an exception, and sane here means untouched: what is not an entity is text, and stays text. + def test_decoding_leaves_a_malformed_entity_exactly_as_it_arrived + ["¬anentity;", "&#xZZ;", "&#;", "�", "…", "AT&T", "a > b", "&"].each do |text| + assert_equal text, TranslationDiff::Markup.decode_entities(text) + end + end + + # A numeric reference can name a surrogate; decoding one would hand back invalid UTF-8 for a later regexp to raise on. + def test_decoding_refuses_a_reference_that_would_not_be_valid_utf8 + decoded = TranslationDiff::Markup.decode_entities("�") + + assert_equal "�", decoded + assert_predicate decoded, :valid_encoding? end - # < is left alone on purpose: restoring a bare angle is the escape's job, and - # a real tag inside a protected element must stay a real tag. - def test_encoding_puts_back_the_characters_decoding_took - assert_equal "&   ", TranslationDiff::Markup.encode_entities("& \u00A0 ") + # Only the two characters that are unsafe in HTML text; a decoded character stays the character it decoded to. + def test_encoding_touches_only_the_ampersand_and_the_opening_angle + assert_equal "& <b> > \" ' \u00A0", TranslationDiff::Markup.encode_entities("& > \" ' \u00A0") + end + + # -- entities we never decoded ------------------------------------------- + + # Every one of these came back with its `&` escaped a second time before the decode was made whole. + def test_an_entity_outside_the_decoded_set_is_not_escaped_again + assert_round_trips("A > B here. Fine.") + assert_round_trips("AT&T is a company. Fine.") + assert_round_trips("© 2026. Fine.") + assert_round_trips("¬anentity; here. Fine.") + end + + # The bargain, in a test: bytes are promised only while a segment is untranslated. + def test_a_translated_sentence_renders_equivalent_markup_rather_than_equal_bytes + assert_equal "A > B here. Fine.", echoed("A > B here. Fine.") + assert_equal "AT&T is a company. Fine.", echoed("AT&T is a company. Fine.") + assert_equal "Hard\u00A0space here. Fine.", echoed("Hard space here. Fine.") + end + + # CGI's table is the HTML specials and the numeric forms, so a `©` that is translated is spelled, not resolved. + def test_a_named_entity_cgi_cannot_decode_survives_translation_as_text + assert_equal "&copy; 2026. Fine.", echoed("© 2026. Fine.") end # -- what still has to hold ---------------------------------------------- @@ -126,6 +171,11 @@ def test_a_notranslate_element_keeps_its_tags_through_a_render assert_round_trips(%(Bold Mountain is a good place.)) end + # Passed through untouched means untouched: an `&` inside a protected element is not ours to respell. + def test_a_notranslate_element_keeps_its_ampersands_through_a_render + assert_round_trips(%(R&D & more Fine.)) + end + def test_an_entity_inside_markup_is_left_for_the_browser assert_round_trips(%(Link text. After.)) assert_round_trips("Before.After.") diff --git a/test/translation_diff/pipeline_corpus_test.rb b/test/translation_diff/pipeline_corpus_test.rb index 5339eca..3940794 100644 --- a/test/translation_diff/pipeline_corpus_test.rb +++ b/test/translation_diff/pipeline_corpus_test.rb @@ -11,15 +11,27 @@ def self.baseline_outputs @baseline_outputs ||= File.read(BASELINE_PATH).scan(/^=== (.+) ===\nOUTPUT: (.*)\n/).to_h end + # Still the old pipeline until task 9, so this half of an EXPECTED_TO_CHANGE case only holds the line. def translated(name) TranslationDiff.translate(PipelineCorpus::INPUTS.fetch(name), from: "en", to: "ru", provider: :null).inspect end - # What Passage now hands a provider, which is where the four fixed inputs actually differ. - def provider_texts(name) - passage = TranslationDiff::Passage.new(PipelineCorpus::INPUTS.fetch(name), - segmenter: TranslationDiff::Segmenters::Pragmatic.new) - passage.segments.reject(&:empty?).map(&:core) + def passage(name) + TranslationDiff::Passage.new(PipelineCorpus::INPUTS.fetch(name), + segmenter: TranslationDiff::Segmenters::Pragmatic.new) + end + + # What Passage now hands a provider, which is where the fixed inputs actually differ. + def provider_texts(name) = passage(name).segments.reject(&:empty?).map(&:core) + + # Nothing translated, so the document owes its caller the bytes it arrived as. + def untranslated_render(name) = passage(name).render + + # Every sentence back as it was sent, which is what :null does -- the new pipeline's answer to the document column. + def echoed_render(name) + subject = passage(name) + subject.segments.reject(&:empty?).each { |segment| segment.translation = segment.core } + subject.render end def self.method_name_for(name) = :"test_#{name.gsub(/[^a-zA-Z0-9]+/, '_')}" @@ -30,21 +42,51 @@ def self.method_name_for(name) = :"test_#{name.gsub(/[^a-zA-Z0-9]+/, '_')}" end end - # The four named in EXPECTED_TO_CHANGE, written out: the document each produces, and the texts a provider is sent. - # The document is unchanged and has to stay so -- :null echoes, so an echoed document proves the round trip only. - # What the rewrite fixes is the second half. The old path sends ["Salt & pepper.", "Fine."] for the first, - # ["Hard space here.", "Fine."] for the second, ["if a"] for the third and ["5", "6.", "True."] for the fourth. + # The names in EXPECTED_TO_CHANGE, written out: the texts a provider is sent, the document the old path still + # produces, and both of Passage's renders -- byte-exact untranslated, equivalent markup once every sentence is back. + # The old path sends ["Salt & pepper.", "Fine."] for the first, ["Hard space here.", "Fine."] for the + # second, ["if a"] for the third, ["5", "6.", "True."] for the fourth and ["a"] for the fifth. CHANGED = { - "entity ampersand" => ["Salt & pepper. Fine.", ["Salt & pepper.", "Fine."]], - "entity nbsp" => ["Hard space here. Fine.", ["Hard\u00A0space here.", "Fine."]], - "bare less-than" => ["if a < b then stop. Fine.", ["if a < b then stop.", "Fine."]], - "bare less-than and greater" => ["5 < 6 and 7 > 6. True.", ["5 < 6 and 7 > 6.", "True."]] + "entity ampersand" => { + texts: ["Salt & pepper.", "Fine."], + document: "Salt & pepper. Fine.", + echoed: "Salt & pepper. Fine." + }, + "entity nbsp" => { + texts: ["Hard\u00A0space here.", "Fine."], + document: "Hard space here. Fine.", + # The entity is spelled as the character it means, which is the same document to a browser and not the same bytes. + echoed: "Hard\u00A0space here. Fine." + }, + "bare less-than" => { + texts: ["if a < b then stop.", "Fine."], + document: "if a < b then stop. Fine.", + echoed: "if a < b then stop. Fine." + }, + "bare less-than and greater" => { + texts: ["5 < 6 and 7 > 6.", "True."], + document: "5 < 6 and 7 > 6. True.", + echoed: "5 < 6 and 7 > 6. True." + }, + # The recorded limit: ` { + texts: ["a"], + document: "a Date: Thu, 10 Sep 2026 01:42:42 +0400 Subject: [PATCH 10/21] fix: resolve the whole HTML5 named entity set, not just CGI's five A translated sentence holding ©, — or ’ rendered the entity spelled out, because CGI.unescapeHTML knows only the specials and the numeric forms, so encoding escaped the & it left behind. Ox knows every name and is already a dependency. Resolve one name at a time, alone in an element of its own -- whole prose raises on a lone & and returns garbage for an out-of-range reference -- and keep numeric references, and the invalid-UTF-8 guard, on the CGI path. --- lib/translation_diff/markup.rb | 38 +++++++++++++++++++++++----- test/translation_diff/markup_test.rb | 19 +++++++++++--- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/lib/translation_diff/markup.rb b/lib/translation_diff/markup.rb index 4e6f924..e359922 100644 --- a/lib/translation_diff/markup.rb +++ b/lib/translation_diff/markup.rb @@ -13,10 +13,11 @@ module TranslationDiff::Markup # The bargain: an untranslated segment renders byte-exact, a translated one renders equivalent HTML, not equal bytes. - # Named and numeric alike, so nothing an `&` opens survives to be escaped again; ` ` because Google breaks on it. - DECODABLE = /&(?:nbsp|amp|lt|gt|quot|apos|#\d+|#[xX]\h+);/ + # Named and numeric alike, so nothing an `&` opens survives to be escaped again and rendered as its own spelling. + DECODABLE = /&(?:[A-Za-z][A-Za-z0-9]*|#\d+|#[xX]\h+);/ - NBSP = "\u00A0".freeze + # Which of the two decoders an entity belongs to: Ox knows every HTML5 name, CGI knows both numeric forms. + NAMED = /\A&([A-Za-z][A-Za-z0-9]*);\z/ # Only the two characters that are unsafe in HTML text; every other decoded character is left as the character it is. ENCODED = { "&" => "&", "<" => "<" }.freeze @@ -39,14 +40,37 @@ def self.restore_bare_angles(rendered) # What a provider is sent is text, so it gets the characters; one left-to-right pass, so nothing is decoded twice. def self.decode_entities(text) = text.gsub(DECODABLE) { |entity| decoded(entity) } - # An entity CGI cannot decode stays as it arrived, and so does a surrogate: that decodes to invalid UTF-8. + # An entity neither decoder knows stays as it arrived, and so does a surrogate: that decodes to invalid UTF-8. def self.decoded(entity) - return NBSP if entity == " " - - plain = CGI.unescapeHTML(entity) + name = entity[NAMED, 1] + plain = name ? named(name) : CGI.unescapeHTML(entity) plain.valid_encoding? ? plain : entity end + # A document repeats the same handful of names, and only a name that resolved is kept, so the table cannot be grown. + def self.named(name) = resolved[name] || resolve(name) + + def self.resolved = @resolved ||= {} + + # One well-formed entity alone in an element is the only input Ox decodes safely -- prose with a lone `&` raises. + def self.resolve(name) + entity = "&#{name};" + resolver = Resolver.new + Ox.sax_html(resolver, StringIO.new("#{entity}")) + return entity if resolver.text.nil? || resolver.text == entity + + resolved[name] = resolver.text + rescue StandardError + entity + end + # What a document renders is markup, so text that changed is made safe again -- and only where it is unsafe. def self.encode_entities(text) = text.gsub(ENCODABLE, ENCODED) + + # Ox hands back the decoded text of the one element it was given; a name it does not know arrives as the text it was. + class Resolver < Ox::Sax + attr_reader :text + + def value(value) = @text = value.as_s + end end diff --git a/test/translation_diff/markup_test.rb b/test/translation_diff/markup_test.rb index 533696a..400f408 100644 --- a/test/translation_diff/markup_test.rb +++ b/test/translation_diff/markup_test.rb @@ -118,13 +118,14 @@ def test_every_mixed_input_round_trips_through_a_passage # Every entity is decoded, not the two that were measured: an `&` we leave behind is an `&` encoding corrupts. def test_decoding_resolves_named_and_numeric_entities + assert_equal "\u00A9 \u2014 \u2026", TranslationDiff::Markup.decode_entities("© — …") assert_equal "& < > \" ' \u00A0", TranslationDiff::Markup.decode_entities("& < > " '  ") assert_equal "& & \u00A0 \u00A0", TranslationDiff::Markup.decode_entities("& &    ") end # Sane rather than an exception, and sane here means untouched: what is not an entity is text, and stays text. def test_decoding_leaves_a_malformed_entity_exactly_as_it_arrived - ["¬anentity;", "&#xZZ;", "&#;", "�", "…", "AT&T", "a > b", "&"].each do |text| + ["¬anentity;", "&#xZZ;", "&#;", "�", "&Bogus9;", "AT&T", "a > b", "&"].each do |text| assert_equal text, TranslationDiff::Markup.decode_entities(text) end end @@ -150,6 +151,7 @@ def test_an_entity_outside_the_decoded_set_is_not_escaped_again assert_round_trips("AT&T is a company. Fine.") assert_round_trips("© 2026. Fine.") assert_round_trips("¬anentity; here. Fine.") + assert_round_trips("One — two … three. Fine.") end # The bargain, in a test: bytes are promised only while a segment is untranslated. @@ -159,9 +161,18 @@ def test_a_translated_sentence_renders_equivalent_markup_rather_than_equal_bytes assert_equal "Hard\u00A0space here. Fine.", echoed("Hard space here. Fine.") end - # CGI's table is the HTML specials and the numeric forms, so a `©` that is translated is spelled, not resolved. - def test_a_named_entity_cgi_cannot_decode_survives_translation_as_text - assert_equal "&copy; 2026. Fine.", echoed("© 2026. Fine.") + # The whole HTML5 named set, not just the specials CGI knows: a spelled-out `©` would display as text, not as ©. + def test_a_named_entity_outside_cgis_table_is_translated_as_the_character_it_means + assert_equal ["\u00A9 2026.", "Fine."], cores("© 2026. Fine.") + assert_equal "\u00A9 2026. Fine.", echoed("© 2026. Fine.") + assert_equal "One \u2014 two \u2026 three. Fine.", echoed("One — two … three. Fine.") + end + + # A lone `&` is invalid XML and Ox raises on it, so nothing but a single well-formed entity is ever handed over. + def test_a_lone_ampersand_never_reaches_the_entity_resolver + assert_equal ["AT&T is a company.", "Fine."], cores("AT&T is a company. Fine.") + assert_equal "AT&T is a company. Fine.", echoed("AT&T is a company. Fine.") + assert_equal "R&D; x", TranslationDiff::Markup.decode_entities("R&D; x") end # -- what still has to hold ---------------------------------------------- From be9d0f748a4836aefc972bfcdfe8b19d73b53e92 Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 01:55:35 +0400 Subject: [PATCH 11/21] feat: group segments into provider-sized requests Batch.pack fills batches against a provider's declared max_batch_size and max_request_size (escaped length), and Batch#apply attaches a reply to the segments that produced it by the batch's own index rather than by position matched afterwards. Not wired into the pipeline yet. --- lib/translation_diff.rb | 1 + lib/translation_diff/batch.rb | 95 +++++++++++++++++++++++++++++ test/translation_diff/batch_test.rb | 88 ++++++++++++++++++++++++++ 3 files changed, 184 insertions(+) create mode 100644 lib/translation_diff/batch.rb create mode 100644 test/translation_diff/batch_test.rb diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index 3801d76..0418e20 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -16,6 +16,7 @@ require "translation_diff/document" require "translation_diff/markup" require "translation_diff/segment" +require "translation_diff/batch" require "translation_diff/fragment" require "translation_diff/passage" require "translation_diff/configuration" diff --git a/lib/translation_diff/batch.rb b/lib/translation_diff/batch.rb new file mode 100644 index 0000000..1b9e8c5 --- /dev/null +++ b/lib/translation_diff/batch.rb @@ -0,0 +1,95 @@ +# Segments grouped to one provider request; a reply lands back on them through #apply, never by position after the fact. +class TranslationDiff::Batch + attr_reader :segments + + def initialize(segments) + @segments = segments + end + + def texts = segments.map(&:core) + + # Each translation lands on the segment at the same index in this batch, the segment that produced it. + def apply(translations) + ensure_reply_size!(translations) + segments.each_with_index { |segment, index| segment.translation = translations[index] } + end + + class << self + # Skips blank segments -- a provider has no use for whitespace -- then fills batches within both declared limits. + def pack(segments, capabilities:) + filler = Filler.new(capabilities) + segments.reject(&:empty?).each { |segment| filler.add(segment) } + filler.batches + end + end + + private + + def ensure_reply_size!(translations) + return if translations.size == segments.size + + raise TranslationDiff::ResponseError, + "Provider returned #{translations.size} translations for #{segments.size} segments" + end + + # Fills one batch until its count or its cumulative escaped size would cross the limit, then starts the next. + class Filler + def initialize(capabilities) + @capabilities = capabilities + @batches = [] + @current = [] + @current_size = 0 + end + + def add(segment) + size = escaped_size(segment) + flush if full?(size) + @current << segment + @current_size += size + end + + def batches + flush + @batches + end + + private + + # Escaped, because that is the size a provider's own limit is documented against and what goes over the wire. + def escaped_size(segment) + size = CGI.escape(segment.core).size + ensure_sendable!(segment, size) + size + end + + # A text over either declared limit can never be sent, alone or otherwise, so this raises before any batch fills. + def ensure_sendable!(segment, size) + limit = [@capabilities.max_request_size, @capabilities.max_text_size].compact.min + return if size <= limit + + raise TranslationDiff::Error, + "#{preview(segment.core)} is #{size} characters once escaped, over this provider's limit of #{limit}" + end + + # A short prefix locates the offending text without reproducing it -- the rest is the customer's content. + def preview(text) + return text.dup if text.size <= 20 + + "#{text[0, 20]}..." + end + + def full?(size) + return false if @current.empty? + + @current.size >= @capabilities.max_batch_size || @current_size + size > @capabilities.max_request_size + end + + def flush + return if @current.empty? + + @batches << TranslationDiff::Batch.new(@current) + @current = [] + @current_size = 0 + end + end +end diff --git a/test/translation_diff/batch_test.rb b/test/translation_diff/batch_test.rb new file mode 100644 index 0000000..6559bdf --- /dev/null +++ b/test/translation_diff/batch_test.rb @@ -0,0 +1,88 @@ +require "test_helper" + +class BatchTest < Minitest::Test + def capabilities(request_size: 1_000, batch_size: 10, text_size: nil) + TranslationDiff::Capabilities.new( + max_request_size: request_size, max_batch_size: batch_size, max_text_size: text_size, + html: :none, notranslate: false, detects_language: false, reports_billing: false + ) + end + + def segments(*sources) = sources.map { |s| TranslationDiff::Segment.new(s) } + + def pack(sources, **) + TranslationDiff::Batch.pack(segments(*sources), capabilities: capabilities(**)) + end + + def test_everything_fits_in_one_batch_when_it_fits + batches = pack(%w[one two three]) + + assert_equal 1, batches.size + assert_equal %w[one two three], batches.first.texts + end + + def test_the_count_limit_starts_a_new_batch + batches = pack(%w[one two three four five], batch_size: 2) + + assert_equal [%w[one two], %w[three four], %w[five]], batches.map(&:texts) + end + + def test_the_size_limit_starts_a_new_batch + batches = pack(%w[aaaa bbbb cccc], request_size: 9) + + assert_equal [%w[aaaa bbbb], %w[cccc]], batches.map(&:texts) + end + + # A provider that cannot batch gets one text per request, and that is + # correct rather than a degenerate case -- Amazon Translate has no batch + # form of its endpoint at all. + def test_a_batch_size_of_one_produces_one_request_per_text + assert_equal [%w[one], %w[two], %w[three]], pack(%w[one two three], batch_size: 1).map(&:texts) + end + + # Size is the escaped length, because that is what goes over the wire. + def test_size_is_measured_escaped_not_in_characters + batches = pack(%w[привет привет], request_size: 40) + + assert_equal 2, batches.size, "each Cyrillic word is 36 escaped characters" + end + + def test_a_text_larger_than_the_request_limit_raises_naming_the_limit + error = assert_raises(TranslationDiff::Error) { pack(["x" * 50], request_size: 10) } + + assert_match(/10/, error.message) + assert_match(/50/, error.message) + end + + # The error locates the sentence without reproducing it: the whole sentence + # is the customer's content and this message may be logged. + def test_the_too_long_error_does_not_carry_the_whole_sentence + sentence = "Secret #{'y' * 200}" + error = assert_raises(TranslationDiff::Error) { pack([sentence], request_size: 10) } + + refute_includes error.message, "y" * 200 + end + + def test_apply_puts_each_translation_on_its_own_segment + batch = pack(%w[one two]).first + batch.apply(%w[один два]) + + assert_equal %w[один два], batch.segments.map(&:translation) + end + + def test_apply_refuses_a_reply_of_the_wrong_length + batch = pack(%w[one two]).first + + assert_raises(TranslationDiff::ResponseError) { batch.apply(%w[один]) } + end + + def test_empty_segments_are_never_packed + batches = TranslationDiff::Batch.pack(segments("one", " ", "two"), capabilities: capabilities) + + assert_equal [%w[one two]], batches.map(&:texts) + end + + def test_no_segments_produce_no_batches + assert_empty TranslationDiff::Batch.pack([], capabilities: capabilities) + end +end From a0a0f7bdc6ab0b7760f33eb6ccc1d54e20d25d7e Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 02:08:32 +0400 Subject: [PATCH 12/21] feat: cache sentences without consuming the caller's collection SentenceCache derives its cache key from a digest of the sentence's raw, pre-decode text plus any per-call provider options -- the format is pinned against the pipeline-baseline recorded keys so existing users don't miss their entire cache on upgrade. fill and store both hand back fresh arrays rather than draining the collections they are given. --- lib/translation_diff.rb | 1 + lib/translation_diff/sentence_cache.rb | 42 ++++++ test/translation_diff/sentence_cache_test.rb | 130 +++++++++++++++++++ 3 files changed, 173 insertions(+) create mode 100644 lib/translation_diff/sentence_cache.rb create mode 100644 test/translation_diff/sentence_cache_test.rb diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index 0418e20..d464a3b 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -19,6 +19,7 @@ require "translation_diff/batch" require "translation_diff/fragment" require "translation_diff/passage" +require "translation_diff/sentence_cache" require "translation_diff/configuration" require "translation_diff/configuration/provider_option_owners" diff --git a/lib/translation_diff/sentence_cache.rb b/lib/translation_diff/sentence_cache.rb new file mode 100644 index 0000000..d7a5169 --- /dev/null +++ b/lib/translation_diff/sentence_cache.rb @@ -0,0 +1,42 @@ +# Reads and writes sentence translations under the pipeline's cache key format, without touching the caller's array. +class TranslationDiff::SentenceCache + # Segment strips this same class of whitespace into leading/trailing padding; this recovers the undecoded body. + UNPADDED = /[^[:space:]].*[^[:space:]]|[^[:space:]]/m + + def initialize(store:, provider:, from:, to:, options: {}) + @store = store + @provider = provider + @from = from.downcase + @to = to.downcase + @options = options + end + + # provider:from:to, then one digest covering both the per-call options and the sentence as it appeared in markup. + def key(segment) + "#{@provider}:#{@from}:#{@to}:#{Digest::MD5.hexdigest("#{options_digest}#{raw_text(segment)}")}" + end + + # Sets a translation on every segment the store already has cached, and hands back the rest as a new array. + def fill(segments) + values = @store.read_multi(segments.map { |segment| key(segment) }) + misses = [] + segments.each_with_index do |segment, index| + value = values[index] + value.nil? ? misses << segment : segment.translation = value + end + misses + end + + # Writes back only the segments that carry a translation; an untranslated segment has nothing worth caching. + def store(segments) + segments.select(&:translated?).each { |segment| @store.write(key(segment), segment.translation) } + end + + private + + # Empty for the common case of no per-call options, which is what the recorded baseline keys assume. + def options_digest = @options.empty? ? "" : @options.sort.to_s + + # The sentence as it appeared in markup, before entity decoding, which is what the recorded keys were hashed from. + def raw_text(segment) = segment.source.partition(UNPADDED)[1] +end diff --git a/test/translation_diff/sentence_cache_test.rb b/test/translation_diff/sentence_cache_test.rb new file mode 100644 index 0000000..e385719 --- /dev/null +++ b/test/translation_diff/sentence_cache_test.rb @@ -0,0 +1,130 @@ +require "test_helper" + +class SentenceCacheTest < Minitest::Test + # Records what it is asked for, so a test can assert on keys as well as + # values. Minitest 6 has no mocking library; this is the whole contract. + class RecordingStore + attr_reader :reads, :writes + + def initialize(values = {}) + @values = values + @reads = [] + @writes = {} + end + + def read_multi(keys) + @reads.concat(keys) + keys.map { |k| @values[k] } + end + + def write(key, value) = @writes[key] = value + end + + def cache(store, **) + TranslationDiff::SentenceCache.new( + store: store, provider: "deepl", from: "en", to: "ru", ** + ) + end + + def segments(*sources) = sources.map { |s| TranslationDiff::Segment.new(s) } + + def test_a_hit_fills_the_segment_and_is_not_returned_as_a_miss + subject = segments("One.") + store = RecordingStore.new + key = cache(store).key(subject.first) + store = RecordingStore.new(key => "Один.") + + misses = cache(store).fill(subject) + + assert_equal "Один.", subject.first.translation + assert_empty misses + end + + def test_a_miss_is_returned_and_left_untranslated + subject = segments("One.") + + misses = cache(RecordingStore.new).fill(subject) + + assert_equal subject, misses + refute_predicate subject.first, :translated? + end + + def test_store_writes_only_the_translated_ones + subject = segments("One.", "Two.") + subject.first.translation = "Один." + store = RecordingStore.new + subject_cache = cache(store) + + subject_cache.store(subject) + + assert_equal ["Один."], store.writes.values + end + + # The bug this replaces: the old cache consumed the array of updates it was + # handed, emptying a collection that belonged to its caller. + def test_neither_operation_modifies_the_collection_it_is_given + subject = segments("One.", "Two.") + original = subject.dup + + subject_cache = cache(RecordingStore.new) + subject_cache.fill(subject) + subject_cache.store(subject) + + assert_equal original, subject + assert_equal 2, subject.size + end + + def test_the_same_sentence_in_two_languages_gets_two_keys + segment = segments("One.").first + en_ru = cache(RecordingStore.new).key(segment) + en_de = TranslationDiff::SentenceCache.new( + store: RecordingStore.new, provider: "deepl", from: "en", to: "de" + ).key(segment) + + refute_equal en_ru, en_de + end + + def test_two_providers_do_not_share_a_key + segment = segments("One.").first + deepl = cache(RecordingStore.new).key(segment) + google = TranslationDiff::SentenceCache.new( + store: RecordingStore.new, provider: "google", from: "en", to: "ru" + ).key(segment) + + refute_equal deepl, google + end + + def test_per_call_options_are_part_of_the_key + segment = segments("One.").first + plain = cache(RecordingStore.new).key(segment) + formal = cache(RecordingStore.new, options: { formality: :more }).key(segment) + + refute_equal plain, formal + end + + # The promise that a released cache keeps working. These values come from + # the baseline Task 1 recorded against the pipeline being replaced. + # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength + def test_keys_match_the_ones_the_previous_pipeline_produced + subject_cache = TranslationDiff::SentenceCache.new( + store: RecordingStore.new, provider: "null", from: "en", to: "ru" + ) + + assert_equal "null:en:ru:9d6a2963872077db674a27a39c492e61", + subject_cache.key(segments("Hello there.").first) + assert_equal "null:en:ru:1520f71fffb5adf0da75e7c17059bfd1", + subject_cache.key(segments("Second sentence!").first) + assert_equal "null:en:ru:900019fa233e608091ba641d50d69b81", + subject_cache.key(segments("One.").first) + assert_equal "null:en:ru:fdb02803abc46fba06ce1cc96d6399c5", + subject_cache.key(segments("Two.").first) + assert_equal "null:en:ru:3f77101fc43570a61d5bc042bb908651", + subject_cache.key(segments("Third.").first) + assert_equal "null:en:ru:fe3bf43723a64fb32bcac8d99bb431af", + subject_cache.key(segments(" Padded sentence. ").first) + assert_equal "null:en:ru:05d12994070fdde458e566149f42472f", + subject_cache.key(segments("Salt & pepper.").first) + assert_equal "null:en:ru:b5010567e209726a125c9ed59162eca5", + subject_cache.key(segments("Fine.").first) + end +end From 6f2fb103fcd0384a3dc30171780a18a5d8326a86 Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 02:17:54 +0400 Subject: [PATCH 13/21] fix: give the options digest its own cache key field The per-call options were folded into the sentence digest, so a caller passing formality: or a glossary id computed a key no cache could ever hold. The format has five fields when options are present and four when they are not, which is why the recorded baseline -- captured without options -- could not catch this. Canonicalise the options by sorting on the key's string form: @options.sort raised ArgumentError on a hash mixing Symbol and String keys. Render values with #inspect and raise a named SentenceCache::Error when one renders as an object address, rather than emitting a key that changes every process and reports nothing. Hash Segment#body instead of a local copy of Segment's whitespace-boundary regex. The two agreed, which is the reason to close it: this branch has already been bitten by two hand-synchronised definitions of padding. --- lib/translation_diff/segment.rb | 2 +- lib/translation_diff/sentence_cache.rb | 33 ++++++++++---- test/translation_diff/sentence_cache_test.rb | 47 ++++++++++++++++++++ 3 files changed, 73 insertions(+), 9 deletions(-) diff --git a/lib/translation_diff/segment.rb b/lib/translation_diff/segment.rb index 9810e35..9b04f0a 100644 --- a/lib/translation_diff/segment.rb +++ b/lib/translation_diff/segment.rb @@ -1,6 +1,6 @@ # A sentence with the whitespace it was found in: its core is the text a provider sees, its render is markup again. class TranslationDiff::Segment - attr_reader :source, :core + attr_reader :source, :body, :core attr_accessor :translation # A core is compared decoded, so a sentence that was only ` ` counts as the padding it is and is never sent. diff --git a/lib/translation_diff/sentence_cache.rb b/lib/translation_diff/sentence_cache.rb index d7a5169..850fae4 100644 --- a/lib/translation_diff/sentence_cache.rb +++ b/lib/translation_diff/sentence_cache.rb @@ -1,7 +1,10 @@ # Reads and writes sentence translations under the pipeline's cache key format, without touching the caller's array. class TranslationDiff::SentenceCache - # Segment strips this same class of whitespace into leading/trailing padding; this recovers the undecoded body. - UNPADDED = /[^[:space:]].*[^[:space:]]|[^[:space:]]/m + # Its own class, so rescuing an unusable option cannot also swallow a store or provider failure. + class Error < TranslationDiff::Error; end + + # Ruby's default rendering of an object is its address, which would move the key every process. + ADDRESS = /#<[^>]*0x\h+/ def initialize(store:, provider:, from:, to:, options: {}) @store = store @@ -11,9 +14,9 @@ def initialize(store:, provider:, from:, to:, options: {}) @options = options end - # provider:from:to, then one digest covering both the per-call options and the sentence as it appeared in markup. + # provider:from:to:sentence, with a digest of the per-call options wedged in as a field of its own when there are any. def key(segment) - "#{@provider}:#{@from}:#{@to}:#{Digest::MD5.hexdigest("#{options_digest}#{raw_text(segment)}")}" + [@provider, @from, @to, *options_digest, Digest::MD5.hexdigest(segment.body)].join(":") end # Sets a translation on every segment the store already has cached, and hands back the rest as a new array. @@ -34,9 +37,23 @@ def store(segments) private - # Empty for the common case of no per-call options, which is what the recorded baseline keys assume. - def options_digest = @options.empty? ? "" : @options.sort.to_s + # No options contributes no field at all, which is the four-field key every already-warm cache is keyed on. + def options_digest + return [] if @options.empty? + + [Digest::MD5.hexdigest(canonical_options)[0, 8]] + end - # The sentence as it appeared in markup, before entity decoding, which is what the recorded keys were hashed from. - def raw_text(segment) = segment.source.partition(UNPADDED)[1] + # Sorted on the name's string form, because a Symbol and a String key are not comparable with each other. + def canonical_options + @options.sort_by { |name, _| name.to_s }.map { |name, value| "#{name}=#{stable(name, value)}" }.join("&") + end + + # A value that renders as an address makes a key nothing can ever hit twice, so say so where a caller will see it. + def stable(name, value) + rendered = value.inspect + raise Error, "cache option #{name} (a #{value.class}) has no stable string form" if rendered.match?(ADDRESS) + + rendered + end end diff --git a/test/translation_diff/sentence_cache_test.rb b/test/translation_diff/sentence_cache_test.rb index e385719..06a5ec5 100644 --- a/test/translation_diff/sentence_cache_test.rb +++ b/test/translation_diff/sentence_cache_test.rb @@ -127,4 +127,51 @@ def test_keys_match_the_ones_the_previous_pipeline_produced assert_equal "null:en:ru:b5010567e209726a125c9ed59162eca5", subject_cache.key(segments("Fine.").first) end + + # Four fields without options, five with: the options digest is its own field, not folded into the sentence. + def test_the_options_digest_is_a_field_of_its_own + segment = segments("One.").first + + assert_equal 4, cache(RecordingStore.new).key(segment).split(":").size + assert_equal 5, cache(RecordingStore.new, options: { formality: :more }).key(segment).split(":").size + end + + # Pinned as a literal: anyone passing formality or a glossary id has to keep hitting this exact key. + def test_a_key_carrying_options_is_pinned_field_by_field + subject_cache = TranslationDiff::SentenceCache.new( + store: RecordingStore.new, provider: "null", from: "en", to: "ru", options: { formality: :more } + ) + + assert_equal "null:en:ru:c09f3c46:900019fa233e608091ba641d50d69b81", + subject_cache.key(segments("One.").first) + end + + # A Symbol and a String are not comparable with each other, so sorting on the raw keys raises on this hash. + def test_mixed_option_key_types_canonicalise_instead_of_raising + segment = segments("One.").first + symbol_first = cache(RecordingStore.new, options: { formality: :more, "glossary" => "g" }) + string_first = cache(RecordingStore.new, options: { "glossary" => "g", formality: :more }) + + assert_equal symbol_first.key(segment), string_first.key(segment) + end + + # A value rendering as an address gives a key that can never be hit twice, so it fails where a user can see it. + def test_an_option_with_no_stable_string_form_raises_and_names_it + subject_cache = cache(RecordingStore.new, options: { glossary: Object.new }) + + error = assert_raises(TranslationDiff::SentenceCache::Error) { subject_cache.key(segments("One.").first) } + + assert_kind_of TranslationDiff::Error, error + assert_includes error.message, "glossary" + end + + # One definition of padding, the one Segment already makes: the key hashes the body it cut, not a copy of its regex. + def test_the_key_hashes_the_body_segment_cut + segment = segments(" Padded sentence. ").first + + key = cache(RecordingStore.new).key(segment) + + assert_equal "Padded sentence.", segment.body + assert_equal Digest::MD5.hexdigest(segment.body), key.split(":").last + end end From 003aa49fe534c050e273a71b29f23d30cc3adc50 Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 02:20:15 +0400 Subject: [PATCH 14/21] fix: join canonical cache options with a comma, not an ampersand The separator could not be observed from a single-option key, so the last pass picked one. Running the pipeline being replaced against a recording store shows it is ",": "&" gives 92c55e66 where the old pipeline asks for c1ee2461, and every caller passing two options misses their whole cache. Pin one-option, two-option and out-of-order-key hashes as literals. The two-option case is the first that has anything to put a separator between, and {b: 2, a: 1} is the only one whose literal order differs from its sorted order, so it is what proves the sort rather than a lucky ordering. --- lib/translation_diff/sentence_cache.rb | 2 +- test/translation_diff/sentence_cache_test.rb | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/lib/translation_diff/sentence_cache.rb b/lib/translation_diff/sentence_cache.rb index 850fae4..d0b6dc9 100644 --- a/lib/translation_diff/sentence_cache.rb +++ b/lib/translation_diff/sentence_cache.rb @@ -46,7 +46,7 @@ def options_digest # Sorted on the name's string form, because a Symbol and a String key are not comparable with each other. def canonical_options - @options.sort_by { |name, _| name.to_s }.map { |name, value| "#{name}=#{stable(name, value)}" }.join("&") + @options.sort_by { |name, _| name.to_s }.map { |name, value| "#{name}=#{stable(name, value)}" }.join(",") end # A value that renders as an address makes a key nothing can ever hit twice, so say so where a caller will see it. diff --git a/test/translation_diff/sentence_cache_test.rb b/test/translation_diff/sentence_cache_test.rb index 06a5ec5..ea7d610 100644 --- a/test/translation_diff/sentence_cache_test.rb +++ b/test/translation_diff/sentence_cache_test.rb @@ -136,14 +136,20 @@ def test_the_options_digest_is_a_field_of_its_own assert_equal 5, cache(RecordingStore.new, options: { formality: :more }).key(segment).split(":").size end - # Pinned as a literal: anyone passing formality or a glossary id has to keep hitting this exact key. - def test_a_key_carrying_options_is_pinned_field_by_field - subject_cache = TranslationDiff::SentenceCache.new( - store: RecordingStore.new, provider: "null", from: "en", to: "ru", options: { formality: :more } - ) + # Recovered from the pipeline being replaced by running it against a recording store: one option, two + # options, and two whose sort order is not their literal order, which is the pair that proves the sort. + def test_the_options_digest_matches_the_keys_the_previous_pipeline_produced + segment = segments("One.").first - assert_equal "null:en:ru:c09f3c46:900019fa233e608091ba641d50d69b81", - subject_cache.key(segments("One.").first) + { { formality: :more } => "null:en:ru:c09f3c46:900019fa233e608091ba641d50d69b81", + { formality: :more, glossary_id: "g1" } => "null:en:ru:c1ee2461:900019fa233e608091ba641d50d69b81", + { b: 2, a: 1 } => "null:en:ru:9dc867b7:900019fa233e608091ba641d50d69b81" }.each do |options, expected| + subject_cache = TranslationDiff::SentenceCache.new( + store: RecordingStore.new, provider: "null", from: "en", to: "ru", options: options + ) + + assert_equal expected, subject_cache.key(segment) + end end # A Symbol and a String are not comparable with each other, so sorting on the raw keys raises on this hash. From feee312214b082f017702d1d556b4edbfb0f0a29 Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 02:26:34 +0400 Subject: [PATCH 15/21] fix: recurse into container cache options, and permit by type A Hash or Array option was rendered with #inspect, giving Ruby literal syntax where the pipeline being replaced recurses. DeepL's splitting_tags and ignore_tags are list-valued and options splat straight through translate, so those callers were missing their whole cache. Recovered the shape by capturing nine option hashes from the old pipeline through a recording store and brute-forcing 5,625 delimiter, separator and pair-joiner combinations against all nine digests at once. One reproduces them all: containers carry no delimiters, a Hash canonicalises the way the options hash does, an Array its elements in order, both joined with ",". It is lossy -- ["x", ["y", "z"]] and ["x", "y", "z"] collide -- and that is the interface, not a defect to fix here. Replace the address-sniffing denylist with an allowlist of the types the format can render. A Struct and a Set have a tidy #inspect and no address, so they were being given keys the old pipeline refuses to build, which is the silent wrong key the error exists to prevent. --- lib/translation_diff/sentence_cache.rb | 24 ++++++++++++------- test/translation_diff/sentence_cache_test.rb | 25 ++++++++++++++++++++ 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/lib/translation_diff/sentence_cache.rb b/lib/translation_diff/sentence_cache.rb index d0b6dc9..9c18207 100644 --- a/lib/translation_diff/sentence_cache.rb +++ b/lib/translation_diff/sentence_cache.rb @@ -3,8 +3,8 @@ class TranslationDiff::SentenceCache # Its own class, so rescuing an unusable option cannot also swallow a store or provider failure. class Error < TranslationDiff::Error; end - # Ruby's default rendering of an object is its address, which would move the key every process. - ADDRESS = /#<[^>]*0x\h+/ + # The value types the key format can render. An allowlist: what it cannot render must raise, not be guessed at. + SCALARS = [String, Symbol, Numeric, TrueClass, FalseClass, NilClass].freeze def initialize(store:, provider:, from:, to:, options: {}) @store = store @@ -46,14 +46,22 @@ def options_digest # Sorted on the name's string form, because a Symbol and a String key are not comparable with each other. def canonical_options - @options.sort_by { |name, _| name.to_s }.map { |name, value| "#{name}=#{stable(name, value)}" }.join(",") + @options.sort_by { |name, _| name.to_s }.map { |name, value| "#{name}=#{canonical(name, value)}" }.join(",") end - # A value that renders as an address makes a key nothing can ever hit twice, so say so where a caller will see it. - def stable(name, value) - rendered = value.inspect - raise Error, "cache option #{name} (a #{value.class}) has no stable string form" if rendered.match?(ADDRESS) + # A Hash canonicalises the way the options hash itself does, an Array its elements in order, joined the same way. + def canonical(name, value) + case value + when Hash then value.sort_by { |k, _| k.to_s }.map { |k, v| "#{k}=#{canonical(name, v)}" }.join(",") + when Array then value.map { |element| canonical(name, element) }.join(",") + else scalar(name, value) + end + end + + # A key that is silently wrong costs a caller their whole cache and tells them nothing, so refuse to build one. + def scalar(name, value) + return value.inspect if SCALARS.any? { |type| value.is_a?(type) } - rendered + raise Error, "cache option #{name} (a #{value.class}) has no stable string form" end end diff --git a/test/translation_diff/sentence_cache_test.rb b/test/translation_diff/sentence_cache_test.rb index ea7d610..c70f8bb 100644 --- a/test/translation_diff/sentence_cache_test.rb +++ b/test/translation_diff/sentence_cache_test.rb @@ -152,6 +152,31 @@ def test_the_options_digest_matches_the_keys_the_previous_pipeline_produced end end + # Container values recurse: a Hash canonicalises the way the options hash itself does, an Array in order. + # The first two were captured from the pipeline being replaced; the third goes a level deeper than either. + def test_container_option_values_recurse_the_way_the_previous_pipeline_did + segment = segments("One.").first + + { { glossary: { a: 1, b: 2 } } => "null:en:ru:4dbc310c:900019fa233e608091ba641d50d69b81", + { tags: %w[x y] } => "null:en:ru:6b1bf7d6:900019fa233e608091ba641d50d69b81", + { glossary: { a: [1, 2] } } => "null:en:ru:1948f701:900019fa233e608091ba641d50d69b81" }.each do |options, key| + subject_cache = TranslationDiff::SentenceCache.new( + store: RecordingStore.new, provider: "null", from: "en", to: "ru", options: options + ) + + assert_equal key, subject_cache.key(segment) + end + end + + # An allowlist, not a denylist: a tidy #inspect with no address in it is still one the old pipeline refused. + def test_a_value_outside_the_permitted_types_raises_even_when_its_inspect_is_stable + subject_cache = cache(RecordingStore.new, options: { glossary: Struct.new(:x).new(1) }) + + error = assert_raises(TranslationDiff::SentenceCache::Error) { subject_cache.key(segments("One.").first) } + + assert_includes error.message, "glossary" + end + # A Symbol and a String are not comparable with each other, so sorting on the raw keys raises on this hash. def test_mixed_option_key_types_canonicalise_instead_of_raising segment = segments("One.").first From 72c3a5db46c90ea0751e6e329dd9a841636f3df7 Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 02:37:29 +0400 Subject: [PATCH 16/21] feat: coordinate the pipeline without a god object --- lib/translation_diff.rb | 1 + lib/translation_diff/translator.rb | 128 +++++++++++ test/translation_diff/translator_test.rb | 280 +++++++++++++++++++++++ 3 files changed, 409 insertions(+) create mode 100644 lib/translation_diff/translator.rb create mode 100644 test/translation_diff/translator_test.rb diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index d464a3b..2cf1e75 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -48,6 +48,7 @@ require "translation_diff/redis_cache_store" require "translation_diff/redis_rate_limiter" require "translation_diff/instrumentation" +require "translation_diff/translator" require "translation_diff/request" require "translation_diff/context" diff --git a/lib/translation_diff/translator.rb b/lib/translation_diff/translator.rb new file mode 100644 index 0000000..e9941ff --- /dev/null +++ b/lib/translation_diff/translator.rb @@ -0,0 +1,128 @@ +# One translation, coordinated: each step hands its value to the next, and a translation rides home on its own segment. +class TranslationDiff::Translator + # Its own class, so rescuing a mis-driven translator cannot also swallow a provider or a cache failure. + class Error < TranslationDiff::Error; end + + include TranslationDiff::Instrumentation + + attr_reader :config + + # `provider:` and `config:` are reserved; every other keyword is forwarded to the provider untouched. + def initialize(values, from: nil, to: nil, provider: nil, config: nil, **options) + @values = values + @from = from + @to = to + @options = options + @config = config || TranslationDiff.config + @provider = resolve(provider) + @name = provider_name(@provider) + log("provider #{@provider.class}") + end + + # Hands back the caller's value untouched unless something in it was actually translated. + def call + document = TranslationDiff::Document.new(@values) + passages = document.strings.map { |string| passage(string) } + segments = passages.flat_map(&:segments).reject(&:empty?) + return @values if segments.empty? + + from = source_language(segments) + return @values if same_language?(from) + + instrument("translate", from: from, to: @to, provider: @name, values: passages.size) do + fill(segments, from) + rebuild(document, passages) + end + end + + private + + # A provider arrives as a name to build, as an object to use as it is, or not at all -- then it is the configured one. + def resolve(provider) + return config.provider_instance if provider.nil? + return TranslationDiff::Providers.build(provider, config) if provider.is_a?(Symbol) || provider.is_a?(String) + + TranslationDiff::Providers.ensure_provider!(provider) + end + + # The cache key names the provider in every payload too: it is the one identifier every provider must have. + def provider_name(provider) + key = provider.cache_key.to_s + return key unless key.strip.empty? + + raise Error, + "#{provider.class} must define #cache_key: a blank one would file its translations " \ + "in every other provider's cache namespace." + end + + def passage(string) + TranslationDiff::Passage.new(string, segmenter: config.segmenter_instance, language: @from) + end + + # The strings walk and the map walk visit the same leaves in the same order, and the value itself was never touched. + def rebuild(document, passages) + rendered = passages.map(&:render) + document.map { rendered.shift } + end + + # Detection is attempted only where it is declared: every provider inherits a #detect that raises. + def source_language(segments) + return @from unless @from.nil? + + ensure_detects_language! + @provider.detect(segments.first.core) + end + + def ensure_detects_language! + return if @provider.class.capabilities.detects_language? + + raise Error, + "Provider #{@name} cannot detect the source language. Pass `from:` with the " \ + "source language code of the values you are translating." + end + + # A detected language arrives as a String while `to:` is usually a Symbol, so neither type nor case can be assumed. + def same_language?(from) = from.to_s.casecmp?(@to.to_s) + + # The cache answers for what it has, the provider for the rest, and only what came back is written home. + def fill(segments, from) + cache = sentence_cache(from) + misses = cache.fill(segments) + instrument("cache", provider: @name, hits: segments.size - misses.size, misses: misses.size) + dispatch(misses, from) + cache.store(misses) + end + + def sentence_cache(from) + TranslationDiff::SentenceCache.new(store: config.cache_store, provider: @name, + from: from, to: @to, options: @options) + end + + def dispatch(segments, from) + capabilities = @provider.class.capabilities + TranslationDiff::Batch.pack(segments, capabilities: capabilities).each { |batch| send_batch(batch, from) } + end + + # The batch applies the reply to the segments that produced it, so no step ever correlates by position again. + def send_batch(batch, from) + texts = batch.texts + characters = texts.sum(&:size) + throttle(characters) + response = instrument("request", provider: @name, batch: texts.size, characters: characters) do + @provider.translate(request(texts, from)) + end + batch.apply(response.texts) + end + + def request(texts, from) + TranslationDiff::Translation::Request.new(texts: texts, from: from, to: @to, options: @options) + end + + # Consulted with what is about to be sent, before it is sent; nil means no rate limiting was configured at all. + def throttle(characters) + limiter = config.rate_limiter_instance + return if limiter.nil? + + instrument("rate_limit", provider: @name, characters: characters) { limiter.check(characters) } + end +end diff --git a/test/translation_diff/translator_test.rb b/test/translation_diff/translator_test.rb new file mode 100644 index 0000000..5bd270e --- /dev/null +++ b/test/translation_diff/translator_test.rb @@ -0,0 +1,280 @@ +require "test_helper" + +class TranslatorTest < ConfiguredTest + class RecordingProvider < TranslationDiff::Provider + def self.capabilities + TranslationDiff::Capabilities.new( + max_request_size: 1_000, max_batch_size: 10, max_text_size: nil, + html: :none, notranslate: false, detects_language: true, reports_billing: false + ) + end + + attr_reader :requests + + def initialize(config) + super + @requests = [] + end + + def translate(request) + @requests << request + TranslationDiff::Translation::Response.build( + request: request, texts: request.texts.map(&:upcase) + ) + end + + def detect(_text) = "en" + def cache_key = "recording" + end + + # The capability is the only honest test: every provider inherits a #detect that raises. + class BlindProvider < RecordingProvider + def self.capabilities + TranslationDiff::Capabilities.new( + max_request_size: 1_000, max_batch_size: 10, max_text_size: nil, + html: :none, notranslate: false, detects_language: false, reports_billing: false + ) + end + + def cache_key = "blind" + end + + # An empty cache key would file this provider's translations in every other provider's namespace. + class NamelessProvider < RecordingProvider + def cache_key = " " + end + + class Recorder + attr_reader :events + + def initialize = @events = [] + + def instrument(name, payload) + @events << [name, payload] + yield if block_given? + end + end + + # Always lets the call through, so the `rate_limit` event fires without a real Redis connection. + class FakeRateLimiter + attr_reader :sizes + + def initialize = @sizes = [] + + def check(size) = @sizes << size + end + + def setup + super + @provider = RecordingProvider.new(TranslationDiff::Configuration.new) + # Pinned so a developer with REDIS_URL set doesn't have these tests open a real socket. + TranslationDiff.configure { |c| c.cache = :memory } + end + + def translate(values, **) + TranslationDiff::Translator.new(values, provider: @provider, **).call + end + + def test_it_translates_a_nested_structure_and_keeps_its_shape + result = translate({ title: "one.", body: ["two.", 42] }, from: "en", to: "ru") + + assert_equal({ title: "ONE.", body: ["TWO.", 42] }, result) + end + + def test_the_same_language_never_reaches_the_provider + assert_equal "one.", translate("one.", from: "en", to: "en") + assert_empty @provider.requests + end + + def test_the_same_language_is_compared_across_string_and_symbol + assert_equal "one.", translate("one.", from: "EN", to: :en) + assert_empty @provider.requests + end + + def test_a_value_with_nothing_to_translate_never_reaches_the_provider + assert_equal "", translate("", from: "en", to: "ru") + assert_nil translate(nil, from: "en", to: "ru") + assert_empty @provider.requests + end + + def test_a_missing_source_language_is_detected + translate("one.", to: "ru") + + assert_equal "en", @provider.requests.first.from + end + + def test_per_call_options_reach_the_provider_untouched + translate("one.", from: "en", to: "ru", formality: :less) + + assert_equal({ formality: :less }, @provider.requests.first.options) + end + + def test_a_second_translation_of_the_same_sentence_is_served_from_cache + translate("one.", from: "en", to: "ru") + translate("one.", from: "en", to: "ru") + + assert_equal 1, @provider.requests.size + end + + def test_only_the_missing_sentences_are_sent + translate("one. two.", from: "en", to: "ru") + translate("one. three.", from: "en", to: "ru") + + assert_equal [%w[one. two.], %w[three.]], @provider.requests.map(&:texts) + end + + # The old check was `respond_to?(:detect)`, satisfied by inheriting the base class's raising stub. + def test_a_provider_that_cannot_detect_says_so_before_it_is_called + @provider = BlindProvider.new(TranslationDiff::Configuration.new) + + error = assert_raises(TranslationDiff::Translator::Error) { translate("one.", to: "ru") } + + assert_match(/cannot detect/, error.message) + assert_match(/blind/, error.message) + assert_empty @provider.requests + end + + def test_a_provider_whose_cache_key_is_blank_is_refused_rather_than_sharing_a_namespace + @provider = NamelessProvider.new(TranslationDiff::Configuration.new) + + error = assert_raises(TranslationDiff::Translator::Error) { translate("one.", from: "en", to: "ru") } + + assert_match(/must define #cache_key/, error.message) + end + + def test_a_provider_named_for_one_call_overrides_the_configured_one + configured = RecordingProvider.new(TranslationDiff::Configuration.new) + TranslationDiff.configure { |c| c.provider = configured } + + assert_equal "ONE.", translate("one.", from: "en", to: "ru") + assert_empty configured.requests + end + + def test_without_a_provider_keyword_the_configured_provider_is_used + TranslationDiff.configure { |c| c.provider = @provider } + + assert_equal "ONE.", TranslationDiff::Translator.new("one.", from: "en", to: "ru").call + assert_equal 1, @provider.requests.size + end + + def test_an_object_that_is_not_a_provider_is_refused + assert_raises(TranslationDiff::InvalidProviderError) do + TranslationDiff::Translator.new("one.", from: "en", to: "ru", provider: Object.new).call + end + end + + def test_a_translation_emits_translate_cache_request_and_rate_limit_events + recorder = instrumented { |c| c.rate_limiter = FakeRateLimiter.new } + instrumented_translate("Hello there.") + + assert_equal ALL_EVENT_NAMES, recorder.events.map(&:first).sort + end + + def test_the_translate_event_carries_languages_provider_and_a_count + recorder = instrumented + instrumented_translate(%w[one two]) + + payload = payload_for(recorder, "translate") + + assert_equal "en", payload[:from] + assert_equal "ru", payload[:to] + assert_equal "recording", payload[:provider] + assert_equal 2, payload[:values] + end + + def test_the_cache_event_carries_hit_and_miss_counts + recorder = instrumented + instrumented_translate("Hello there.") + + payload = payload_for(recorder, "cache") + + assert_equal 0, payload[:hits] + assert_equal 1, payload[:misses] + assert_equal "recording", payload[:provider] + end + + def test_the_request_event_carries_the_provider_a_batch_size_and_a_character_count + recorder = instrumented + instrumented_translate("Hello there.") + + payload = payload_for(recorder, "request") + + assert_equal "recording", payload[:provider] + assert_equal 1, payload[:batch] + assert_equal "Hello there.".size, payload[:characters] + end + + def test_the_rate_limit_event_carries_the_provider_and_a_character_count + limiter = FakeRateLimiter.new + recorder = instrumented { |c| c.rate_limiter = limiter } + instrumented_translate("Hello there.") + + payload = payload_for(recorder, "rate_limit") + + assert_equal "recording", payload[:provider] + assert_equal "Hello there.".size, payload[:characters] + assert_equal ["Hello there.".size], limiter.sizes + end + + # The limiter is consulted with what is about to be sent, before it is sent. + def test_the_rate_limiter_is_consulted_before_the_provider + limiter = FakeRateLimiter.new + recorder = instrumented { |c| c.rate_limiter = limiter } + instrumented_translate("Hello there.") + + names = recorder.events.map(&:first).select { |name| name.start_with?("rate_limit", "request") } + + assert_equal %w[rate_limit.translation_diff request.translation_diff], names + end + + def test_no_rate_limit_event_without_a_rate_limiter + recorder = instrumented + instrumented_translate("Hello there.") + + refute_includes recorder.events.map(&:first), "rate_limit.translation_diff" + end + + # A call that returns early reaches no provider and so reports nothing. + def test_a_call_that_returns_early_emits_no_events + recorder = instrumented + instrumented_translate("Hello there.", to: "en") + + assert_empty recorder.events + end + + ALL_EVENT_NAMES = %w[translate.translation_diff cache.translation_diff + request.translation_diff rate_limit.translation_diff].sort.freeze + + # A guard that only checked payload content would pass even if an event quietly stopped firing. + def test_no_payload_ever_contains_the_text_being_translated + secret = "Zaphod Beeblebrox is president." + recorder = instrumented { |c| c.rate_limiter = FakeRateLimiter.new } + instrumented_translate(secret) + + assert_equal ALL_EVENT_NAMES, recorder.events.map(&:first).sort + + serialised = recorder.events.map { |name, payload| "#{name}#{payload}" }.join + refute_includes serialised, "Zaphod" + refute_includes serialised, secret + refute_includes serialised, secret.upcase + end + + private + + def instrumented + Recorder.new.tap do |recorder| + TranslationDiff.configure do |c| + c.instrumenter = recorder + yield c if block_given? + end + end + end + + def instrumented_translate(values, from: "en", to: "ru") + TranslationDiff::Translator.new(values, from: from, to: to, provider: @provider).call + end + + def payload_for(recorder, event) + recorder.events.find { |name, _| name == "#{event}.translation_diff" }.last + end +end From b6251ffdd8f5801673aca5fbaed7ca309df6fb54 Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 02:49:52 +0400 Subject: [PATCH 17/21] fix: keep the pipeline's answers for nil, payloads and lazy resolution --- lib/translation_diff.rb | 1 + lib/translation_diff/leaves.rb | 22 ++++++ lib/translation_diff/translator.rb | 98 +++++++++++++----------- test/translation_diff/leaves_test.rb | 39 ++++++++++ test/translation_diff/translator_test.rb | 66 ++++++++++++++++ 5 files changed, 183 insertions(+), 43 deletions(-) create mode 100644 lib/translation_diff/leaves.rb create mode 100644 test/translation_diff/leaves_test.rb diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index 2cf1e75..14d12ef 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -14,6 +14,7 @@ require "translation_diff/translation/response" require "translation_diff/registry" require "translation_diff/document" +require "translation_diff/leaves" require "translation_diff/markup" require "translation_diff/segment" require "translation_diff/batch" diff --git a/lib/translation_diff/leaves.rb b/lib/translation_diff/leaves.rb new file mode 100644 index 0000000..de6d191 --- /dev/null +++ b/lib/translation_diff/leaves.rb @@ -0,0 +1,22 @@ +# The two things this pipeline has always promised about a caller's structure that Document deliberately does not: +# a nested nil comes back as the empty string, and the size of a document is every leaf in it, translatable or not. +module TranslationDiff::Leaves + # Document hands every non-String leaf straight back, which is the honest behaviour for a general structural map. + # Answering a nested nil with "" is the pipeline's own promise, so it is made here rather than there. + def self.collapse_nils(node) + case node + when Hash then node.to_h { |key, value| [key, collapse_nils(value)] } + when Array then node.map { |value| collapse_nils(value) } + when nil then "" + else node + end + end + + # Every leaf the caller wrote, whatever its type: the number a `translate` payload reports as `values`. + def self.count(node) + return node.each_value.sum { |value| count(value) } if node.is_a?(Hash) + return node.sum { |value| count(value) } if node.is_a?(Array) + + 1 + end +end diff --git a/lib/translation_diff/translator.rb b/lib/translation_diff/translator.rb index e9941ff..63b11d0 100644 --- a/lib/translation_diff/translator.rb +++ b/lib/translation_diff/translator.rb @@ -9,50 +9,65 @@ class Error < TranslationDiff::Error; end # `provider:` and `config:` are reserved; every other keyword is forwarded to the provider untouched. def initialize(values, from: nil, to: nil, provider: nil, config: nil, **options) + raise ArgumentError, "a translation needs a target language: pass `to:` a language code." if to.nil? + @values = values @from = from @to = to @options = options @config = config || TranslationDiff.config - @provider = resolve(provider) - @name = provider_name(@provider) - log("provider #{@provider.class}") + @requested_provider = provider end # Hands back the caller's value untouched unless something in it was actually translated. def call - document = TranslationDiff::Document.new(@values) + document = TranslationDiff::Document.new(TranslationDiff::Leaves.collapse_nils(@values)) passages = document.strings.map { |string| passage(string) } segments = passages.flat_map(&:segments).reject(&:empty?) return @values if segments.empty? - from = source_language(segments) + provider = resolve_provider + from = source_language(provider, segments) return @values if same_language?(from) - instrument("translate", from: from, to: @to, provider: @name, values: passages.size) do - fill(segments, from) + translated(document, passages, segments, provider, from) + end + + private + + # The `translate` event wraps everything a call that reaches a provider does, and nothing an early return does. + def translated(document, passages, segments, provider, from) + values = TranslationDiff::Leaves.count(@values) + payload = { from: from.to_s, to: @to.to_s, provider: provider.cache_key, values: values } + instrument("translate", payload) do + fill(provider, segments, from) rebuild(document, passages) end end - private + # Resolved at first use, never in the constructor: a value with nothing to translate needs no provider at all. + def resolve_provider + build_provider.tap do |provider| + log("provider #{provider.class}") + ensure_cache_key!(provider) + end + end # A provider arrives as a name to build, as an object to use as it is, or not at all -- then it is the configured one. - def resolve(provider) - return config.provider_instance if provider.nil? - return TranslationDiff::Providers.build(provider, config) if provider.is_a?(Symbol) || provider.is_a?(String) + def build_provider + requested = @requested_provider + return config.provider_instance if requested.nil? + return TranslationDiff::Providers.build(requested, config) if requested.is_a?(Symbol) || requested.is_a?(String) - TranslationDiff::Providers.ensure_provider!(provider) + TranslationDiff::Providers.ensure_provider!(requested) end # The cache key names the provider in every payload too: it is the one identifier every provider must have. - def provider_name(provider) - key = provider.cache_key.to_s - return key unless key.strip.empty? + def ensure_cache_key!(provider) + return unless provider.cache_key.to_s.strip.empty? - raise Error, - "#{provider.class} must define #cache_key: a blank one would file its translations " \ - "in every other provider's cache namespace." + raise Error, "#{provider.class} must define #cache_key: a blank one would file its " \ + "translations in every other provider's cache namespace." end def passage(string) @@ -66,51 +81,48 @@ def rebuild(document, passages) end # Detection is attempted only where it is declared: every provider inherits a #detect that raises. - def source_language(segments) + def source_language(provider, segments) return @from unless @from.nil? - ensure_detects_language! - @provider.detect(segments.first.core) + ensure_detects_language!(provider) + provider.detect(segments.first.core) end - def ensure_detects_language! - return if @provider.class.capabilities.detects_language? + def ensure_detects_language!(provider) + return if provider.class.capabilities.detects_language? - raise Error, - "Provider #{@name} cannot detect the source language. Pass `from:` with the " \ - "source language code of the values you are translating." + raise Error, "Provider #{provider.cache_key} cannot detect the source language. Pass " \ + "`from:` with the source language code of the values you are translating." end # A detected language arrives as a String while `to:` is usually a Symbol, so neither type nor case can be assumed. def same_language?(from) = from.to_s.casecmp?(@to.to_s) # The cache answers for what it has, the provider for the rest, and only what came back is written home. - def fill(segments, from) - cache = sentence_cache(from) + def fill(provider, segments, from) + cache = sentence_cache(provider, from) misses = cache.fill(segments) - instrument("cache", provider: @name, hits: segments.size - misses.size, misses: misses.size) - dispatch(misses, from) + instrument("cache", provider: provider.cache_key, hits: segments.size - misses.size, misses: misses.size) + dispatch(provider, misses, from) cache.store(misses) end - def sentence_cache(from) - TranslationDiff::SentenceCache.new(store: config.cache_store, provider: @name, + def sentence_cache(provider, from) + TranslationDiff::SentenceCache.new(store: config.cache_store, provider: provider.cache_key, from: from, to: @to, options: @options) end - def dispatch(segments, from) - capabilities = @provider.class.capabilities - TranslationDiff::Batch.pack(segments, capabilities: capabilities).each { |batch| send_batch(batch, from) } + def dispatch(provider, segments, from) + batches = TranslationDiff::Batch.pack(segments, capabilities: provider.class.capabilities) + batches.each { |batch| send_batch(provider, batch, from) } end # The batch applies the reply to the segments that produced it, so no step ever correlates by position again. - def send_batch(batch, from) + def send_batch(provider, batch, from) texts = batch.texts - characters = texts.sum(&:size) - throttle(characters) - response = instrument("request", provider: @name, batch: texts.size, characters: characters) do - @provider.translate(request(texts, from)) - end + payload = { provider: provider.cache_key, batch: texts.size, characters: texts.sum(&:size) } + throttle(provider, payload[:characters]) + response = instrument("request", payload) { provider.translate(request(texts, from)) } batch.apply(response.texts) end @@ -119,10 +131,10 @@ def request(texts, from) end # Consulted with what is about to be sent, before it is sent; nil means no rate limiting was configured at all. - def throttle(characters) + def throttle(provider, characters) limiter = config.rate_limiter_instance return if limiter.nil? - instrument("rate_limit", provider: @name, characters: characters) { limiter.check(characters) } + instrument("rate_limit", provider: provider.cache_key, characters: characters) { limiter.check(characters) } end end diff --git a/test/translation_diff/leaves_test.rb b/test/translation_diff/leaves_test.rb new file mode 100644 index 0000000..13f8f73 --- /dev/null +++ b/test/translation_diff/leaves_test.rb @@ -0,0 +1,39 @@ +require "test_helper" + +class LeavesTest < Minitest::Test + def collapse(value) = TranslationDiff::Leaves.collapse_nils(value) + + def count(value) = TranslationDiff::Leaves.count(value) + + def test_a_nested_nil_becomes_an_empty_string + assert_equal({ a: "one", skip: "" }, collapse({ a: "one", skip: nil })) + assert_equal ["one", "", 42], collapse(["one", nil, 42]) + end + + def test_a_bare_nil_becomes_an_empty_string_too + assert_equal "", collapse(nil) + end + + def test_every_other_leaf_is_handed_back_as_it_is + assert_equal({ a: "one", n: 42, f: false }, collapse({ a: "one", n: 42, f: false })) + end + + def test_the_caller_s_own_structure_is_never_modified + value = { a: "one", nested: ["two", nil] } + collapse(value) + + assert_equal({ a: "one", nested: ["two", nil] }, value) + end + + def test_counting_covers_every_leaf_whatever_its_type + assert_equal 3, count({ a: "one", n: 42, skip: nil }) + assert_equal 5, count({ a: "one", b: { c: "two", d: [1, "three", nil] } }) + end + + def test_a_scalar_is_one_leaf_and_an_empty_container_is_none + assert_equal 1, count("one") + assert_equal 1, count(nil) + assert_equal 0, count([]) + assert_equal 0, count({}) + end +end diff --git a/test/translation_diff/translator_test.rb b/test/translation_diff/translator_test.rb index 5bd270e..678867a 100644 --- a/test/translation_diff/translator_test.rb +++ b/test/translation_diff/translator_test.rb @@ -44,6 +44,14 @@ class NamelessProvider < RecordingProvider def cache_key = " " end + class FakeLogger + attr_reader :lines + + def initialize = @lines = [] + + def debug(&) = @lines << yield + end + class Recorder attr_reader :events @@ -259,6 +267,64 @@ def test_no_payload_ever_contains_the_text_being_translated refute_includes serialised, secret.upcase end + # Captured from the old pipeline: a nested nil has always come back as "" from a call that reached a provider. + def test_a_nested_nil_collapses_to_an_empty_string + assert_equal({ a: "ONE.", n: 42, skip: "" }, + translate({ a: "one.", n: 42, skip: nil }, from: "en", to: "ru")) + assert_equal ["ONE.", "", 42], translate(["one.", nil, 42], from: "en", to: "ru") + end + + # And only then: a call with nothing to translate hands the caller's value back exactly as it was given. + def test_a_nil_survives_a_call_that_translates_nothing + assert_nil translate(nil, from: "en", to: "ru") + assert_equal [nil], translate([nil], from: "en", to: "ru") + assert_equal({ a: nil }, translate({ a: nil }, from: "en", to: "ru")) + end + + # A subscriber grouping by payload[:to] must not see :ru and "ru" as two different series. + def test_the_translate_event_reports_languages_as_strings_whatever_the_caller_passed + recorder = instrumented + TranslationDiff::Translator.new("Hello there.", from: :en, to: :ru, provider: @provider).call + + payload = payload_for(recorder, "translate") + + assert_equal "en", payload[:from] + assert_equal "ru", payload[:to] + end + + # `values` is the size of the document as the caller wrote it, not the number of translatable strings in it. + def test_the_translate_event_counts_every_leaf_the_caller_wrote + recorder = instrumented + instrumented_translate({ a: "one.", n: 42, skip: nil }) + + assert_equal 3, payload_for(recorder, "translate")[:values] + end + + def test_a_value_with_nothing_to_translate_never_resolves_a_provider + assert_equal 42, TranslationDiff::Translator.new(42, from: "en", to: "ru", provider: Object.new).call + end + + def test_the_provider_is_logged_once_and_only_when_it_is_resolved + logger = FakeLogger.new + TranslationDiff.configure { |c| c.logger = logger } + + translate(42, from: "en", to: "ru") + + assert_empty logger.lines + + translate("one.", from: "en", to: "ru") + + assert_equal 1, logger.lines.size + assert_match(/RecordingProvider/, logger.lines.first) + end + + # A nil target used to reach the cache key and die there on #downcase; it is a caller's mistake, not a defect. + def test_a_missing_target_language_is_refused_by_name + error = assert_raises(ArgumentError) { translate("one.", from: "en") } + + assert_match(/to:/, error.message) + end + private def instrumented From 1ca58ff26d631c7be26a7e6bf6cda3fba90f7b6e Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 02:56:38 +0400 Subject: [PATCH 18/21] fix: settle a given source language before resolving a provider --- lib/translation_diff/translator.rb | 2 ++ test/translation_diff/translator_test.rb | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/lib/translation_diff/translator.rb b/lib/translation_diff/translator.rb index 63b11d0..5d7bdb7 100644 --- a/lib/translation_diff/translator.rb +++ b/lib/translation_diff/translator.rb @@ -25,6 +25,7 @@ def call passages = document.strings.map { |string| passage(string) } segments = passages.flat_map(&:segments).reject(&:empty?) return @values if segments.empty? + return @values if same_language?(@from) provider = resolve_provider from = source_language(provider, segments) @@ -96,6 +97,7 @@ def ensure_detects_language!(provider) end # A detected language arrives as a String while `to:` is usually a Symbol, so neither type nor case can be assumed. + # A `from:` the caller gave settles this before a provider is resolved; a nil one cannot, and never matches. def same_language?(from) = from.to_s.casecmp?(@to.to_s) # The cache answers for what it has, the provider for the rest, and only what came back is written home. diff --git a/test/translation_diff/translator_test.rb b/test/translation_diff/translator_test.rb index 678867a..e84fbab 100644 --- a/test/translation_diff/translator_test.rb +++ b/test/translation_diff/translator_test.rb @@ -304,6 +304,21 @@ def test_a_value_with_nothing_to_translate_never_resolves_a_provider assert_equal 42, TranslationDiff::Translator.new(42, from: "en", to: "ru", provider: Object.new).call end + # A call that hands the caller's value straight back should not need a provider to do it: an app that has + # configured none at all still gets its value. Configured with something that is not a provider rather than + # left unset, so resolving one raises whatever the developer has in their environment. + def test_the_same_language_does_not_even_resolve_a_provider + TranslationDiff.configure { |c| c.provider = Object.new } + + assert_equal "Hello", TranslationDiff::Translator.new("Hello", from: :ru, to: :ru).call + end + + def test_a_value_with_nothing_to_translate_does_not_resolve_the_configured_provider_either + TranslationDiff.configure { |c| c.provider = Object.new } + + assert_equal 42, TranslationDiff::Translator.new(42, from: "en", to: "ru").call + end + def test_the_provider_is_logged_once_and_only_when_it_is_resolved logger = FakeLogger.new TranslationDiff.configure { |c| c.logger = logger } From a69abee94b8f230e72588b4cff6c8de6df0c95fc Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 03:08:47 +0400 Subject: [PATCH 19/21] refactor!: replace the pipeline with a design of our own TranslationDiff.translate and Context#translate now build a Translator. Linearizer, Spacing, Chunker, Tokenizer, Cache and Request are deleted, along with their tests: they derived from google_translate_diff, which carries no licence at all, and this gem ships under MIT. The replacement -- Document, Leaves, Passage, Fragment, Segment, Markup, Batch, SentenceCache and Translator -- was written from tests and a spec without reading them. Cache keys are unmoved except where the CHANGELOG says otherwise, pinned by test against values recorded from the pipeline being replaced. The README's attribution to GoogleTranslateDiff goes with this commit and no other: it was true while the code was derived, and stops being true here. --- CHANGELOG.md | 73 +++- README.md | 2 - docs/caching.md | 29 ++ docs/errors.md | 23 +- docs/how-it-works.md | 68 +++- docs/instrumentation.md | 13 +- docs/providers.md | 2 +- lib/translation_diff.rb | 8 +- lib/translation_diff/cache.rb | 69 ---- lib/translation_diff/chunker.rb | 57 --- lib/translation_diff/context.rb | 2 +- lib/translation_diff/linearizer.rb | 27 -- lib/translation_diff/request.rb | 198 ---------- lib/translation_diff/spacing.rb | 29 -- lib/translation_diff/tokenizer.rb | 163 --------- test/translation_diff/cache_test.rb | 148 -------- test/translation_diff/chunker_test.rb | 64 ---- test/translation_diff/context_test.rb | 8 + test/translation_diff/linearizer_test.rb | 17 - test/translation_diff/pipeline_corpus_test.rb | 16 +- test/translation_diff/request_test.rb | 343 ------------------ test/translation_diff/spacing_test.rb | 11 - test/translation_diff/tokenizer_test.rb | 156 -------- .../translation/response_test.rb | 2 +- test/translation_diff_test.rb | 32 ++ 25 files changed, 234 insertions(+), 1326 deletions(-) delete mode 100644 lib/translation_diff/cache.rb delete mode 100644 lib/translation_diff/chunker.rb delete mode 100644 lib/translation_diff/linearizer.rb delete mode 100644 lib/translation_diff/request.rb delete mode 100644 lib/translation_diff/spacing.rb delete mode 100644 lib/translation_diff/tokenizer.rb delete mode 100644 test/translation_diff/cache_test.rb delete mode 100644 test/translation_diff/chunker_test.rb delete mode 100644 test/translation_diff/linearizer_test.rb delete mode 100644 test/translation_diff/request_test.rb delete mode 100644 test/translation_diff/spacing_test.rb delete mode 100644 test/translation_diff/tokenizer_test.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index c15d19e..5b08ce8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,15 +106,53 @@ described below. Everything here is relative to `deepl_diff` 2.2.0. configuration still setting `deepl_host` raises `NoMethodError` on `TranslationDiff.configure`. Rename it. - A provider returning the wrong number of translations now raises - `TranslationDiff::ResponseError`, not `TranslationDiff::Request::Error`. - `Request::Error` still exists, and still means "`from:` is missing and the - provider cannot detect"; a `rescue TranslationDiff::Request::Error` written - to catch a short response no longer catches one. Both are - `TranslationDiff::Error`, so a rescue of the base class is unaffected. + `TranslationDiff::ResponseError`, not the error that used to live on + `Request`; a `rescue` written to catch a short response that way no longer + catches one. Both are `TranslationDiff::Error`, so a rescue of the base + class is unaffected. - A provider returning a well-formed response that carries no translation for one input -- Azure answers 200 for a batch where a single string failed -- also raises `TranslationDiff::ResponseError`, naming the position. It - previously reached `Spacing.restore` and died there as `NoMethodError`. + previously reached the spacing step and died there as `NoMethodError`. +- **The translation pipeline is new code.** `TranslationDiff::Linearizer`, + `Spacing`, `Chunker`, `Tokenizer`, `Cache` and `Request` are gone as public + constants. What replaces them: `Document` and `Leaves` (walking the + caller's structure), `Passage`, `Fragment` and `Segment` (markup and prose, + cut into sentences), `Markup` (entity references and a `<` that opens no + tag), `Batch` (packing sentences into provider requests), `SentenceCache` + (the cache key, read and write) and `Translator` (the coordinator + `TranslationDiff.translate` and `Context#translate` now build). See + [How it works](docs/how-it-works.md). If you referenced any of the six by + name, that reference is now a `NameError`. +- `TranslationDiff::Request::Error` is now `TranslationDiff::Translator::Error` + and `TranslationDiff::Cache::Error` is now + `TranslationDiff::SentenceCache::Error`. There is no alias for either: this + gem has never been published under the name `translation_diff` with those + constants in it. `TranslationDiff::Chunker::Error` is gone with no + replacement -- a single sentence too large to send now raises + `TranslationDiff::Error` from `Batch`. All three remain + `TranslationDiff::Error`, so a rescue of the base class is unaffected. +- **`TranslationDiff.translate` and `Context#translate` raise `ArgumentError` + when `to:` is missing or `nil`.** The keyword still defaults to `nil` in the + signature, and the message names it. Previously a `nil` target compared + equal to a `nil` source, the call short-circuited as "same language" and + your values came back untranslated, silently. If you have a caller reading + `to:` out of a configuration that can be blank, it has been a no-op and will + now raise. +- **The `cache` instrumentation event fires once per `translate` call, not + once per chunk.** The cache is now consulted for every sentence in one + `read_multi` before anything is batched. `hits` and `misses` still sum to + the same totals over a call, so a counter that adds them up is unaffected; + a counter of *events*, or a histogram of per-chunk hit ratios, will see the + cardinality drop. `request` and `rate_limit` still fire once per batch sent. +- **Two more cache keys move, beyond the entity and `<` fixes below.** A + sentence padded with Unicode whitespace -- a non-breaking space, say -- now + keys as the bare sentence: the pipeline uses one Unicode-aware definition + of padding everywhere, where the key used to be built with ASCII `strip`, + which leaves a `U+00A0` in place. And the whole document key format is + otherwise unmoved: it is pinned by test against recorded values, and every + other input in the corpus this rewrite was judged against produces the same + key it did before. ### Removed @@ -297,6 +335,29 @@ described below. Everything here is relative to `deepl_diff` 2.2.0. - DeepL's batch limit was declared as 300 sentences per request; DeepL documents 50. The request-size limit (1,700 escaped characters) was already correct and is unchanged. +- **An entity reference no longer reaches the provider raw.** `Salt & + pepper.` was sent to the provider as the six characters `&`, so the + provider translated the entity's spelling as if it were words -- and was + billed for it. It is now sent as `Salt & pepper.`, the text the document + actually says, and re-encoded on the way out. Named entities, `&` and + `&` alike are decoded; anything neither decoder knows is left as it + arrived. +- **A bare `<` no longer swallows the rest of the sentence.** `if a < b then + stop. Fine.` was parsed by `ox` as prose followed by an unclosed tag, so + only `if a` was ever sent for translation and everything after the `<` came + back untranslated. A `<` that no element name, closing name, declaration or + instruction follows is now escaped before parsing and restored after, so the + whole sentence is translated. `5 < 6 and 7 > 6. True.` was sent as three + fragments and is now sent as two sentences. +- **These two fixes move the cache key for the documents they affect.** A + document containing an entity reference, or a `<` that opens no tag, will + miss the cache once and be re-translated. That is the point: what was cached + for it was translated from the wrong text. +- **A known remaining limit: `<` still reaches a provider undecoded**, and + `a ` and + `", - [ - ["аль", :text], - ["", :markup], - ["бра", :text], - ["", :markup], - ["кил", :text], - ["", :markup] - ] - ], - "text_split_into_sentences" => [ - "! Киловольт. Смеркалось. Ворчало. Кричало.", - [ - # A lone terminator with no preceding content stays merged: a missed boundary, not a false one. - ["! Киловольт. ", :text], - ["", :markup], - ["Смеркалось. ", :text], - ["Ворчало. ", :text], - ["Кричало.", :text], - ["", :markup] - ] - ], - "notranslate_spans_kept_as_text" => [ - "test\nxy", - [ - ["test", :text], - ["", :markup], - ["\nxy", :text], - ["", :markup] - ] - ], - "a_notranslate_span_inside_another_span" => [ - "foobar
baz", - [ - ["", :markup], - ["foobar
baz", :text], - ["", :markup] - ] - ], - "a_notranslate_span_inside_another_notranslate_span" => [ - NESTED_NOTRANSLATE, - [[NESTED_NOTRANSLATE, :text]] - ], - "a_br_tag_before_a_closing_tag" => [ - "Смеркалось.
", - [ - ["", :markup], - ["Смеркалось.", :text], - ["
", :markup] - ] - ], - "a_processing_instruction" => [ - "Hey!
Look!", - [ - ["Hey!", :text], - ["
", :markup], - ["Look!", :text], - ["", :markup] - ] - ], - # Without a handler, the bytes a comment/doctype/CDATA event covers vanish from the rebuilt string. - "an_html_comment" => [ - " Visible text.", - [ - ["", :markup], - [" Visible text.", :text] - ] - ], - "a_doctype" => [ - "

Body text.

", - [ - ["

", :markup], - ["Body text.", :text], - ["

", :markup] - ] - ], - "a_cdata_section" => [ - "Before.After.", - [ - ["Before.", :text], - ["", :markup], - ["After.", :text] - ] - ], - "a_comment_between_two_sentences" => [ - "First sentence. Second sentence.", - [ - ["First sentence. ", :text], - ["", :markup], - [" Second sentence.", :text] - ] - ], - "sentences_separated_by_blank_lines" => [ - "Набор «Солнечная механика» от 4М — это 6 экспериментов." \ - "\n\nЮному изобретателю предстоит воочию посмотреть на чудеса.", - [ - ["Набор «Солнечная механика» от 4М — это 6 экспериментов.\n\n", :text], - ["Юному изобретателю предстоит воочию посмотреть на чудеса.", :text] - ] - ] - }.freeze - - CASES.each do |name, (source, expected)| - define_method(:"test_tokenizes_#{name}") do - assert_equal expected, TranslationDiff::Tokenizer.tokenize(source, segmenter: segmenter) - end - end - - private - - # Passed explicitly now that the tokenizer takes its segmenter as a collaborator, not a global. - def segmenter = TranslationDiff::Segmenters::Pragmatic.new -end diff --git a/test/translation_diff/translation/response_test.rb b/test/translation_diff/translation/response_test.rb index 1ce7f7e..96bc511 100644 --- a/test/translation_diff/translation/response_test.rb +++ b/test/translation_diff/translation/response_test.rb @@ -25,7 +25,7 @@ def test_build_raises_when_the_provider_returned_the_wrong_number_of_texts assert_match(/2/, error.message) end - # A nil translation used to reach Spacing.restore and die there as NoMethodError, naming nothing. + # A nil translation used to reach the old pipeline's spacing step and die there as NoMethodError, naming nothing. def test_build_raises_when_a_translation_is_not_a_string error = assert_raises(TranslationDiff::ResponseError) do TranslationDiff::Translation::Response.build(request: request, texts: ["один", nil]) diff --git a/test/translation_diff_test.rb b/test/translation_diff_test.rb index 0b4f449..bf62fea 100644 --- a/test/translation_diff_test.rb +++ b/test/translation_diff_test.rb @@ -1,6 +1,15 @@ require "test_helper" class TranslationDiffTest < ConfiguredTest + def setup + super + TranslationDiff.configure do |c| + c.provider = :null + # Pinned so a developer with REDIS_URL set doesn't have these tests reach for a socket. + c.cache = :memory + end + end + def test_has_a_version_number refute_nil TranslationDiff::VERSION end @@ -8,4 +17,27 @@ def test_has_a_version_number def test_the_default_segmenter_is_pragmatic assert_instance_of TranslationDiff::Segmenters::Pragmatic, TranslationDiff.config.segmenter_instance end + + # The entry point builds a Translator, so a nested value comes back with its shape and its non-strings intact. + def test_translate_runs_the_values_through_the_pipeline + result = TranslationDiff.translate({ title: "One. Two.", count: 42 }, from: "en", to: "ru") + + assert_equal({ title: "One. Two.", count: 42 }, result) + end + + # A caller who splats the same options hash into every call must get the same hash back out of it. + def test_repeated_calls_leave_the_callers_options_hash_alone + options = { from: :en, to: :ru } + + 2.times { assert_equal "Some string.", TranslationDiff.translate("Some string.", **options) } + + assert_equal({ from: :en, to: :ru }, options) + end + + # `to:` still defaults to nil in the signature, so the keyword it names is what the caller has to read. + def test_a_missing_target_language_is_refused_by_name + error = assert_raises(ArgumentError) { TranslationDiff.translate("One.", from: "en") } + + assert_match(/to:/, error.message) + end end From 5d144af8322faa2bfbddfd01c005a858ad585510 Mon Sep 17 00:00:00 2001 From: IG Date: Thu, 10 Sep 2026 03:23:43 +0400 Subject: [PATCH 20/21] fix: give Batch its own error class, restore an independent corpus assertion Batch::Filler#ensure_sendable! raised the bare TranslationDiff::Error, the only pipeline failure without its own class, so a caller could not catch "sentence too long for this provider" apart from an unrelated registry miss. Add Batch::Error and document it alongside Translator::Error and SentenceCache::Error. pipeline_corpus_test.rb dropped its document: assertion for the five EXPECTED_TO_CHANGE inputs, leaving only echoed: checked against itself via two paths. Restore document: as its own literal, independent of echoed:, so a regression in Translator or Passage#render is caught by its own assertion rather than being invisible behind one value checked twice. --- CHANGELOG.md | 2 +- docs/errors.md | 17 +++++++++++------ lib/translation_diff/batch.rb | 5 ++++- test/translation_diff/batch_test.rb | 5 +++-- test/translation_diff/pipeline_corpus_test.rb | 13 ++++++++++--- 5 files changed, 29 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b08ce8..53d7ff4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,7 +130,7 @@ described below. Everything here is relative to `deepl_diff` 2.2.0. gem has never been published under the name `translation_diff` with those constants in it. `TranslationDiff::Chunker::Error` is gone with no replacement -- a single sentence too large to send now raises - `TranslationDiff::Error` from `Batch`. All three remain + `TranslationDiff::Batch::Error`. All three remain `TranslationDiff::Error`, so a rescue of the base class is unaffected. - **`TranslationDiff.translate` and `Context#translate` raise `ArgumentError` when `to:` is missing or `nil`.** The keyword still defaults to `nil` in the diff --git a/docs/errors.md b/docs/errors.md index df9e80b..34b0756 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -28,6 +28,8 @@ TranslationDiff::Error │ # assigned provider object ├── TranslationDiff::SentenceCache::Error # provider options have no stable │ # serialisation for the cache key +├── TranslationDiff::Batch::Error # one sentence, once escaped, is larger +│ # than the provider's declared limit ├── TranslationDiff::Segmenters::Pragmatic::Error │ # Pragmatic computed offsets that │ # violate its own postcondition -- @@ -41,12 +43,15 @@ and `#status` (the HTTP status code), so a caller can log or branch on which service and which response caused the failure without parsing the message. `TranslationDiff::Registry` -- which backs the provider, cache store and -segmenter registries -- also raises `TranslationDiff::Error` directly (not a +segmenter registries -- raises `TranslationDiff::Error` directly (not a dedicated subclass) for an unknown name, listing what is actually -registered. So does `TranslationDiff::Batch`, when one sentence is larger -once escaped than the provider's declared `max_request_size` or -`max_text_size` and so could never be sent even in a batch of its own; the -message names a short prefix of the offending text and both numbers. +registered. `TranslationDiff::Batch::Error` is its own class rather than a +direct `TranslationDiff::Error`, so a caller can catch "this sentence is too +long for this provider" without also catching an unrelated registry miss; it +is raised when one sentence is larger once escaped than the provider's +declared `max_request_size` or `max_text_size` and so could never be sent +even in a batch of its own. The message names a short prefix of the +offending text and both numbers. `ArgumentError`, not a `TranslationDiff::Error`, is what `TranslationDiff.translate` and `Context#translate` raise when `to:` is @@ -57,5 +62,5 @@ the message names the keyword. `TranslationDiff::Translator::Error` and `TranslationDiff::Cache::Error` is now `TranslationDiff::SentenceCache::Error`; both classes they hung off are gone. `TranslationDiff::Chunker::Error` is gone with no replacement -- the -condition it named now raises `TranslationDiff::Error` from `Batch`. A +condition it named now raises `TranslationDiff::Batch::Error`. A `rescue TranslationDiff::Error` catches all three exactly as before. diff --git a/lib/translation_diff/batch.rb b/lib/translation_diff/batch.rb index 1b9e8c5..7b82e0f 100644 --- a/lib/translation_diff/batch.rb +++ b/lib/translation_diff/batch.rb @@ -1,5 +1,8 @@ # Segments grouped to one provider request; a reply lands back on them through #apply, never by position after the fact. class TranslationDiff::Batch + # Its own class, so rescuing a sentence too long to send cannot also swallow a registry miss. + class Error < TranslationDiff::Error; end + attr_reader :segments def initialize(segments) @@ -67,7 +70,7 @@ def ensure_sendable!(segment, size) limit = [@capabilities.max_request_size, @capabilities.max_text_size].compact.min return if size <= limit - raise TranslationDiff::Error, + raise TranslationDiff::Batch::Error, "#{preview(segment.core)} is #{size} characters once escaped, over this provider's limit of #{limit}" end diff --git a/test/translation_diff/batch_test.rb b/test/translation_diff/batch_test.rb index 6559bdf..75c1362 100644 --- a/test/translation_diff/batch_test.rb +++ b/test/translation_diff/batch_test.rb @@ -47,8 +47,9 @@ def test_size_is_measured_escaped_not_in_characters assert_equal 2, batches.size, "each Cyrillic word is 36 escaped characters" end + # A caller who wants to catch "too long for this provider" must not also catch an unrelated registry miss. def test_a_text_larger_than_the_request_limit_raises_naming_the_limit - error = assert_raises(TranslationDiff::Error) { pack(["x" * 50], request_size: 10) } + error = assert_raises(TranslationDiff::Batch::Error) { pack(["x" * 50], request_size: 10) } assert_match(/10/, error.message) assert_match(/50/, error.message) @@ -58,7 +59,7 @@ def test_a_text_larger_than_the_request_limit_raises_naming_the_limit # is the customer's content and this message may be logged. def test_the_too_long_error_does_not_carry_the_whole_sentence sentence = "Secret #{'y' * 200}" - error = assert_raises(TranslationDiff::Error) { pack([sentence], request_size: 10) } + error = assert_raises(TranslationDiff::Batch::Error) { pack([sentence], request_size: 10) } refute_includes error.message, "y" * 200 end diff --git a/test/translation_diff/pipeline_corpus_test.rb b/test/translation_diff/pipeline_corpus_test.rb index 9fb6814..0a7f4e8 100644 --- a/test/translation_diff/pipeline_corpus_test.rb +++ b/test/translation_diff/pipeline_corpus_test.rb @@ -41,31 +41,38 @@ def self.method_name_for(name) = :"test_#{name.gsub(/[^a-zA-Z0-9]+/, '_')}" end end - # The names in EXPECTED_TO_CHANGE, written out: the texts a provider is sent, and both of Passage's renders -- - # byte-exact untranslated, equivalent markup once every sentence is back. + # The names in EXPECTED_TO_CHANGE, written out: the texts a provider is sent, what TranslationDiff.translate + # returns end to end, and Passage's render round trip -- byte-exact untranslated, equivalent markup once every + # sentence is back. document: and echoed: are independent literals, kept apart even where they agree, so a + # regression in either translate or render is caught by its own assertion rather than by one value checked twice. # The pipeline this replaced sent ["Salt & pepper.", "Fine."] for the first, ["Hard space here.", "Fine."] # for the second, ["if a"] for the third, ["5", "6.", "True."] for the fourth and ["a"] for the fifth. CHANGED = { "entity ampersand" => { texts: ["Salt & pepper.", "Fine."], + document: "Salt & pepper. Fine.", echoed: "Salt & pepper. Fine." }, "entity nbsp" => { texts: ["Hard\u00A0space here.", "Fine."], # The entity is spelled as the character it means, which is the same document to a browser and not the same bytes. + document: "Hard\u00A0space here. Fine.", echoed: "Hard\u00A0space here. Fine." }, "bare less-than" => { texts: ["if a < b then stop.", "Fine."], + document: "if a < b then stop. Fine.", echoed: "if a < b then stop. Fine." }, "bare less-than and greater" => { texts: ["5 < 6 and 7 > 6.", "True."], + document: "5 < 6 and 7 > 6. True.", echoed: "5 < 6 and 7 > 6. True." }, # The recorded limit: ` { texts: ["a"], + document: "a Date: Thu, 10 Sep 2026 04:18:24 +0400 Subject: [PATCH 21/21] test: commit the pipeline baseline instead of reading it from a home directory The corpus test read its baseline from a path outside the repository, under the author's home directory. It passed on the machine that wrote the file and failed everywhere else: CI reported 27 errors, all `Errno::ENOENT`, on both supported Ruby versions. That path came from a standing rule that design documents stay out of git, applied to something that is not a design document. The baseline is test data -- the pipeline's recorded output and the cache keys it asked for -- and a test whose fixture exists on one laptop is not a test. It carries nothing that needed keeping out: the sentences are invented for the corpus and the keys are digests of them, with no credential and no customer text anywhere in the file. --- test/fixtures/pipeline_baseline.txt | 130 ++++++++++++++++++ test/translation_diff/pipeline_corpus_test.rb | 9 +- 2 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 test/fixtures/pipeline_baseline.txt diff --git a/test/fixtures/pipeline_baseline.txt b/test/fixtures/pipeline_baseline.txt new file mode 100644 index 0000000..3bb896b --- /dev/null +++ b/test/fixtures/pipeline_baseline.txt @@ -0,0 +1,130 @@ +# Baseline captured before the pipeline rewrite, against the :null provider (cache_key "null"). +# Every input below is translated with from: "en", to: "ru", provider: :null. +# Cache keys are the sorted, de-duplicated set of every key this run passed to read_multi or write, +# on a fresh cache store per input -- so the list is exactly what that one input asked for. +# Generated 2026-09-09 by .superpowers/sdd/2026-09-09-pipeline-plan task 1. + +=== plain sentence === +OUTPUT: "Hello there." +CACHE_KEYS: ["null:en:ru:9d6a2963872077db674a27a39c492e61"] + +=== two sentences === +OUTPUT: "Hello there. Second sentence!" +CACHE_KEYS: ["null:en:ru:1520f71fffb5adf0da75e7c17059bfd1", "null:en:ru:9d6a2963872077db674a27a39c492e61"] + +=== nested hash === +OUTPUT: {title: "One. Two.", body: "Third."} +CACHE_KEYS: ["null:en:ru:3f77101fc43570a61d5bc042bb908651", "null:en:ru:900019fa233e608091ba641d50d69b81", "null:en:ru:fdb02803abc46fba06ce1cc96d6399c5"] + +=== nested array === +OUTPUT: ["A. B.", ["C."], "D."] +CACHE_KEYS: ["null:en:ru:9bce147872014965a531500da2666847", "null:en:ru:d0904fd99a2cfb13f897223b9213c6f1", "null:en:ru:d898b880dda075d3ce1b5feee4a7bb6f"] + +=== hash with non-strings === +OUTPUT: {title: "One.", count: 42, missing: "", flag: true} +CACHE_KEYS: ["null:en:ru:900019fa233e608091ba641d50d69b81"] + +=== empty string === +OUTPUT: "" +CACHE_KEYS: [] + +=== nil === +OUTPUT: nil +CACHE_KEYS: [] + +=== not a string === +OUTPUT: 42 +CACHE_KEYS: [] + +=== bold markup === +OUTPUT: "Bold text here. Second sentence." +CACHE_KEYS: ["null:en:ru:114c3050111d8b8ddd830b99ccebd246", "null:en:ru:dd05cca0ae0eb637e49788e6daf8c23f", "null:en:ru:edcabe153bd4ad9b4a376c3af9d2dc8f"] + +=== attributes preserved === +OUTPUT: "Link text. After." +CACHE_KEYS: ["null:en:ru:3300dc3f426997c4c0c8e9bcd4af8863", "null:en:ru:97f4ef09e0c768ce8bf87f7b64a3faad"] + +=== void element === +OUTPUT: "One line.
Two lines." +CACHE_KEYS: ["null:en:ru:41d035f96f8712251d0ac9bb500e24cd", "null:en:ru:694fe8da2b6d6101fb916106f5c23ca0"] + +=== unclosed paragraph === +OUTPUT: "

First para.

Second para." +CACHE_KEYS: ["null:en:ru:24e35fe61038aed7aea27bf6dfcbe623", "null:en:ru:a95237990255526ebf59cf9ffc511064"] + +=== uppercase tags === +OUTPUT: "Bold text." +CACHE_KEYS: ["null:en:ru:114c3050111d8b8ddd830b99ccebd246", "null:en:ru:bde8d910cfb3b4d1670021fce1fd2592"] + +=== script and style === +OUTPUT: "альбракил" +CACHE_KEYS: ["null:en:ru:34430bf88522e4c1c49479eb2feeb3f3", "null:en:ru:3c11ac7405538ffcaad48db9d7149347", "null:en:ru:b1d36c7c01d8359faddba68b671e7d47"] + +=== processing instruction === +OUTPUT: "Hey!
Look!" +CACHE_KEYS: ["null:en:ru:6ae99d4d2de5e3cbd29fec87ae7d76eb", "null:en:ru:8bd5669756914fb1c9a11cbd755dcd31"] + +=== comment === +OUTPUT: " Visible text here." +CACHE_KEYS: ["null:en:ru:d41646cf35fd5a9a6959f6715d5e14bc"] + +=== doctype === +OUTPUT: "

Body text.

" +CACHE_KEYS: ["null:en:ru:ac9d6317208a722c1c19bd667c374d22"] + +=== cdata === +OUTPUT: "Before.After." +CACHE_KEYS: ["null:en:ru:97f4ef09e0c768ce8bf87f7b64a3faad", "null:en:ru:d5639cb2ca1fb06574bd02a8435be8ad"] + +=== notranslate span === +OUTPUT: "Bold Mountain is a good place." +CACHE_KEYS: ["null:en:ru:68980e62d2581d949019300826e1b197"] + +=== nested notranslate === +OUTPUT: "foobarbaz" +CACHE_KEYS: ["null:en:ru:eafcec3766a26334358d8df31c5a82cc"] + +=== notranslate inside span === +OUTPUT: "foobar
baz" +CACHE_KEYS: ["null:en:ru:5b513613cd49691d47e6df65921f5264"] + +=== br before closing tag === +OUTPUT: "Смеркалось.
" +CACHE_KEYS: ["null:en:ru:e1583f8217a1cfc2b7ee21afd908e955"] + +=== blank line between sentences === +OUTPUT: "Первое предложение.\n\nВторое предложение." +CACHE_KEYS: ["null:en:ru:b83f03a6405a6e2d24f9d89380c661d4", "null:en:ru:f1af66af7c0dc420d4267696c7d0aa13"] + +=== single newline === +OUTPUT: "test\nphrase" +CACHE_KEYS: ["null:en:ru:fae8d0840a5c100965609a9d7a55135e"] + +=== leading and trailing space === +OUTPUT: " Padded sentence. " +CACHE_KEYS: ["null:en:ru:fe3bf43723a64fb32bcac8d99bb431af"] + +=== many sentences === +OUTPUT: "Sentence number 1. Sentence number 2. Sentence number 3. Sentence number 4. Sentence number 5. Sentence number 6. Sentence number 7. Sentence number 8. Sentence number 9. Sentence number 10. Sentence number 11. Sentence number 12. Sentence number 13. Sentence number 14. Sentence number 15. Sentence number 16. Sentence number 17. Sentence number 18. Sentence number 19. Sentence number 20. Sentence number 21. Sentence number 22. Sentence number 23. Sentence number 24. Sentence number 25. Sentence number 26. Sentence number 27. Sentence number 28. Sentence number 29. Sentence number 30. Sentence number 31. Sentence number 32. Sentence number 33. Sentence number 34. Sentence number 35. Sentence number 36. Sentence number 37. Sentence number 38. Sentence number 39. Sentence number 40." +CACHE_KEYS: ["null:en:ru:028166de0c432e30aa01d79966d36e27", "null:en:ru:042f7dfcdaf51c2812e9975436ff7163", "null:en:ru:0b1fa3a903f7429246b94933e1c65a22", "null:en:ru:13b8656fe18559c2e98fff3aa3b25136", "null:en:ru:1c3a311a4eef441e056310213a1051e2", "null:en:ru:27c36e35e03a99369e2b60cee64758df", "null:en:ru:2fde1f57bcf20df4087f574ae00d9692", "null:en:ru:42c04e1e1ea3177d2921386c91d0c579", "null:en:ru:43d0f7cdc6be74da0e4cfd4ef8e9b886", "null:en:ru:4c94cb29ddbbbc231ac855011456f46d", "null:en:ru:4d00adc400cec01a812aed2a741778ef", "null:en:ru:522df2f9bbac92f9714bbbb2aa229bc7", "null:en:ru:5b885db48f0a06635ab079fd96c3c7e6", "null:en:ru:5c6d3d369e65d4e33708b48263f7aa16", "null:en:ru:62992cce554f47436aa2ebc09bfa4389", "null:en:ru:80c4df8aced58f66cc564ec992836c09", "null:en:ru:817375736915088db4c098d047232820", "null:en:ru:81c38a3ec70d697ec7c10480b40a82d2", "null:en:ru:831d768717f5c58649e33141ba610e22", "null:en:ru:881e7f42d84eeac1ee76c4db213304a3", "null:en:ru:a298c866d524ed0e6a6f81eea80eb3c6", "null:en:ru:a677508b1eadd12106cc9f3f604d7d51", "null:en:ru:a98a84ab5702ea2f29c01ac35ebd4a2e", "null:en:ru:aa0e7ef164f4b6a60049265162d7b1eb", "null:en:ru:b0dc3760c308f5db9454ff53afed6842", "null:en:ru:bb716ddca5d51bae619894f6fc092337", "null:en:ru:bbd3042479feacc0e179bc05eca8a762", "null:en:ru:be90cce4c0800e090bb1127e102aaa72", "null:en:ru:bfc9abd6522e8eb83e1c05d2a84f5b67", "null:en:ru:c41c34620d095e2db18519e409a58dcf", "null:en:ru:c6a1605fb42cad1439b88d5e1a62ea97", "null:en:ru:c8a3cab86135767607d8b6cff57f7e26", "null:en:ru:d49160d08ac9cb12ca8b1c2373085b12", "null:en:ru:daf84cfee6e6a35e13e9e994850e39e3", "null:en:ru:e299181f4980628d4b3ef62ad0042c4c", "null:en:ru:e6560e25669e4af82fc667a67b74b001", "null:en:ru:e9acf5c038149e3f8e03f6c62c317c89", "null:en:ru:eba0b3330f77dbe64b70db689be5e046", "null:en:ru:fb7c729c821d5b4d5ebdab2e5273b869", "null:en:ru:fe9379e1a22ff0052f72ef22dbaf1d96"] + +=== non-ascii === +OUTPUT: "Привет. Как дела? Всё хорошо." +CACHE_KEYS: ["null:en:ru:3ae1adb068af8b1ca435825a5f6a2dd9", "null:en:ru:6440c78297d465869ba8f86b114f7328", "null:en:ru:cc34260931ed7b37f90ec7dc5b44f3ad"] + +=== entity ampersand === +OUTPUT: "Salt & pepper. Fine." +CACHE_KEYS: ["null:en:ru:05d12994070fdde458e566149f42472f", "null:en:ru:b5010567e209726a125c9ed59162eca5"] + +=== entity nbsp === +OUTPUT: "Hard space here. Fine." +CACHE_KEYS: ["null:en:ru:98abf3f0bf3b826e8f2c21fd72823687", "null:en:ru:b5010567e209726a125c9ed59162eca5"] + +=== bare less-than === +OUTPUT: "if a < b then stop. Fine." +CACHE_KEYS: ["null:en:ru:56b4445b422a53efdff59cb72a58f599"] + +=== bare less-than and greater === +OUTPUT: "5 < 6 and 7 > 6. True." +CACHE_KEYS: ["null:en:ru:4363b12ae39947f045a4fb5fad740dc8", "null:en:ru:65364203e430439161a97b3d245a0168", "null:en:ru:e4da3b7fbbce2345d7772b0674a318d5"] + diff --git a/test/translation_diff/pipeline_corpus_test.rb b/test/translation_diff/pipeline_corpus_test.rb index 0a7f4e8..01b083c 100644 --- a/test/translation_diff/pipeline_corpus_test.rb +++ b/test/translation_diff/pipeline_corpus_test.rb @@ -1,11 +1,12 @@ require "test_helper" require "support/pipeline_corpus" -# Judges the pipeline rewrite against the baseline captured in ~/JetRockets/.deepl_diff-specs/pipeline-baseline.txt -# before any of the pipeline changed, translating every input the same way the baseline script did: through the -# :null provider, from "en" to "ru". +# Judges the pipeline rewrite against test/fixtures/pipeline_baseline.txt, captured before any of the pipeline +# changed, translating every input the way the baseline script did: through the :null provider, en to ru. class PipelineCorpusTest < ConfiguredTest - BASELINE_PATH = File.expand_path("~/JetRockets/.deepl_diff-specs/pipeline-baseline.txt") + # Committed, not read from a developer's home directory: a test that depends on an untracked file on one machine + # passes there and fails everywhere else, which is what it did in CI. + BASELINE_PATH = File.expand_path("../fixtures/pipeline_baseline.txt", __dir__) def self.baseline_outputs @baseline_outputs ||= File.read(BASELINE_PATH).scan(/^=== (.+) ===\nOUTPUT: (.*)\n/).to_h