Refuse to convert when the bytes do not identify the encoding

sources/EncodingChecker.Tests/EncodingAmbiguityTests.cs

@@ -0,0 +1,120 @@
+using System.Text;
+
+namespace EncodingChecker.Tests;
+
+/// <summary>
+/// A corpus audit of 5,078 files found 262 where the bytes do not identify the encoding
+/// that wrote them: single-byte code pages map 256 values independently, so a file valid
+/// in windows-1252 is equally valid in iso-8859-1 and no inspection decides between them.
+/// Detection still answers, and converting on that answer rewrites the file into one of
+/// several possible readings without saying so.
+///
+/// These pin the distinction the refusal rests on. Two failure directions matter equally:
+/// converting a genuinely ambiguous file, and refusing one whose encoding the bytes do
+/// determine. A gate that refuses everything is not safe, it is broken.
+/// </summary>
+public sealed class EncodingAmbiguityTests
+{
+    private static AmbiguityAnalysis Analyze(string text, string charset) =>
+        EncodingAmbiguity.Analyze(
+            Encoding.GetEncoding(charset).GetBytes(text),
+            Encoding.GetEncoding(charset));
+
+    [Theory]
+    [InlineData("utf-8", "Hello 世界 café")]
+    [InlineData("shift_jis", "こんにちは世界。日本語のテキスト")]
+    [InlineData("euc-jp", "こんにちは世界")]
+    [InlineData("big5", "你好世界。這是繁體中文")]
+    [InlineData("gb18030", "这是简体中文文本")]
+    [InlineData("euc-kr", "안녕하세요 세계")]
+    [InlineData("utf-16", "Hello 世界 café")]
+    public void StructuredEncodingsAreNotTreatedAsAmbiguous(string charset, string text)
+    {
+        // These constrain their byte sequences, so a file valid under one was not valid
+        // by accident. Other codecs "reading" it differently are codecs that cannot
+        // refuse anything, which is not a competing claim.
+        AmbiguityAnalysis analysis = Analyze(text, charset);
+
+        Assert.NotEqual(AmbiguityClass.TextChanging, analysis.Class);
+        Assert.True(analysis.IsSafeToConvertAutomatically);
+    }
+
+    [Theory]
+    [InlineData("windows-1252", "Le café était déjà prêt")]
+    [InlineData("koi8-r", "Привет мир")]
+    [InlineData("iso-8859-7", "Γειά σου κόσμε")]
+    public void SingleByteTextWithNoDistinguishingStructureIsAmbiguous(
+        string charset, string text)
+    {
+        AmbiguityAnalysis analysis = Analyze(text, charset);
+
+        Assert.Equal(AmbiguityClass.TextChanging, analysis.Class);
+        Assert.False(analysis.IsSafeToConvertAutomatically);
+        Assert.NotEmpty(analysis.CompetingCandidates);
+    }
+
+    [Fact]
+    public void PureAsciiIsAmbiguousInLabelButNotInText()
+    {
+        // Every candidate agrees on what this file says, so the label is undetermined and
+        // the content is not. Refusing here would protect nothing.
+        AmbiguityAnalysis analysis = EncodingAmbiguity.Analyze(
+            "plain ascii, no high bytes at all"u8, Encoding.ASCII);
+
+        Assert.NotEqual(AmbiguityClass.TextChanging, analysis.Class);
+        Assert.True(analysis.IsSafeToConvertAutomatically);
+        Assert.Empty(analysis.CompetingCandidates);
+    }
+
+    [Fact]
+    public void TheRefusalNamesTheEncodingsActuallyInConflict()
+    {
+        // "Low confidence" gives a user nothing to act on. The competing encodings and
+        // the next step do.
+        AmbiguityAnalysis analysis = Analyze("Le café était déjà prêt", "windows-1252");
+
+        string message = analysis.Describe("windows-1252");
+
+        Assert.Contains("could not be determined uniquely", message);
+        Assert.Contains("windows-1252", message);
+        Assert.Contains("would produce different text", message);
+        Assert.Contains("specify the source encoding explicitly", message);
+    }
+
+    [Fact]
+    public void TheSameBytesAlwaysGetTheSameAnswer()
+    {
+        // An earlier version sampled probe positions randomly and moved with the seed on
+        // short files. Whether a conversion is refused must not depend on a random draw.
+        byte[] bytes = Encoding.GetEncoding("windows-1252").GetBytes("Le café était déjà prêt");
+        Encoding detected = Encoding.GetEncoding("windows-1252");
+
+        AmbiguityClass first = EncodingAmbiguity.Analyze(bytes, detected).Class;
+
+        for (int i = 0; i < 8; i++)
+            Assert.Equal(first, EncodingAmbiguity.Analyze(bytes, detected).Class);
+    }
+
+    [Fact]
+    public void AliasesOfOneEncodingAreNotCountedAsRivalReadings()
+    {
+        // The candidate set is deduplicated by code page. cp949 and ks_c_5601-1987 name
+        // one encoding, and listing both would manufacture a disagreement out of a
+        // spelling difference - a mistake this project has made before, in the audit that
+        // compared detector output by label.
+        AmbiguityAnalysis analysis = Analyze("안녕하세요 세계", "euc-kr");
+
+        Assert.Equal(
+            analysis.CompetingCandidates.Count,
+            analysis.CompetingCandidates.Distinct().Count());
+    }
+
+    [Fact]
+    public void AnEmptySampleIsNotClaimedToBeAmbiguous()
+    {
+        AmbiguityAnalysis analysis = EncodingAmbiguity.Analyze(
+            ReadOnlySpan<byte>.Empty, Encoding.UTF8);
+
+        Assert.True(analysis.IsSafeToConvertAutomatically);
+    }
+}

sources/EncodingChecker.Tests/ExplicitSourceEncodingTests.cs

@@ -0,0 +1,204 @@
+using System.Text;
+
+namespace EncodingChecker.Tests;
+
+/// <summary>
+/// The contract for <c>-From</c>, and the escape from an ambiguity refusal.
+///
+/// The refusal message tells a user to specify the source encoding, so specifying it has
+/// to work — otherwise the safety feature issues advice its own interface cannot take.
+///
+/// But it replaces detection and nothing else. Every guarantee from the conversion engine
+/// still holds: the bytes must strictly decode as the named encoding, the output must
+/// re-decode to exactly the same text, and a failed backup still aborts. <c>-From</c>
+/// answers "which encoding is this?", not "convert it regardless".
+/// </summary>
+public sealed class ExplicitSourceEncodingTests : IDisposable
+{
+    private readonly string _root =
+        Directory.CreateTempSubdirectory("ec_from_").FullName;
+
+    public void Dispose()
+    {
+        try
+        {
+            Directory.Delete(_root, recursive: true);
+        }
+        catch (IOException)
+        {
+            // Best-effort cleanup.
+        }
+    }
+
+    private string Write(string name, byte[] content)
+    {
+        string path = Path.Combine(_root, name);
+        File.WriteAllBytes(path, content);
+        return path;
+    }
+
+    private List<ConversionReportEntry> Scan(string? from, bool backup = false)
+    {
+        var results = new List<ConversionReportEntry>();
+
+        ScanEngine.ScanDirectory(
+            new ScanDirectoryOptions
+            {
+                BaseDirectory = _root,
+                IncludeSubdirectories = true,
+                IncludePatterns = ["*"],
+                Action = ScanAction.Convert,
+                TargetCharset = "utf-8",
+                TargetWriteBom = false,
+                SourceCharset = from,
+                Backup = backup,
+            },
+            results.Add,
+            CancellationToken.None);
+
+        return results;
+    }
+
+    [Fact]
+    public void AmbiguousAutoDetection_IsRefused()
+    {
+        Write("ambiguous.txt",
+            Encoding.GetEncoding("windows-1252").GetBytes("Le café était déjà prêt"));
+
+        ConversionReportEntry entry = Assert.Single(Scan(from: null));
+
+        Assert.Equal(ConversionRowResult.Error, entry.Result);
+        Assert.Contains("could not be determined uniquely", entry.Diagnostic);
+    }
+
+    [Fact]
+    public void SameTextAlternativeCodecs_AreAllowed()
+    {
+        // Every candidate agrees on the content, so the label is undetermined and the
+        // text is not. There is nothing to protect the user from.
+        Write("ascii.txt", "plain ascii, no high bytes"u8.ToArray());
+
+        ConversionReportEntry entry = Assert.Single(Scan(from: null));
+
+        Assert.NotEqual(ConversionRowResult.Error, entry.Result);
+    }
+
+    [Fact]
+    public void ExplicitSource_LetsTheConversionProceed()
+    {
+        // The same file the detector refuses. Somebody has now said which encoding it is.
+        const string text = "Le café était déjà prêt";
+        string path = Write("resolved.txt",
+            Encoding.GetEncoding("windows-1252").GetBytes(text));
+
+        ConversionReportEntry entry = Assert.Single(Scan(from: "windows-1252"));
+
+        Assert.Equal(ConversionRowResult.Converted, entry.Result);
+        Assert.Equal(text, Encoding.UTF8.GetString(File.ReadAllBytes(path)));
+        Assert.True(entry.SourceEncodingWasSpecified);
+    }
+
+    [Fact]
+    public void ExplicitSource_ChangesTheAnswerNotJustThePermission()
+    {
+        // Naming a different encoding for the same bytes must produce different text.
+        // If it did not, -From would be decoration rather than a decision.
+        byte[] bytes = Encoding.GetEncoding("windows-1252").GetBytes("café");
+        string path = Write("interpretation.txt", bytes);
+
+        Assert.Equal(ConversionRowResult.Converted,
+            Assert.Single(Scan(from: "koi8-r")).Result);
+
+        string asKoi8 = Encoding.UTF8.GetString(File.ReadAllBytes(path));
+
+        Assert.NotEqual("café", asKoi8);
+        Assert.Equal(Encoding.GetEncoding("koi8-r").GetString(bytes), asKoi8);
+    }
+
+    [Fact]
+    public void ExplicitSource_WithBytesItCannotDecode_IsStillRefused()
+    {
+        // EUC-JP bytes carrying a JIS X 0212 sequence code page 51932 cannot map.
+        // Naming the encoding does not make the bytes representable.
+        byte[] unrepresentable =
+            [0x8F, 0xB0, 0xDF, 0xB9, 0xA5, 0xA1, 0xA4, 0xC0, 0xA4, 0xB3];
+        string path = Write("undecodable.txt", unrepresentable);
+
+        ConversionReportEntry entry = Assert.Single(Scan(from: "euc-jp"));
+
+        Assert.Equal(ConversionRowResult.Error, entry.Result);
+        Assert.Equal(unrepresentable, File.ReadAllBytes(path));
+    }
+
+    [Fact]
+    public void ExplicitSource_WithContentTheTargetCannotHold_IsStillRefused()
+    {
+        // Converting to a target that cannot represent the text must fail whether the
+        // source encoding was detected or chosen.
+        string path = Path.Combine(_root, "unencodable.txt");
+        byte[] original = Encoding.UTF8.GetBytes("世界 مرحبا");
+        File.WriteAllBytes(path, original);
+
+        var results = new List<ConversionReportEntry>();
+
+        ScanEngine.ScanDirectory(
+            new ScanDirectoryOptions
+            {
+                BaseDirectory = _root,
+                IncludeSubdirectories = true,
+                IncludePatterns = ["*"],
+                Action = ScanAction.Convert,
+                TargetCharset = "windows-1252",
+                TargetWriteBom = false,
+                SourceCharset = "utf-8",
+            },
+            results.Add,
+            CancellationToken.None);
+
+        ConversionReportEntry entry = Assert.Single(results);
+
+        Assert.Equal(ConversionRowResult.Error, entry.Result);
+        Assert.Equal(original, File.ReadAllBytes(path));
+    }
+
+    [Fact]
+    public void ExplicitSource_WithAFailingBackup_IsStillRefused()
+    {
+        byte[] original = Encoding.GetEncoding("windows-1252").GetBytes("café");
+        string path = Write("backupfail.txt", original);
+        Directory.CreateDirectory(path + ".bak");
+
+        ConversionReportEntry entry =
+            Assert.Single(Scan(from: "windows-1252", backup: true));
+
+        Assert.Equal(ConversionRowResult.Error, entry.Result);
+        Assert.Equal(original, File.ReadAllBytes(path));
+    }
+
+    [Fact]
+    public void ExplicitSource_StillWritesTheRecoveryRecord()
+    {
+        // Choosing the encoding does not opt out of being able to undo the result.
+        string path = Write("recorded.txt",
+            Encoding.GetEncoding("windows-1252").GetBytes("café"));
+
+        Assert.Equal(ConversionRowResult.Converted,
+            Assert.Single(Scan(from: "windows-1252", backup: true)).Result);
+
+        RestoreStatus status = ConversionMetadataStore.Inspect(path);
+
+        Assert.Equal(RestoreAvailability.Available, status.Availability);
+        Assert.Equal(1252, status.Metadata!.DetectedCodePage);
+    }
+
+    [Fact]
+    public void DetectionModeIsRecordedSoTheTwoClaimsStayDistinct()
+    {
+        // Detection can be wrong in ways an explicit choice cannot. A later journal
+        // needs to know which one produced a conversion.
+        Write("ascii.txt", "plain ascii"u8.ToArray());
+
+        Assert.False(Assert.Single(Scan(from: null)).SourceEncodingWasSpecified);
+        Assert.True(Assert.Single(Scan(from: "windows-1252")).SourceEncodingWasSpecified);
+    }
+}

sources/EncodingChecker/ConversionReport.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Text;
@@ -48,6 +48,37 @@ internal sealed class ConversionReportEntry
     /// </summary>
     internal string? CurrentCharsetLabel { get; set; }
 
+    /// <summary>
+    /// How far this file's bytes identify the encoding that wrote them, decided during
+    /// detection. Defaults to <see cref="AmbiguityClass.Unambiguous"/>, which is correct
+    /// for a caller that supplies the source encoding rather than having it detected:
+    /// there is nothing ambiguous about an answer somebody gave.
+    /// Internal state; not included in CSV output.
+    /// </summary>
+    internal AmbiguityClass Ambiguity { get; set; } = AmbiguityClass.Unambiguous;
+
+    /// <summary>
+    /// Encodings that read this file differently from the one detected. Empty unless
+    /// <see cref="Ambiguity"/> is <see cref="AmbiguityClass.TextChanging"/>.
+    /// Internal state; not included in CSV output.
+    /// </summary>
+    internal IReadOnlyList<string> CompetingEncodings { get; set; } = [];
+
+    /// <summary>
+    /// Why this file received its <see cref="Ambiguity"/> classification.
+    /// Internal state; not included in CSV output.
+    /// </summary>
+    internal AmbiguityReason AmbiguityReason { get; set; } =
+        AmbiguityReason.ExplicitlySpecified;
+
+    /// <summary>
+    /// Whether the source encoding was chosen by the caller rather than detected.
+    /// Recorded because the two are different claims: detection can be wrong in ways an
+    /// explicit choice cannot, and a later journal should be able to say which was used.
+    /// Internal state; not included in CSV output.
+    /// </summary>
+    internal bool SourceEncodingWasSpecified { get; set; }
+
     /// <summary>Additional error detail; not included in CSV output.</summary>
     internal string? Diagnostic { get; set; }
 }