diff --git a/README.md b/README.md index c81202d..3262bfb 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ Each [release](https://github.com/amrali-eg/EncodingChecker/releases) publishes - Layered detection: byte-order-mark and heuristic checks for Unicode encodings, [UtfUnknown](https://github.com/CharsetDetector/UTF-unknown) for legacy code pages, each candidate independently verified by strict decoding before being trusted. - Lossless, safe conversion: every write is verified afterward by comparing a SHA-256 hash of the decoded content, so a silent encoder substitution (e.g. an unrepresentable character) is caught and reported as an error instead of corrupting the file. +- Refuses to convert files whose encoding the bytes do not determine, naming the encodings actually in conflict, with `-From` to supply the answer yourself. +- `-Plan`/`-Apply` preflight: review what a conversion would do, then carry out exactly that — the plan is bound to the files' hashes and is refused whole if they change. - Optional `.bak` backup before overwriting, and a `-WhatIf` dry-run mode that reports what would happen without touching any file. - Covered by an xUnit test suite exercising the detection/conversion engine, CLI argument parsing, and CSV report formatting across multilingual content and edge cases. @@ -40,6 +42,11 @@ EncodingChecker.exe -Validate "" # Validate mode: flag files not in this list -DetectOnly # Read-only detection mode + [-From ""] # Treat every file as this encoding instead of + # detecting it (Convert mode only) + [-Plan ] # Write a conversion plan; change nothing + [-Apply ] # Carry out a plan written by -Plan + [-Report ] # Also write a CSV report to this path [-MaxParallelism ] # Default: min(logical processor count, 4) [-WhatIf] # Convert mode: report without writing @@ -55,6 +62,79 @@ EncodingChecker.exe `-Backup` only ever writes a `.bak` when a real conversion happens: a file that already matches the target is left alone, and under `-WhatIf` nothing is written at all, so no backup is created. +### Ambiguous encodings, and `-From` + +Some files do not identify the encoding that wrote them. A file valid in +windows-1252 is equally valid in iso-8859-1 and in koi8-r, and each reads it as +different text; nothing in the bytes decides between them. Detection still +produces an answer, and converting on that answer rewrites the file into one of +several possible readings without saying so. + +EncodingChecker refuses those conversions and names the encodings actually in +conflict: + +``` +Error: notes.txt: The encoding could not be determined uniquely from the file's +contents. iso-8859-1 and cp866, ibm852, ibm855, iso-8859-13, and 17 more all match +this file and would produce different text. No conversion was performed; specify +the source encoding explicitly to convert it. +``` + +The refusal applies only where the rival readings *disagree about the text*. A +file whose encoding is undetermined but whose candidates all decode it +identically — plain ASCII being the common case — is converted normally, because +there is nothing to protect the user from. Nor does it apply where the file's own +structure picks the encoding out: valid UTF-8, Shift_JIS or Big5 byte sequences +are not valid by accident, and codecs that accept any byte sequence are not +offering a competing reading so much as failing to object. + +`-From` supplies the answer detection could not. It replaces detection and +nothing else: the bytes must still decode strictly as the named encoding, the +output is still verified to hold exactly the same text, and a failed backup still +aborts the conversion. Naming an encoding says which one it is, not "convert it +regardless". + +### Preflight: `-Plan` and `-Apply` + +`-Plan` writes down what a conversion would do and changes nothing. For every +file the plan records the action, the source encoding, whether it was detected or +specified, whether the bytes identify it uniquely, which encodings compete for +it, and the reason behind any refusal — as JSON, alongside a summary on stdout: + +``` +Selected: 3 + +Will convert: 2 + encoding determined: 2 + same text either way: 0 +Already in target encoding: 0 +Encoding not identified: 0 +Refused, ambiguous encoding: 1 +Refused, unreadable: 0 + +Backups: enabled +Target: utf-8 without BOM + +No files modified. +``` + +`-Apply` carries that plan out. It does not detect anything a second time: the +encodings, the target, and the backup setting all come from the plan, so +`-BasePath`, `-Target`, `-From`, and `-Backup` are rejected rather than silently +ignored. + +The binding is the point of the feature, not the preview. Every scheduled file +carries the SHA-256 it had when the plan was made, and `-Apply` verifies each one +first. If any file has changed or been deleted in between, **nothing is +converted** — not even the files that still match. A plan reviewed as a whole +belongs to the directory it was reviewed against, and the files most likely to +have changed are the ones something else is actively writing. + +```bash +EncodingChecker.exe -BasePath . -Include "*" -Target "utf-8" -Plan plan.json +EncodingChecker.exe -Apply plan.json +``` + Exit codes: `0` clean, `1` usage/argument error (nothing was scanned), `2` `-FailOnChanges` triggered, `3` the run did not complete cleanly — one or more files failed to process, the scan itself failed, or the `-Report` file could not be written, `4` cancelled (Ctrl+C). These are the same codes as [LineEndingNormalizer](https://github.com/amrali-eg/LineEndingNormalizer), a companion Windows CLI tool that normalizes line endings, so a script driving both can share one exit-code mapping. It additionally returns `5` for a missing base directory and `6` for a reparse-point `-BasePath`, both of which are reported here as `1` — so no code means two different things across the two tools, and treating `1`, `5` and `6` alike handles either. @@ -113,6 +193,14 @@ These are the guarantees the implementation actually provides. always reversible — but only for someone who still knows which codec was used, and that is recorded solely in the conversion report. The CLI leaves `-Backup` opt-in, since a script can keep the report. +- A conversion whose source encoding cannot be determined from the file's own + bytes is refused rather than guessed at, when the competing encodings would + produce different text. `-From` overrides the detection, not the conversion + safeguards. See [Ambiguous encodings](#ambiguous-encodings-and--from). +- A plan written by `-Plan` is bound to the SHA-256 of every file it schedules. + `-Apply` verifies all of them before writing anything and refuses the plan + whole if any file has changed, so a decision made about one set of bytes is + never applied to a different one. - `-BasePath` itself is rejected if it is a symbolic link, junction, or other reparse point. Reparse-point subdirectories are skipped during traversal, and a file that is (or becomes) a reparse point is rejected at diff --git a/sources/EncodingChecker.Tests/ConversionPlanTests.cs b/sources/EncodingChecker.Tests/ConversionPlanTests.cs new file mode 100644 index 0000000..cb83447 --- /dev/null +++ b/sources/EncodingChecker.Tests/ConversionPlanTests.cs @@ -0,0 +1,321 @@ +using System.Text; +using System.Text.Json; + +namespace EncodingChecker.Tests; + +/// +/// The contract for -Plan and -Apply. +/// +/// A preview whose only guarantee is "we looked at these files once" is worth very +/// little: between the preview and the conversion the directory can change, and a +/// second detection pass over changed bytes can reach different conclusions than the +/// one the user read and approved. So the plan records the SHA-256 of every file it +/// schedules, and applying it verifies each hash before anything is written. +/// +/// The invariants pinned here are: planning writes nothing; applying converts exactly +/// what was previewed without detecting again; and a plan that no longer describes the +/// files on disk is refused whole rather than applied in part. +/// +public sealed class ConversionPlanTests : IDisposable +{ + private readonly string _root = + Directory.CreateTempSubdirectory("ec_plan_").FullName; + + private string PlanPath => Path.Combine(_root, "plan.json"); + + public void Dispose() + { + try + { + Directory.Delete(_root, recursive: true); + } + catch (IOException) + { + // Best-effort cleanup. + } + } + + private static int Run(params string[] args) + { + TextWriter originalOut = Console.Out; + TextWriter originalError = Console.Error; + + try + { + Console.SetOut(new StringWriter()); + Console.SetError(new StringWriter()); + + return Program.RunConsoleMode(args); + } + finally + { + Console.SetOut(originalOut); + Console.SetError(originalError); + } + } + + private string Write(string name, string text, string charset) + { + string path = Path.Combine(_root, name); + File.WriteAllBytes(path, Encoding.GetEncoding(charset).GetBytes(text)); + return path; + } + + private static Dictionary Snapshot(string directory) => + Directory + .EnumerateFiles(directory, "*", SearchOption.AllDirectories) + .ToDictionary(p => p, File.ReadAllBytes); + + private int Plan(params string[] extra) => + Run(["-BasePath", _root, "-Target", "utf-8", "-Plan", PlanPath, "-Quiet", .. extra]); + + private ConversionPlan LoadPlan() + { + ConversionPlan? plan = ConversionPlan.Load(PlanPath, out string? error); + + Assert.Null(error); + Assert.NotNull(plan); + + return plan; + } + + private PlannedFile PlannedFor(string name) => + Assert.Single( + LoadPlan().Files, + f => string.Equals(Path.GetFileName(f.Path), name, StringComparison.Ordinal)); + + [Fact] + public void PlanningWritesNothing() + { + // The one thing a dry run must never do. + Write("jp.txt", "こんにちは世界。日本語のテキストです。", "shift_jis"); + Write("ru.txt", "Привет мир, это русский текст", "koi8-r"); + Write("plain.txt", "just ascii here", "ascii"); + + Dictionary before = Snapshot(_root); + + Assert.Equal(0, Plan()); + + // The plan file lands in the same directory, so compare the originals rather + // than the directory listing. + foreach ((string path, byte[] content) in before) + Assert.Equal(content, File.ReadAllBytes(path)); + + Assert.True(File.Exists(PlanPath)); + } + + [Fact] + public void ApplyingConvertsExactlyWhatWasPreviewed() + { + const string text = "こんにちは世界。日本語のテキストです。"; + string path = Write("jp.txt", text, "shift_jis"); + + Assert.Equal(0, Plan()); + Assert.Equal(PlannedAction.Convert, PlannedFor("jp.txt").Action); + + Assert.Equal(0, Run("-Apply", PlanPath)); + Assert.Equal(text, Encoding.UTF8.GetString(File.ReadAllBytes(path))); + } + + [Fact] + public void AFileChangedAfterPlanningInvalidatesTheWholePlan() + { + // All-or-nothing on purpose. Converting the files that still match would apply a + // plan the user reviewed as a whole to a directory that is no longer the one they + // reviewed - and the files most likely to have changed are the ones something + // else is actively writing. + string stable = Write("stable.txt", "こんにちは世界。テキスト", "shift_jis"); + string moved = Write("moved.txt", "さようなら世界。テキスト", "shift_jis"); + + Assert.Equal(0, Plan()); + + byte[] stableBefore = File.ReadAllBytes(stable); + File.WriteAllBytes(moved, Encoding.UTF8.GetBytes("replaced after the plan")); + + Assert.Equal(3, Run("-Apply", PlanPath)); + + // Neither file was touched, not only the one that changed. + Assert.Equal(stableBefore, File.ReadAllBytes(stable)); + Assert.Equal("replaced after the plan", File.ReadAllText(moved)); + } + + [Fact] + public void AFileDeletedAfterPlanningInvalidatesThePlan() + { + string kept = Write("kept.txt", "こんにちは世界。テキスト", "shift_jis"); + string removed = Write("removed.txt", "さようなら世界。テキスト", "shift_jis"); + + Assert.Equal(0, Plan()); + + byte[] keptBefore = File.ReadAllBytes(kept); + File.Delete(removed); + + Assert.Equal(3, Run("-Apply", PlanPath)); + Assert.Equal(keptBefore, File.ReadAllBytes(kept)); + } + + [Fact] + public void ApplyingUsesThePlansEncodingRatherThanDetectingAgain() + { + // The point of the whole feature. These bytes detect as one thing and were + // planned as another, so the resulting text says which pass decided. + byte[] bytes = Encoding.GetEncoding("windows-1252").GetBytes("café"); + string path = Path.Combine(_root, "reinterpreted.txt"); + File.WriteAllBytes(path, bytes); + + Assert.Equal(0, Plan("-From", "koi8-r")); + Assert.Equal("koi8-r", PlannedFor("reinterpreted.txt").SourceEncoding); + + Assert.Equal(0, Run("-Apply", PlanPath)); + + Assert.Equal( + Encoding.GetEncoding("koi8-r").GetString(bytes), + Encoding.UTF8.GetString(File.ReadAllBytes(path))); + } + + [Fact] + public void ARefusedFileIsRecordedAsRefusedAndNeverConverted() + { + byte[] original = + Encoding.GetEncoding("windows-1252").GetBytes("Le café était déjà prêt"); + string path = Path.Combine(_root, "ambiguous.txt"); + File.WriteAllBytes(path, original); + + Assert.Equal(0, Plan()); + + PlannedFile planned = PlannedFor("ambiguous.txt"); + + Assert.Equal(PlannedAction.Refuse, planned.Action); + Assert.True(planned.MayChangeText); + Assert.NotEmpty(planned.CompetingEncodings); + Assert.Contains("could not be determined uniquely", planned.Reason); + + Assert.Equal(0, Run("-Apply", PlanPath)); + Assert.Equal(original, File.ReadAllBytes(path)); + } + + [Fact] + public void TheBackupChoiceIsTakenFromThePlanNotFromTheApplyingRun() + { + // What the user reviewed included whether originals would be kept. Applying must + // not quietly convert without backups because the second command line omitted a + // flag the first one carried. + string path = Write("jp.txt", "こんにちは世界。テキスト", "shift_jis"); + + Assert.Equal(0, Plan("-Backup")); + Assert.True(LoadPlan().BackupEnabled); + + Assert.Equal(0, Run("-Apply", PlanPath)); + + Assert.True(File.Exists(path + ".bak")); + Assert.Equal( + RestoreAvailability.Available, + ConversionMetadataStore.Inspect(path).Availability); + } + + [Fact] + public void EveryScheduledFileCarriesTheHashItHadWhenPlanned() + { + string path = Write("jp.txt", "こんにちは世界。テキスト", "shift_jis"); + + Assert.Equal(0, Plan()); + + Assert.Equal( + ConversionMetadataStore.ComputeSha256(path), + PlannedFor("jp.txt").Sha256); + } + + [Fact] + public void AnUnreadablePlanIsReportedRatherThanIgnored() + { + File.WriteAllText(PlanPath, "{ not json at all"); + + Assert.Equal(1, Run("-Apply", PlanPath)); + Assert.Null(ConversionPlan.Load(PlanPath, out string? error)); + Assert.NotNull(error); + } + + [Fact] + public void APlanFromAFutureVersionIsRefusedRatherThanGuessedAt() + { + Write("jp.txt", "こんにちは世界。テキスト", "shift_jis"); + Assert.Equal(0, Plan()); + + using (JsonDocument document = JsonDocument.Parse(File.ReadAllText(PlanPath))) + { + Dictionary fields = document.RootElement + .EnumerateObject() + .ToDictionary(p => p.Name, p => p.Value.Clone()); + + fields["PlanVersion"] = JsonSerializer.SerializeToElement(99); + File.WriteAllText(PlanPath, JsonSerializer.Serialize(fields)); + } + + Assert.Equal(1, Run("-Apply", PlanPath)); + } + + [Fact] + public void PlanAndApplyCannotRunTogether() + { + // Two commands with a human decision in between is the entire mechanism. + Write("jp.txt", "こんにちは世界。テキスト", "shift_jis"); + + Assert.Equal(1, Run( + "-BasePath", _root, "-Target", "utf-8", + "-Plan", PlanPath, "-Apply", PlanPath)); + } + + [Theory] + [InlineData("-Backup")] + [InlineData("-Target", "utf-16")] + [InlineData("-From", "koi8-r")] + [InlineData("-BasePath", ".")] + public void ApplyRejectsFlagsThePlanAlreadyFixes(params string[] flag) + { + // -Backup is the one that matters: silently ignoring it would let a user write + // what reads as an instruction to keep the originals and get a run that does not. + Write("jp.txt", "こんにちは世界。テキスト", "shift_jis"); + Assert.Equal(0, Plan()); + + Assert.Equal(1, Run(["-Apply", PlanPath, .. flag])); + } + + [Fact] + public void ApplyRejectsAPlanThatDoesNotExist() + { + Assert.Equal(1, Run("-Apply", Path.Combine(_root, "absent.json"))); + } + + [Theory] + [InlineData("-DetectOnly")] + [InlineData("-Validate", "utf-8")] + public void PlanIsRejectedInModesThatConvertNothing(params string[] mode) + { + Assert.Equal(1, Run( + ["-BasePath", _root, "-Target", "utf-8", "-Plan", PlanPath, .. mode])); + } + + [Fact] + public void TheSummaryAccountsForEverySelectedFile() + { + // A reader who cannot check that the parts sum to the whole has to trust the + // numbers instead, which is the opposite of what a preflight is for. + Write("jp.txt", "こんにちは世界。日本語のテキストです。", "shift_jis"); + Write("ambiguous.txt", "Le café était déjà prêt", "windows-1252"); + Write("plain.txt", "just ascii here", "ascii"); + + Assert.Equal(0, Plan()); + + ConversionPlan plan = LoadPlan(); + + Assert.Equal(3, plan.Files.Count); + Assert.Equal( + plan.Files.Count, + plan.Files.Count(f => f.Action == PlannedAction.Convert) + + plan.Files.Count(f => f.Action == PlannedAction.Unchanged) + + plan.Files.Count(f => f.Action == PlannedAction.Skip) + + plan.Files.Count(f => f.Action == PlannedAction.Refuse)); + + Assert.Contains("No files modified.", plan.Summarize()); + } +} diff --git a/sources/EncodingChecker/ConversionPlan.cs b/sources/EncodingChecker/ConversionPlan.cs new file mode 100644 index 0000000..d5a02e3 --- /dev/null +++ b/sources/EncodingChecker/ConversionPlan.cs @@ -0,0 +1,313 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace EncodingChecker; + +/// What EC intends to do with one file. +[JsonConverter(typeof(JsonStringEnumConverter))] +internal enum PlannedAction +{ + /// Convert it. + Convert, + + /// Already in the target encoding; nothing to do. + Unchanged, + + /// The encoding could not be identified, so it is left alone. + Skip, + + /// Converting it cannot be shown to be safe. + Refuse, +} + +/// One file's entry in a conversion plan. +internal sealed record PlannedFile +{ + public required string Path { get; init; } + + public required long Size { get; init; } + + /// + /// The file's bytes when the plan was made. What binds the plan to reality: if this + /// no longer matches, the plan describes a file that no longer exists. + /// + public required string Sha256 { get; init; } + + public required PlannedAction Action { get; init; } + + public required string SourceEncoding { get; init; } + + public required int SourceCodePage { get; init; } + + public required bool SourceHasBom { get; init; } + + public required bool SourceWasSpecified { get; init; } + + [JsonConverter(typeof(JsonStringEnumConverter))] + public required AmbiguityClass Ambiguity { get; init; } + + [JsonConverter(typeof(JsonStringEnumConverter))] + public required AmbiguityReason AmbiguityReason { get; init; } + + /// Encodings that read this file differently, when there are any. + public IReadOnlyList CompetingEncodings { get; init; } = []; + + /// Why this action, in words, when the action is not a plain conversion. + public string? Reason { get; init; } + + /// Whether converting this file could change its Unicode content. + public bool MayChangeText => Ambiguity == AmbiguityClass.TextChanging; +} + +/// +/// A conversion plan: what EC would do, recorded so it can be reviewed before anything is +/// changed and then executed exactly as reviewed. +/// +/// +/// The point is not the preview but the binding. A preview that is followed by a fresh +/// detection pass is a demonstration, not a promise: the second pass can reach different +/// conclusions, and the user approved the first. Every file therefore carries the hash it +/// had when the plan was made, and applying the plan verifies each one. A file that +/// changed in between invalidates the plan rather than being converted on the strength of +/// a decision made about different bytes. +/// +internal sealed record ConversionPlan +{ + public int PlanVersion { get; init; } = 1; + + public required string CreatedUtc { get; init; } + + public required string ECVersion { get; init; } + + public required string BaseDirectory { get; init; } + + public required string TargetEncoding { get; init; } + + public required bool TargetHasBom { get; init; } + + public required bool BackupEnabled { get; init; } + + /// The source encoding named by the caller, if any. + public string? ExplicitSourceEncoding { get; init; } + + public required IReadOnlyList Files { get; init; } + + private static readonly JsonSerializerOptions Options = new() + { + WriteIndented = true, + }; + + internal static ConversionPlan FromEntries( + IEnumerable entries, + string baseDirectory, + string targetEncoding, + bool targetHasBom, + bool backupEnabled, + string? explicitSource) + { + var files = new List(); + + foreach (ConversionReportEntry entry in entries) + { + PlannedAction action = entry.Result switch + { + ConversionRowResult.Unchanged => PlannedAction.Unchanged, + ConversionRowResult.Skipped => PlannedAction.Skip, + ConversionRowResult.Error => PlannedAction.Refuse, + _ => PlannedAction.Convert, + }; + + string hash; + long size; + + try + { + hash = ConversionMetadataStore.ComputeSha256(entry.FilePath); + size = new FileInfo(entry.FilePath).Length; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // A file that cannot be read now cannot be planned for. Recorded as a + // refusal rather than omitted, so the plan still accounts for it. + hash = string.Empty; + size = 0; + action = PlannedAction.Refuse; + } + + int codePage = 0; + + try + { + codePage = Encoding.GetEncoding(entry.SourceEncoding).CodePage; + } + catch (ArgumentException) + { + // Leave it at zero: an unrecognised label is itself part of the record. + } + + files.Add(new PlannedFile + { + Path = entry.FilePath, + Size = size, + Sha256 = hash, + Action = action, + SourceEncoding = entry.SourceEncoding, + SourceCodePage = codePage, + SourceHasBom = entry.SourceHasBom, + SourceWasSpecified = entry.SourceEncodingWasSpecified, + Ambiguity = entry.Ambiguity, + AmbiguityReason = entry.AmbiguityReason, + CompetingEncodings = entry.CompetingEncodings, + Reason = string.IsNullOrEmpty(entry.Diagnostic) ? null : entry.Diagnostic, + }); + } + + return new ConversionPlan + { + CreatedUtc = DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture), + ECVersion = typeof(ConversionPlan).Assembly.GetName().Version?.ToString() + ?? "unknown", + BaseDirectory = baseDirectory, + TargetEncoding = targetEncoding, + TargetHasBom = targetHasBom, + BackupEnabled = backupEnabled, + ExplicitSourceEncoding = explicitSource, + Files = files, + }; + } + + internal string? Save(string path) + { + try + { + File.WriteAllText( + path, JsonSerializer.Serialize(this, Options), new UTF8Encoding(false)); + return null; + } + catch (Exception ex) when ( + ex is IOException or UnauthorizedAccessException or JsonException) + { + return ex.Message; + } + } + + internal static ConversionPlan? Load(string path, out string? error) + { + try + { + ConversionPlan? plan = JsonSerializer.Deserialize( + File.ReadAllText(path)); + + if (plan is null) + { + error = $"'{path}' is empty."; + return null; + } + + if (plan.PlanVersion != 1) + { + error = $"Plan version {plan.PlanVersion} is not supported."; + return null; + } + + error = null; + return plan; + } + catch (Exception ex) when ( + ex is IOException or UnauthorizedAccessException or JsonException) + { + error = ex.Message; + return null; + } + } + + /// + /// Confirms every file is still exactly what it was when the plan was made. + /// + /// + /// The files that no longer match, empty when the plan is still valid. + /// + /// + /// Deliberately all-or-nothing. Converting the files that still match and skipping + /// the rest would apply a plan the user reviewed as a whole to a directory that is no + /// longer the one they reviewed, and the files most likely to have changed are the + /// ones something else is actively writing. + /// + internal IReadOnlyList FindStaleFiles() + { + var stale = new List(); + + foreach (PlannedFile file in Files) + { + if (file.Action != PlannedAction.Convert) + continue; + + try + { + if (!File.Exists(file.Path)) + { + stale.Add($"{file.Path} (no longer exists)"); + continue; + } + + if (ConversionMetadataStore.ComputeSha256(file.Path) != file.Sha256) + stale.Add($"{file.Path} (contents changed since the plan was made)"); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + stale.Add($"{file.Path} ({ex.Message})"); + } + } + + return stale; + } + + /// The summary a person reads before deciding. + /// + /// Written so the counts add up on the page. A reader who cannot see that the + /// sub-totals sum to the whole has to trust the numbers instead of checking them, + /// which is the opposite of what a preflight is for. + /// + internal string Summarize() + { + int Count(PlannedAction action) => Files.Count(f => f.Action == action); + + int convert = Count(PlannedAction.Convert); + int equivalent = Files.Count( + f => f.Action == PlannedAction.Convert + && f.Ambiguity == AmbiguityClass.TextEquivalent); + int changing = Files.Count(f => f.MayChangeText); + int otherRefusals = Count(PlannedAction.Refuse) - changing; + + var lines = new List + { + $"Selected: {Files.Count}", + string.Empty, + $"Will convert: {convert}", + $" encoding determined: {convert - equivalent}", + $" same text either way: {equivalent}", + $"Already in target encoding: {Count(PlannedAction.Unchanged)}", + $"Encoding not identified: {Count(PlannedAction.Skip)}", + $"Refused, ambiguous encoding: {changing}", + $"Refused, unreadable: {otherRefusals}", + string.Empty, + $"Backups: {(BackupEnabled ? "enabled" : "DISABLED")}", + $"Target: {TargetEncoding}" + + (TargetHasBom ? " with BOM" : " without BOM"), + }; + + if (!string.IsNullOrEmpty(ExplicitSourceEncoding)) + lines.Add($"Source encoding: {ExplicitSourceEncoding} (specified)"); + + lines.Add(string.Empty); + lines.Add("No files modified."); + + return string.Join(Environment.NewLine, lines); + } +} diff --git a/sources/EncodingChecker/EncodingAmbiguity.cs b/sources/EncodingChecker/EncodingAmbiguity.cs index 35f9062..11af0d6 100644 --- a/sources/EncodingChecker/EncodingAmbiguity.cs +++ b/sources/EncodingChecker/EncodingAmbiguity.cs @@ -282,8 +282,14 @@ .. byHash if (detectedIsDetermined) competing = []; + // Structurally determined is not the same as text-equivalent, and the difference + // matters as soon as anything shows the class to a user. The dismissed codecs here + // do read these bytes differently; they were set aside because they have no hold + // on them, not because they agree. Calling that "several codecs, same text" states + // something false about the file. AmbiguityClass classification = competing.Count > 0 ? AmbiguityClass.TextChanging + : detectedIsDetermined ? AmbiguityClass.Unambiguous : candidates.Count > 1 ? AmbiguityClass.TextEquivalent : AmbiguityClass.Unambiguous; diff --git a/sources/EncodingChecker/Program.cs b/sources/EncodingChecker/Program.cs index f3cddba..cb52d11 100644 --- a/sources/EncodingChecker/Program.cs +++ b/sources/EncodingChecker/Program.cs @@ -100,6 +100,8 @@ internal sealed class CliOptions internal List Exclude = []; internal string? Target; internal string? From; + internal string? PlanPath; + internal string? ApplyPath; internal string? ValidateCharsets; internal bool DetectOnly; internal string? ReportPath; @@ -147,6 +149,26 @@ output is still verified to hold exactly the same text, and a failed backup still aborts. Convert mode only. + Preflight: + [-Plan ] + Write a conversion plan and change nothing. The plan + records, for every file, what would happen and why: + the encoding, whether it was detected or specified, + whether the bytes identify it uniquely, and which + files could come out with different text. + + [-Apply ] + Carry out a plan written by -Plan. Every file is + checked against the hash it had when the plan was + made; if any has changed, nothing is converted. A + plan approved for one set of files is not applied to + a different one. + + Nothing is detected a second time: the encodings, + the target, and the backup setting all come from the + plan, so -BasePath, -Target, -From, and -Backup are + rejected here rather than silently ignored. + Modes: Conversion is the default mode. [-Validate ""] @@ -218,9 +240,121 @@ final summary line. -Report is unaffected. EncodingChecker.exe -BasePath . -Include "*.txt" -From "windows-1252" -Target "utf-8" + EncodingChecker.exe -BasePath . -Include "*" -Target "utf-8" -Plan plan.json + EncodingChecker.exe -Apply plan.json + EncodingChecker.exe -BasePath D:\NetworkShare -Include "*.txt" -Target "utf-8" -MaxParallelism 2 -FailOnChanges """; + /// + /// Carries out a plan written by -Plan, after confirming it still describes the + /// files on disk. + /// + private static int ApplyPlan(CliOptions options) + { + ConversionPlan? plan = ConversionPlan.Load(options.ApplyPath!, out string? loadError); + + if (plan is null) + { + Console.Error.WriteLine($"The plan could not be read: {loadError}"); + return 1; + } + + // The whole reason a plan exists. Re-detecting here would make the preview a + // demonstration rather than a promise: a second pass can reach different + // conclusions, and it was the first that the user approved. + IReadOnlyList stale = plan.FindStaleFiles(); + + if (stale.Count > 0) + { + Console.Error.WriteLine( + $"The plan no longer describes these files, so nothing was converted:"); + + foreach (string entry in stale.Take(20)) + Console.Error.WriteLine($" {entry}"); + + if (stale.Count > 20) + Console.Error.WriteLine($" ...and {stale.Count - 20} more."); + + Console.Error.WriteLine(); + Console.Error.WriteLine( + "Re-run -Plan to produce a plan for the files as they are now."); + return 3; + } + + List entries = + [ + .. plan.Files + .Where(f => f.Action == PlannedAction.Convert) + .Select(f => new ConversionReportEntry + { + FilePath = f.Path, + SourceEncoding = f.SourceEncoding, + SourceHasBom = f.SourceHasBom, + TargetEncoding = plan.TargetEncoding, + TargetHasBom = plan.TargetHasBom, + // The plan already settled this. Re-deriving it would be the second + // detection pass the plan exists to avoid. + Ambiguity = f.Ambiguity, + AmbiguityReason = f.AmbiguityReason, + CompetingEncodings = f.CompetingEncodings, + SourceEncodingWasSpecified = f.SourceWasSpecified, + }) + ]; + + var completed = new List(); + + using var cancellation = new CancellationTokenSource(); + Console.CancelKeyPress += (_, e) => + { + e.Cancel = true; + cancellation.Cancel(); + }; + + try + { + ScanEngine.ConvertFiles( + entries, + plan.TargetEncoding, + plan.TargetHasBom, + options.MaxParallelism ?? ScanEngine.DefaultMaxParallelism, + whatIf: false, + backup: plan.BackupEnabled, + completed.Add, + cancellation.Token); + } + catch (OperationCanceledException) + { + Console.Error.WriteLine("Cancelled."); + return 4; + } + + foreach (ConversionReportEntry entry in completed + .Where(e => e.Result == ConversionRowResult.Error)) + { + Console.Error.WriteLine($"Error: {entry.FilePath}: {entry.Diagnostic}"); + } + + Dictionary byResult = + completed.GroupBy(e => e.Result).ToDictionary(g => g.Key, g => g.Count()); + + int Count(ConversionRowResult result) => byResult.GetValueOrDefault(result); + + int failed = Count(ConversionRowResult.Error); + + // Every planned file is accounted for. A file that the plan scheduled but that + // the run left alone is the interesting case, so it must not disappear into a + // difference between two totals. + Console.Out.WriteLine( + $"Applied plan: {Count(ConversionRowResult.Converted)} converted, " + + $"{Count(ConversionRowResult.Unchanged)} already in the target encoding, " + + $"{Count(ConversionRowResult.Skipped)} skipped, " + + $"{failed} failed, " + + $"{plan.Files.Count - entries.Count} not scheduled for conversion."); + + return failed > 0 ? 3 : 0; + } + // Internal so ExitCodeContractTests can pin the exit codes, which are a published // CLI contract shared with LineEndingNormalizer. internal static int RunConsoleMode(string[] args) @@ -246,6 +380,13 @@ internal static int RunConsoleMode(string[] args) return 1; } + if (!string.IsNullOrWhiteSpace(options.ApplyPath)) + return ApplyPlan(options); + + // A plan is a dry run that is written down, so it must not modify anything. + if (!string.IsNullOrWhiteSpace(options.PlanPath)) + options.WhatIf = true; + ScanAction action = options.DetectOnly ? ScanAction.Detect : options.ValidateCharsets != null @@ -376,6 +517,43 @@ .. collectedEntries.OrderBy( } } + if (!string.IsNullOrWhiteSpace(options.PlanPath)) + { + ConversionPlan plan = ConversionPlan.FromEntries( + entries, + options.BasePath!, + targetCharset!, + targetWriteBom, + options.Backup, + options.From); + + string? saveError = plan.Save(options.PlanPath!); + + if (saveError != null) + { + Console.Error.WriteLine($"Failed to write the plan: {saveError}"); + return 3; + } + + if (!options.Quiet) + { + Console.Out.WriteLine(); + Console.Out.WriteLine(plan.Summarize()); + } + + // A refusal is one of the answers a preflight exists to give, so it does not + // make the preflight itself a failure. Returning 3 here would make the + // ordinary sequence - plan, read it, apply it - unreachable for exactly the + // directories this was built for. + if (options.FailOnChanges && + plan.Files.Any(f => f.Action == PlannedAction.Convert)) + { + return 2; + } + + return 0; + } + if (entries.Any(e => e.Result == ConversionRowResult.Error)) return 3; @@ -496,6 +674,22 @@ internal static bool TryParseArguments( } break; + case "plan": + if (!TryTakeValue(args, ref i, out options.PlanPath)) + { + error = "-Plan requires a value."; + return false; + } + break; + + case "apply": + if (!TryTakeValue(args, ref i, out options.ApplyPath)) + { + error = "-Apply requires a value."; + return false; + } + break; + case "validate": if (!TryTakeValue( args, @@ -573,7 +767,8 @@ internal static bool TryParseArguments( private static readonly HashSet KnownFlagNames = new(StringComparer.OrdinalIgnoreCase) { - "basepath", "include", "exclude", "target", "from", "validate", + "basepath", "include", "exclude", "target", "from", "plan", "apply", + "validate", "detectonly", "report", "maxparallelism", "failonchanges", "whatif", "backup", "quiet", "verbose", }; @@ -616,6 +811,57 @@ internal static bool TryValidateOptions( CliOptions options, [NotNullWhen(false)] out string? error) { + if (!string.IsNullOrWhiteSpace(options.PlanPath) && + !string.IsNullOrWhiteSpace(options.ApplyPath)) + { + error = "-Plan writes a plan and -Apply executes one; use them in " + + "separate runs so the plan can be reviewed in between."; + return false; + } + + if (!string.IsNullOrWhiteSpace(options.ApplyPath)) + { + if (!File.Exists(options.ApplyPath)) + { + error = $"The plan file '{options.ApplyPath}' does not exist."; + return false; + } + + if (options.DetectOnly || !string.IsNullOrWhiteSpace(options.ValidateCharsets)) + { + error = "-Apply performs a conversion; it cannot be combined with " + + "-DetectOnly or -Validate."; + return false; + } + + // The plan already fixes each of these, so accepting them here would let a + // user write a flag that reads as an instruction and is silently ignored - + // -Backup being the one that matters, since it would appear to ask for + // originals to be kept while the plan says otherwise. + string? overridden = + options.BasePath != null ? "-BasePath" + : options.Target != null ? "-Target" + : options.From != null ? "-From" + : options.Backup ? "-Backup" + : null; + + if (overridden != null) + { + error = $"{overridden} has no effect with -Apply: a plan already records " + + "the files, the source and target encodings, and whether " + + "originals are backed up. Re-run -Plan to change any of them."; + return false; + } + } + + if (!string.IsNullOrWhiteSpace(options.PlanPath) && + (options.DetectOnly || !string.IsNullOrWhiteSpace(options.ValidateCharsets))) + { + error = "-Plan previews a conversion; it cannot be combined with " + + "-DetectOnly or -Validate."; + return false; + } + if (!string.IsNullOrWhiteSpace(options.From)) { if (options.DetectOnly || !string.IsNullOrWhiteSpace(options.ValidateCharsets)) @@ -636,6 +882,13 @@ internal static bool TryValidateOptions( } } + // A plan already names every file, so -Apply supplies its own scope. + if (!string.IsNullOrWhiteSpace(options.ApplyPath)) + { + error = null; + return true; + } + if (string.IsNullOrWhiteSpace(options.BasePath)) { error = "-BasePath is required.";