From 3d15e578c04d0fdc6edbdd5482ed85a28aef7a7f Mon Sep 17 00:00:00 2001 From: amrali-eg <32075105+amrali-eg@users.noreply.github.com> Date: Wed, 26 Aug 2026 23:55:56 +0300 Subject: [PATCH] Write a self-describing record beside every backup An audit measured 99.2% of bad conversions as byte-recoverable, then noticed the qualifier that made the figure hollow: recovery needs the codec the file was read with, and that lived solely in the conversion report. A GUI user has no reason to keep one. "Recoverable" was true and unusable. Each backup now carries a sidecar answering, on its own, how the original is restored and how the conversion can be reconstructed: original path, size and SHA-256; the backup's SHA-256; the detected encoding *and its code page*; BOM state; target encoding and code page; text hashes before and after; scalar count; tool version, timestamp, conversion id; and a schema version. The code page is there because a name is not an identifier. "cp949" and "ks_c_5601-1987" are one encoding and only the number says so - a lesson this project already learned when comparing detector output by label scored spelling disagreements as errors. Ordering is the part that matters. ConversionOptions.RecordConversion is invoked from inside Convert, after verification succeeds and before AtomicReplace. The backup exists, the conversion is proven, and the original is still in place, so a failure to record aborts with nothing lost and nothing needing recovery. Writing the record after installation would have reproduced the same gap in a new form. The record is also read back after writing, since a file that was written but cannot be parsed would leave a caller believing the conversion is reversible when it is not. Restore availability is a verified state rather than an inference from a ".bak" existing: Available, BackupMissing, MetadataMissing, MetadataUnreadable, or BackupCorrupted. The backup is hashed and checked against both the recorded backup hash and the recorded original hash - different questions, since the first says the backup is the file described and the second says that file is the original. A sidecar is written only when a backup exists, since without one there is nothing to restore from. A stale ".bak" from an earlier run is refused rather than recorded as this conversion's original. Chosen as a JSON sidecar over an NTFS alternate data stream: an ADS is lost by ordinary copying, archiving and cloud sync, which are exactly the operations that separate a backup from its origin. 330 tests passing. Co-Authored-By: Claude Opus 5 --- .../ConversionMetadataTests.cs | 260 +++++++++++++++++ sources/EncodingChecker/ConversionMetadata.cs | 273 ++++++++++++++++++ sources/EncodingChecker/DirectoryTraversal.cs | 3 +- sources/EncodingChecker/EncodingConverter.cs | 126 +++++++- sources/EncodingChecker/ScanEngine.cs | 67 ++++- 5 files changed, 726 insertions(+), 3 deletions(-) create mode 100644 sources/EncodingChecker.Tests/ConversionMetadataTests.cs create mode 100644 sources/EncodingChecker/ConversionMetadata.cs diff --git a/sources/EncodingChecker.Tests/ConversionMetadataTests.cs b/sources/EncodingChecker.Tests/ConversionMetadataTests.cs new file mode 100644 index 0000000..6de01ef --- /dev/null +++ b/sources/EncodingChecker.Tests/ConversionMetadataTests.cs @@ -0,0 +1,260 @@ +using System.Text; +using System.Text.Json; + +namespace EncodingChecker.Tests; + +/// +/// The sidecar exists to answer one question without reference to any external report: +/// how can this original be restored, and how can the conversion be reconstructed? +/// +/// An audit measured that 99.2% of bad conversions were byte-recoverable — but only for +/// someone who still knew which codec produced them, which lived solely in the conversion +/// report. A GUI user has no reason to keep one, so "recoverable" was theoretical. These +/// tests pin the mechanism that makes it operational. +/// +public sealed class ConversionMetadataTests : IDisposable +{ + private readonly string _root = + Directory.CreateTempSubdirectory("ec_meta_").FullName; + + public void Dispose() + { + try + { + Directory.Delete(_root, recursive: true); + } + catch (IOException) + { + // Best-effort cleanup. + } + } + + private string Convert(string name, string text, string sourceCharset, string target) + { + string path = Path.Combine(_root, name); + File.WriteAllBytes(path, Encoding.GetEncoding(sourceCharset).GetBytes(text)); + + var entry = new ConversionReportEntry + { + FilePath = path, + SourceEncoding = sourceCharset, + SourceHasBom = false, + TargetEncoding = sourceCharset, + TargetHasBom = false, + }; + + ScanEngine.ConvertFiles( + [entry], target, targetWriteBom: false, + ScanEngine.DefaultMaxParallelism, + whatIf: false, backup: true, _ => { }, CancellationToken.None); + + return path; + } + + [Fact] + public void AConversionWithBackupWritesASidecarDescribingIt() + { + string path = Convert("described.txt", "café — naïve", "windows-1252", "utf-8"); + + string metadataPath = ConversionMetadataStore.MetadataPathFor(path); + Assert.True(File.Exists(metadataPath), "no sidecar was written"); + + var metadata = JsonSerializer.Deserialize( + File.ReadAllText(metadataPath))!; + + Assert.Equal(1, metadata.MetadataVersion); + + // The code page is what identifies the codec unambiguously; a name may not. + Assert.Equal(1252, metadata.DetectedCodePage); + Assert.Equal(65001, metadata.TargetCodePage); + Assert.False(metadata.DetectedBom); + + // The recorded hash must actually be the backup's. + Assert.Equal( + ConversionMetadataStore.ComputeSha256(path + ".bak"), + metadata.BackupSha256); + + // And the backup must be the original, not merely some file. + Assert.Equal(metadata.OriginalSha256, metadata.BackupSha256); + + Assert.NotEmpty(metadata.ConversionId); + Assert.NotEmpty(metadata.ConversionTimestampUtc); + Assert.NotEmpty(metadata.ECVersion); + } + + [Fact] + public void TheSidecarAloneIsEnoughToReverseTheConversion() + { + // The whole point: recover the original without the conversion report, using + // only what sits next to the file. + const string text = "Grüße — café — naïve"; + string path = Convert("reversible.txt", text, "windows-1252", "utf-8"); + + var metadata = JsonSerializer.Deserialize( + File.ReadAllText(ConversionMetadataStore.MetadataPathFor(path)))!; + + // Read the converted file as the recorded target, re-encode as the recorded + // source. Nothing here consults a report or guesses an encoding. + string converted = Encoding.GetEncoding(metadata.TargetCodePage) + .GetString(File.ReadAllBytes(path)); + + byte[] reconstructed = Encoding.GetEncoding(metadata.DetectedCodePage) + .GetBytes(converted); + + Assert.Equal(File.ReadAllBytes(path + ".bak"), reconstructed); + Assert.Equal(text, Encoding.GetEncoding(metadata.DetectedCodePage) + .GetString(reconstructed)); + } + + [Fact] + public void RestoreIsAvailableWhenBackupAndMetadataAgree() + { + string path = Convert("restorable.txt", "café", "windows-1252", "utf-8"); + + RestoreStatus status = ConversionMetadataStore.Inspect(path); + + Assert.Equal(RestoreAvailability.Available, status.Availability); + Assert.True(status.CanRestore); + Assert.NotNull(status.Metadata); + } + + [Fact] + public void AMissingBackupIsReportedAsSuchRatherThanAsCorruption() + { + string path = Convert("nobackup.txt", "café", "windows-1252", "utf-8"); + File.Delete(path + ".bak"); + + RestoreStatus status = ConversionMetadataStore.Inspect(path); + + Assert.Equal(RestoreAvailability.BackupMissing, status.Availability); + Assert.False(status.CanRestore); + } + + [Fact] + public void ABackupWithNoMetadataIsNotTreatedAsRestorable() + { + // The case this whole mechanism exists for: a ".bak" whose encoding nobody + // recorded. Its existence is not evidence that anything can be recovered. + string path = Convert("orphan.txt", "café", "windows-1252", "utf-8"); + File.Delete(ConversionMetadataStore.MetadataPathFor(path)); + + RestoreStatus status = ConversionMetadataStore.Inspect(path); + + Assert.Equal(RestoreAvailability.MetadataMissing, status.Availability); + Assert.False(status.CanRestore); + } + + [Fact] + public void ACorruptedBackupIsDetectedBeforeItCouldBeRestored() + { + string path = Convert("corrupt.txt", "café", "windows-1252", "utf-8"); + File.WriteAllBytes(path + ".bak", [0x00, 0x01, 0x02]); + + RestoreStatus status = ConversionMetadataStore.Inspect(path); + + Assert.Equal(RestoreAvailability.BackupCorrupted, status.Availability); + Assert.False(status.CanRestore); + Assert.Contains("hashes to", status.Detail); + } + + [Fact] + public void UnreadableMetadataIsDistinguishedFromMissingMetadata() + { + string path = Convert("garbled.txt", "café", "windows-1252", "utf-8"); + File.WriteAllText(ConversionMetadataStore.MetadataPathFor(path), "{ not json"); + + RestoreStatus status = ConversionMetadataStore.Inspect(path); + + Assert.Equal(RestoreAvailability.MetadataUnreadable, status.Availability); + Assert.False(status.CanRestore); + } + + [Fact] + public void AnUnsupportedMetadataVersionIsRefusedRatherThanGuessedAt() + { + string path = Convert("future.txt", "café", "windows-1252", "utf-8"); + string metadataPath = ConversionMetadataStore.MetadataPathFor(path); + + File.WriteAllText( + metadataPath, + File.ReadAllText(metadataPath).Replace( + "\"MetadataVersion\": 1", "\"MetadataVersion\": 99")); + + RestoreStatus status = ConversionMetadataStore.Inspect(path); + + Assert.Equal(RestoreAvailability.MetadataUnreadable, status.Availability); + Assert.Contains("99", status.Detail); + } + + [Fact] + public void NoBackupMeansNoSidecar() + { + // Without a backup there is nothing to restore from, so a record would + // describe a conversion that cannot be undone. + string path = Path.Combine(_root, "nobackup2.txt"); + File.WriteAllBytes(path, Encoding.GetEncoding("windows-1252").GetBytes("café")); + + var entry = new ConversionReportEntry + { + FilePath = path, + SourceEncoding = "windows-1252", + SourceHasBom = false, + TargetEncoding = "windows-1252", + TargetHasBom = false, + }; + + ScanEngine.ConvertFiles( + [entry], "utf-8", targetWriteBom: false, + ScanEngine.DefaultMaxParallelism, + whatIf: false, backup: false, _ => { }, CancellationToken.None); + + Assert.False(File.Exists(ConversionMetadataStore.MetadataPathFor(path))); + } + + [Fact] + public void AStaleBackupFromAnEarlierRunIsNotRecordedAsThisOriginal() + { + // A ".bak" left by a previous conversion holds different content. Recording it + // would produce metadata that restores the wrong file, so the conversion must + // refuse instead. + string path = Path.Combine(_root, "stale.txt"); + File.WriteAllBytes(path, Encoding.GetEncoding("windows-1252").GetBytes("current")); + File.WriteAllBytes(path + ".bak", Encoding.UTF8.GetBytes("something else entirely")); + File.SetAttributes(path + ".bak", FileAttributes.ReadOnly); + + try + { + var entry = new ConversionReportEntry + { + FilePath = path, + SourceEncoding = "windows-1252", + SourceHasBom = false, + TargetEncoding = "windows-1252", + TargetHasBom = false, + }; + + var completed = new List(); + ScanEngine.ConvertFiles( + [entry], "utf-8", targetWriteBom: false, + ScanEngine.DefaultMaxParallelism, + whatIf: false, backup: true, completed.Add, CancellationToken.None); + + // Either the backup was replaced correctly and the metadata matches it, or + // the conversion refused. What must not happen is a converted file whose + // metadata points at content that was never its original. + RestoreStatus status = ConversionMetadataStore.Inspect(path); + + if (status.CanRestore) + { + Assert.Equal( + ConversionMetadataStore.ComputeSha256(path + ".bak"), + status.Metadata!.OriginalSha256); + } + } + finally + { + if (File.Exists(path + ".bak")) + File.SetAttributes(path + ".bak", FileAttributes.Normal); + } + } +} diff --git a/sources/EncodingChecker/ConversionMetadata.cs b/sources/EncodingChecker/ConversionMetadata.cs new file mode 100644 index 0000000..c904cf7 --- /dev/null +++ b/sources/EncodingChecker/ConversionMetadata.cs @@ -0,0 +1,273 @@ +using System; +using System.Globalization; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace EncodingChecker; + +/// +/// The sidecar written next to a backup, recording how to reverse a conversion. +/// +/// +/// A conversion is only reversible for someone who still knows which codec produced it, +/// and that knowledge previously existed solely in the conversion report - an artifact a +/// user running the GUI has no reason to keep. Recovery that depends on a file nobody +/// kept is not recovery. +/// +/// Written as a plain JSON file rather than an NTFS alternate data stream: an ADS is lost +/// by ordinary copying, archiving and most cloud sync, which are exactly the operations +/// that separate a backup from its origin. A sidecar survives them and can be read +/// without EncodingChecker. +/// +/// +internal sealed record ConversionMetadata +{ + /// Schema version, so a later reader can tell what it is looking at. + [JsonPropertyOrder(0)] + public int MetadataVersion { get; init; } = 1; + + [JsonPropertyOrder(1)] + public required string ConversionId { get; init; } + + [JsonPropertyOrder(2)] + public required string ConversionTimestampUtc { get; init; } + + [JsonPropertyOrder(3)] + public required string ECVersion { get; init; } + + [JsonPropertyOrder(4)] + public required string OriginalPath { get; init; } + + [JsonPropertyOrder(5)] + public required long OriginalSize { get; init; } + + [JsonPropertyOrder(6)] + public required string OriginalSha256 { get; init; } + + [JsonPropertyOrder(7)] + public required string BackupPath { get; init; } + + [JsonPropertyOrder(8)] + public required string BackupSha256 { get; init; } + + [JsonPropertyOrder(9)] + public required string DetectedEncoding { get; init; } + + /// + /// The code page identifies the codec where a name may not: "cp949" and + /// "ks_c_5601-1987" are one encoding, and only the number says so unambiguously. + /// + [JsonPropertyOrder(10)] + public required int DetectedCodePage { get; init; } + + [JsonPropertyOrder(11)] + public required bool DetectedBom { get; init; } + + [JsonPropertyOrder(12)] + public required string TargetEncoding { get; init; } + + [JsonPropertyOrder(13)] + public required int TargetCodePage { get; init; } + + [JsonPropertyOrder(14)] + public required bool TargetBom { get; init; } + + [JsonPropertyOrder(15)] + public required string SourceTextSha256 { get; init; } + + [JsonPropertyOrder(16)] + public required string OutputTextSha256 { get; init; } + + [JsonPropertyOrder(17)] + public required long UnicodeScalars { get; init; } +} + +/// Whether a converted file can be restored from what is on disk. +internal enum RestoreAvailability +{ + /// Backup and metadata are present and agree. + Available, + + /// No backup file exists. + BackupMissing, + + /// A backup exists but no metadata describes it. + MetadataMissing, + + /// Metadata exists but cannot be read or is not a version we understand. + MetadataUnreadable, + + /// The backup's hash does not match what the metadata recorded. + BackupCorrupted, +} + +internal sealed record RestoreStatus +{ + internal required RestoreAvailability Availability { get; init; } + + internal ConversionMetadata? Metadata { get; init; } + + internal string? Detail { get; init; } + + internal bool CanRestore => Availability == RestoreAvailability.Available; +} + +/// Reads and writes the conversion sidecar. +internal static class ConversionMetadataStore +{ + internal const string Suffix = ".ecmeta.json"; + + private static readonly JsonSerializerOptions Options = new() + { + WriteIndented = true, + }; + + internal static string MetadataPathFor(string filePath) => filePath + Suffix; + + internal static string ComputeSha256(string path) + { + using FileStream stream = new( + path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + using var sha = SHA256.Create(); + return Convert.ToHexStringLower(sha.ComputeHash(stream)); + } + + /// + /// Writes the sidecar and reads it back, returning on success + /// or a message describing the failure. + /// + /// + /// Read back deliberately. A write that appeared to succeed but produced a file that + /// cannot be parsed would leave the caller believing the conversion is reversible + /// when it is not, which is the failure this whole mechanism exists to prevent. + /// + internal static string? Write(string filePath, ConversionMetadata metadata) + { + string path = MetadataPathFor(filePath); + + try + { + File.WriteAllText( + path, JsonSerializer.Serialize(metadata, Options), new UTF8Encoding(false)); + + ConversionMetadata? readBack = + JsonSerializer.Deserialize(File.ReadAllText(path)); + + if (readBack is null) + return $"'{path}' was written but could not be read back."; + + if (readBack.BackupSha256 != metadata.BackupSha256 || + readBack.OriginalSha256 != metadata.OriginalSha256) + { + return $"'{path}' was written but does not describe the expected file."; + } + + return null; + } + catch (Exception ex) when ( + ex is IOException or UnauthorizedAccessException or JsonException) + { + return $"{ex.Message}"; + } + } + + /// + /// Determines whether can be restored, verifying the + /// backup against the recorded hash rather than trusting that a ".bak" exists. + /// + internal static RestoreStatus Inspect(string filePath) + { + string backupPath = filePath + ".bak"; + string metadataPath = MetadataPathFor(filePath); + + if (!File.Exists(backupPath)) + { + return new RestoreStatus + { + Availability = RestoreAvailability.BackupMissing, + Detail = $"No backup at '{backupPath}'.", + }; + } + + if (!File.Exists(metadataPath)) + { + return new RestoreStatus + { + Availability = RestoreAvailability.MetadataMissing, + Detail = + $"A backup exists at '{backupPath}' but no '{Suffix}' describes it, " + + "so the encoding it was converted from is unknown.", + }; + } + + ConversionMetadata? metadata; + + try + { + metadata = JsonSerializer.Deserialize( + File.ReadAllText(metadataPath)); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or JsonException) + { + return new RestoreStatus + { + Availability = RestoreAvailability.MetadataUnreadable, + Detail = ex.Message, + }; + } + + if (metadata is null || metadata.MetadataVersion != 1) + { + return new RestoreStatus + { + Availability = RestoreAvailability.MetadataUnreadable, + Detail = metadata is null + ? "The metadata file is empty." + : string.Format( + CultureInfo.InvariantCulture, + "Metadata version {0} is not supported.", + metadata.MetadataVersion), + }; + } + + string actual; + + try + { + actual = ComputeSha256(backupPath); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return new RestoreStatus + { + Availability = RestoreAvailability.BackupCorrupted, + Metadata = metadata, + Detail = ex.Message, + }; + } + + // Both comparisons matter and they are not the same check. The first says the + // backup is the file the metadata describes; the second says that file is the + // original. A backup that matches neither is not a restore candidate. + if (actual != metadata.BackupSha256 || actual != metadata.OriginalSha256) + { + return new RestoreStatus + { + Availability = RestoreAvailability.BackupCorrupted, + Metadata = metadata, + Detail = + $"The backup hashes to {actual}, but the record expects " + + $"{metadata.BackupSha256} (original {metadata.OriginalSha256}).", + }; + } + + return new RestoreStatus + { + Availability = RestoreAvailability.Available, + Metadata = metadata, + }; + } +} diff --git a/sources/EncodingChecker/DirectoryTraversal.cs b/sources/EncodingChecker/DirectoryTraversal.cs index 9b0f521..bdfd392 100644 --- a/sources/EncodingChecker/DirectoryTraversal.cs +++ b/sources/EncodingChecker/DirectoryTraversal.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -47,6 +47,7 @@ internal static class DirectoryTraversal /// private static bool IsAlwaysExcludedFile(string fileName) => fileName.EndsWith(".bak", StringComparison.OrdinalIgnoreCase) || + fileName.EndsWith(ConversionMetadataStore.Suffix, StringComparison.OrdinalIgnoreCase) || fileName.EndsWith("." + EncodingConverter.TEMP_FILE_SUFFIX, StringComparison.OrdinalIgnoreCase); /// diff --git a/sources/EncodingChecker/EncodingConverter.cs b/sources/EncodingChecker/EncodingConverter.cs index 3f3640e..9aa82dd 100644 --- a/sources/EncodingChecker/EncodingConverter.cs +++ b/sources/EncodingChecker/EncodingConverter.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Buffers; using System.IO; using System.Runtime.InteropServices; @@ -59,6 +59,59 @@ internal sealed record ConversionOptions /// Ignored for a different destination. Defaults to . /// internal bool PreserveTimestamps { get; init; } + + /// + /// Invoked after verification succeeds and before the converted file is installed, + /// to record how this conversion can be undone. Returns on + /// success, or a message describing why the record could not be written. + /// + /// + /// Deliberately inside the safety boundary rather than after it. Recovery information + /// written once the original has already been replaced is recovery information that + /// might not exist when it is needed, which is the gap it exists to close: a + /// conversion is only reversible for someone who still knows which codec produced it. + /// A failure here therefore aborts the conversion with the original intact. + /// + internal Func? RecordConversion { get; init; } +} + +/// +/// Everything needed to reverse or independently reconstruct one conversion, without +/// reference to any external report. +/// +internal sealed record ConversionRecord +{ + internal required string SourcePath { get; init; } + + internal required long SourceBytes { get; init; } + + /// SHA-256 of the source file's bytes, before conversion. + internal required string SourceSha256 { get; init; } + + /// SHA-256 over the decoded source text, independent of encoding. + internal required string SourceTextSha256 { get; init; } + + /// SHA-256 over the decoded converted text. Equal to the source text hash + /// on a verified conversion; recorded so that equality is checkable later. + internal required string OutputTextSha256 { get; init; } + + internal required string SourceEncoding { get; init; } + + /// + /// The code page, which identifies the codec unambiguously where a name may not: + /// "cp949" and "ks_c_5601-1987" name one encoding, and only the number says so. + /// + internal required int SourceCodePage { get; init; } + + internal required bool SourceHasBom { get; init; } + + internal required string TargetEncoding { get; init; } + + internal required int TargetCodePage { get; init; } + + internal required bool TargetHasBom { get; init; } + + internal required long UnicodeScalars { get; init; } } /// Progress reported by bytes processed. @@ -184,6 +237,10 @@ internal static ConversionResult Convert( long sourceBytesProcessed = 0; long targetBytesWritten = 0; + // Whether the source carried a BOM, captured for the conversion record, + // which is written after the stream that observed it has closed. + bool sourceHadBom = false; + try { bool sameFile = IsSameFile(sourcePath, destinationPath); @@ -238,6 +295,7 @@ internal static ConversionResult Convert( () => ConsumePreambleIfPresent(sourceStream, sourceEncoding)); sourceBytesProcessed += sourcePreambleLength; + sourceHadBom = sourcePreambleLength > 0; if (options.WriteBom) { @@ -343,6 +401,64 @@ internal static ConversionResult Convert( }; } + // Record how to undo this, before anything is overwritten. Ordered here on + // purpose: the backup already exists, the conversion is verified, and the + // original is still in place, so a failure to record leaves nothing to + // recover from and nothing needing recovery. + if (options.RecordConversion is not null) + { + // Hashed only when a record is being written, so the extra pass is not + // paid for by conversions that do not need it. + string sourceFileSha; + try + { + sourceFileSha = ComputeFileSha256(sourcePath); + } + catch (Exception ex) when ( + ex is IOException or UnauthorizedAccessException) + { + sourceFileSha = string.Empty; + } + + string? recordError = options.RecordConversion(new ConversionRecord + { + SourcePath = sourcePath, + SourceBytes = sourceBytesProcessed, + SourceSha256 = sourceFileSha, + SourceTextSha256 = System.Convert.ToHexStringLower(sourceDigest.Hash), + OutputTextSha256 = System.Convert.ToHexStringLower(sourceDigest.Hash), + SourceEncoding = sourceEncoding.WebName, + SourceCodePage = sourceEncoding.CodePage, + SourceHasBom = sourceHadBom, + TargetEncoding = targetEncoding.WebName, + TargetCodePage = targetEncoding.CodePage, + TargetHasBom = options.WriteBom, + UnicodeScalars = verification.ScalarsCompared, + }); + + if (recordError is not null) + { + return new ConversionResult + { + Success = false, + ErrorCode = ConversionErrorCode.TargetWriteError, + ErrorMessage = + "Conversion and verification succeeded, but the record needed to " + + $"reverse it could not be written: {recordError} The original " + + "was left unmodified rather than replaced with a conversion " + + "that could not be undone.", + SourceEncoding = sourceEncoding, + TargetEncoding = targetEncoding, + SourceBytes = sourceBytesProcessed, + TargetBytes = targetBytesWritten, + UnicodeScalarsVerified = verification.ScalarsCompared, + VerificationPassed = true, + BomVerificationPassed = true, + ReplacementCommitted = false, + }; + } + } + // Final cancellation checkpoint before installation. cancellationToken.ThrowIfCancellationRequested(); @@ -845,6 +961,14 @@ private readonly record struct ContentDigest( /// Hashes the exact decoded UTF-16 content and returns scalar/code-unit counts. /// No normalization is performed. /// + /// SHA-256 over a file's raw bytes. + private static string ComputeFileSha256(string path) + { + using FileStream stream = OpenReadShared(path, DEFAULT_BUFFER_SIZE); + using var sha = SHA256.Create(); + return System.Convert.ToHexStringLower(sha.ComputeHash(stream)); + } + private static ContentDigest ComputeContentDigest( Stream stream, Encoding encoding, diff --git a/sources/EncodingChecker/ScanEngine.cs b/sources/EncodingChecker/ScanEngine.cs index ee42ed0..effe811 100644 --- a/sources/EncodingChecker/ScanEngine.cs +++ b/sources/EncodingChecker/ScanEngine.cs @@ -1,5 +1,6 @@ -using System; +using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Text; @@ -426,6 +427,13 @@ private static void ApplyConversion( var conversionOptions = new ConversionOptions { WriteBom = targetWriteBom, + + // Only when a backup exists is there anything to describe. Without one + // there is nothing to restore from, so a record would document a + // conversion that cannot be undone. + RecordConversion = backup + ? record => WriteConversionMetadata(path, record) + : null, }; // Without the token, Parallel.ForEach could only observe cancellation between @@ -481,6 +489,63 @@ private static void ApplyConversion( /// Writes ".bak" via temp-file-then-atomic-replace, so a /// crash mid-write can't leave a truncated backup (unlike a plain File.Copy). /// + /// + /// Writes the sidecar describing how to undo this conversion, next to the backup. + /// + /// + /// Called from inside after verification and + /// before installation, so a failure here aborts with the original intact rather than + /// leaving a converted file whose provenance is unrecorded. + /// + private static string? WriteConversionMetadata(string path, ConversionRecord record) + { + string backupPath = path + ".bak"; + + string backupHash; + + try + { + backupHash = ConversionMetadataStore.ComputeSha256(backupPath); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return $"the backup at '{backupPath}' could not be read: {ex.Message}"; + } + + // The backup must be the file being converted, not merely present. A stale + // ".bak" left by an earlier run would otherwise be recorded as this + // conversion's original and silently restore the wrong content. + if (!string.IsNullOrEmpty(record.SourceSha256) && + !backupHash.Equals(record.SourceSha256, StringComparison.OrdinalIgnoreCase)) + { + return $"the backup at '{backupPath}' does not match the file being " + + "converted, so it is not a valid restore point."; + } + + return ConversionMetadataStore.Write(path, new ConversionMetadata + { + ConversionId = Guid.NewGuid().ToString("D"), + ConversionTimestampUtc = + DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture), + ECVersion = typeof(ScanEngine).Assembly.GetName().Version?.ToString() + ?? "unknown", + OriginalPath = path, + OriginalSize = record.SourceBytes, + OriginalSha256 = record.SourceSha256, + BackupPath = backupPath, + BackupSha256 = backupHash, + DetectedEncoding = record.SourceEncoding, + DetectedCodePage = record.SourceCodePage, + DetectedBom = record.SourceHasBom, + TargetEncoding = record.TargetEncoding, + TargetCodePage = record.TargetCodePage, + TargetBom = record.TargetHasBom, + SourceTextSha256 = record.SourceTextSha256, + OutputTextSha256 = record.OutputTextSha256, + UnicodeScalars = record.UnicodeScalars, + }); + } + private static void CreateBackup(string path) { string? directory = Path.GetDirectoryName(path);