Prepare EncodingChecker v3.8.0

.github/workflows/detector-parity.yml

@@ -0,0 +1,47 @@
+name: Shared Unicode detector parity
+
+on:
+  push:
+    branches: [ master ]
+  schedule:
+    - cron: "17 3 * * 1"
+  workflow_dispatch:
+
+jobs:
+  parity:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          path: encoding-checker
+
+      - uses: actions/checkout@v7
+        with:
+          repository: amrali-eg/LineEndingNormalizer
+          ref: master
+          path: line-ending-normalizer
+
+      - uses: actions/checkout@v7
+        with:
+          repository: amrali-eg/CorpusTesters
+          ref: master
+          path: corpus-testers
+
+      - name: Verify shared Unicode detector
+        shell: pwsh
+        run: |
+          $ec = Get-Content 'encoding-checker/sources/EncodingChecker/UnicodeDetector.cs' -Raw
+          $len = Get-Content 'line-ending-normalizer/UnicodeDetector.cs' -Raw
+          $corpus = Get-Content 'corpus-testers/CorpusTesting/UnicodeDetector.cs' -Raw
+          $ec = $ec -replace 'namespace EncodingChecker;', 'namespace SharedUnicodeDetector;'
+          $len = $len -replace 'namespace LineEndingNormalizer;', 'namespace SharedUnicodeDetector;'
+          $corpus = $corpus -replace 'namespace CorpusTesting;', 'namespace SharedUnicodeDetector;'
+          $ec = $ec -replace '^using System;\r?\n', ''
+          $len = $len -replace '^using System;\r?\n', ''
+          $corpus = $corpus -replace '^using System;\r?\n', ''
+          $ec = [regex]::Replace($ec, "`r?`n", "`n").TrimStart([char]0xFEFF)
+          $len = [regex]::Replace($len, "`r?`n", "`n").TrimStart([char]0xFEFF)
+          $corpus = [regex]::Replace($corpus, "`r?`n", "`n").TrimStart([char]0xFEFF)
+          if ($ec -cne $len -or $ec -cne $corpus) {
+            throw 'UnicodeDetector.cs differs between EncodingChecker, LineEndingNormalizer, and CorpusTesters. Keep the shared algorithm synchronized; application-specific TextEncoding wrappers may differ.'
+          }

RELEASE-CHECKLIST.md

@@ -6,9 +6,10 @@ cannot answer.
 ## Automated
 
 - [ ] `dotnet test sources/EncodingChecker.Tests/EncodingChecker.Tests.csproj -c Release` — all green.
-- [ ] The 1,033-file oracle sentinel set still agrees with GNU libiconv and ICU.
-- [ ] Detector-drift check passes (the detector sources are duplicated across three
-      repositories and nothing enforces the sync; a fix in one is a fix owed to all three).
+- [ ] The scheduled **Shared Unicode detector parity** workflow is green. It compares
+      the shared detector source in EncodingChecker, LineEndingNormalizer, and
+      CorpusTesters after normalizing namespace, a redundant `using System` import,
+      and line-ending differences.
 
 ## Manual: the GUI smoke test
 
@@ -33,45 +34,35 @@ record the file's SHA-256 before and after:
 Get-FileHash -Algorithm SHA256 <path> | Select-Object -ExpandProperty Hash
 ```
 
-### Test files
-
-| name | contents | encoding | expected classification |
-|---|---|---|---|
-| `jp.txt` | `こんにちは世界。日本語のテキストです。` | Shift_JIS | unambiguous |
-| `french.txt` | `Le café était déjà prêt` | windows-1252 | text-changing |
-| `russian.txt` | `Привет мир, это русский текст` | koi8-r | text-changing |
-| `plain.txt` | `plain ascii, no high bytes at all` | ASCII | text-equivalent |
-
-### Structure it in phases, not one long sequence
-
-The first version of this test used one folder and one final check for the whole matrix.
-That cannot work, and the reason is worth keeping: the stale-plan case **stops the entire
-run**, so every "must have converted" expectation after it is unreachable by construction.
-Worse, the state it leaves is byte-identical to "the tester cancelled everything", so the
-result cannot say *which* protection fired. The first real run produced a FAIL that was
-entirely the instrument's fault, and only inspecting the bytes by hand showed the product
-had behaved correctly.
-
-Each phase therefore gets its own folder, its own short click sequence, and its own check,
-and proves exactly one property. [`tools/gui-smoke-test.py`](tools/gui-smoke-test.py) does the setup and the
-verification; it also refuses to pass a phase whose defining action was skipped — a phase
-that silently tests nothing is the failure mode a manual matrix is most prone to.
-
-### Matrix
-
-| # | step | expected |
-|---|---|---|
-| 1 | **View** the directory | 4 files listed with their encodings |
-| 2 | Tick all, **Convert** to utf-8 | confirmation appears; two files listed as needing an explicit source encoding, with competing encodings named |
-| 3 | **Cancel** | nothing converted; **all four hashes unchanged**; no `.bak` files |
-| 4 | Convert again; untick `russian.txt`; choose `windows-1252` | button reads "Use this encoding for 1 file(s)" |
-| 5 | Confirm the re-planned conversion | `french.txt` converts and reads correctly as French |
-| 6 | Check `russian.txt` | **hash unchanged**; still refused |
-| 7 | Convert again; while the dialog is open, edit one selected file in another editor and save | — |
-| 8 | Confirm | run stops; message names the changed file; **every hash unchanged** |
-| 9 | Convert `jp.txt` alone, backups on | converts; `jp.txt.bak` and `jp.txt.ecmeta.json` present; text reads correctly |
-| 10 | Create a **directory** named `<file>.bak` beside a file, convert it | conversion refused; **source hash unchanged** |
-| 11 | Export report → **Conversion journal (\*.json)** | journal written; refused files present with their competing encodings; `Sha256After` null for everything not converted |
+### Core GUI smoke test
+
+[`tools/gui-smoke-test.py`](tools/gui-smoke-test.py) creates disposable folders on the
+Desktop and verifies the resulting bytes. For every phase, set the printed folder as
+**Directory to check** and choose **utf-8** in **Convert to**. Then run each short phase
+with the Release build:
+
+```powershell
+python tools/gui-smoke-test.py setup A
+# perform the displayed GUI steps
+python tools/gui-smoke-test.py verify A
+```
+
+| Phase | What the GUI check proves |
+|---|---|
+| A | **View** lists the prepared files; Unicode and ASCII are ready, legacy files need a source choice; **Cancel** changes no bytes and creates no recovery files. |
+| B | Unicode and ASCII convert without a source choice and preserve their exact text. |
+| C | A chosen legacy source encoding applies only to the ticked files; unselected legacy files stay unchanged. |
+
+The script verifies hashes and decoded output; status messages alone never count as evidence.
+
+### Accessibility spot check
+
+- [ ] At 100%, 125%, and 150% display scaling, the review text, source-encoding
+      chooser, and its confirmation button are fully visible without horizontal scrolling.
+- [ ] Keyboard-only: Tab reaches the legacy-file list, source chooser, and both final
+      actions; Enter performs only the displayed ready conversion; Escape cancels.
+- [ ] In a Windows high-contrast theme, the review outcomes and legacy warning remain
+      readable and distinguishable.
 
 ### Record
 
@@ -86,50 +77,15 @@ Windows version:
 Date:
 Tester:
 
-Phase A (refuse + cancel change nothing):   PASS / FAIL
-Phase B (explicit source, scoped):          PASS / FAIL
-Phase C (stale plan stops the whole run):   PASS / FAIL
-Phase D (backup + record; backup failure):  PASS / FAIL
+Phase A (review + cancel):                  PASS / FAIL
+Phase B (Unicode + ASCII conversion):       PASS / FAIL
+Phase C (scoped legacy source choice):      PASS / FAIL
 
 Cases where observed differed from expected:
 
 Result: PASS / FAIL
 ```
 
-### Run of 2026-08-27
-
-```text
-EC version:      3.7.0.0
-Commit:          a201a08
-Windows version: Microsoft Windows NT 10.0.26200.0
-.NET version:    10.0.400
-Date:            2026-08-27
-Tester:          amrali-eg
-
-Phase A (refuse + cancel change nothing):   PASS
-Phase B (explicit source, scoped):          PASS
-Phase C (stale plan stops the whole run):   PASS
-Phase D (backup + record; backup failure):  PASS
-
-Result: PASS
-```
-
-Notes from that run, kept because they qualify what the phases actually establish:
-
-- **Phase B proves less on its own than it appears to.** Its French sample decodes
-  identically under windows-1252 and iso-8859-1, so "the text is preserved" cannot show
-  which codec was used. What settles it is the recovery record: `french.txt.ecmeta.json`
-  gives `DetectedCodePage: 1252`, the encoding chosen in the dialog rather than the
-  `iso-8859-1` that detection proposed. A future revision should use content where the two
-  encodings genuinely disagree, so the assertion stands without the sidecar.
-- **Text-equivalent ambiguity is nearly unreachable.** Eight ASCII shapes — short strings,
-  digits, JSON, code, newlines — all classify as `StructurallyDetermined`, because ASCII
-  constrains every byte below 0x80. Only a **one-byte file** reaches `TextEquivalent`,
-  where no codec that decodes it at all can read it differently. The middle class of the
-  three-way taxonomy is far rarer in practice than the taxonomy suggests. The classifier
-  is right in both cases; the corpus has to be contrived to exercise it, and `tiny.txt`
-  exists for that reason alone.
-
 ## Documentation
 
 - [ ] README figures match the current audit run; no stale counts.

docs/CONVERSION-WORKFLOW.md

@@ -0,0 +1,63 @@
+# How conversion works
+
+This page explains what happens after you ask EncodingChecker to convert files. It is the same safety model in the GUI, the command line, and saved conversion plans.
+
+## What you do
+
+1. **View** the folder to see what EC found.
+2. Select the files you want to handle and choose **Convert**.
+3. Read the review before any file is changed.
+4. For legacy text, choose the source encoding if you know it.
+5. Confirm the reviewed conversion.
+
+The review tells you which files will convert, already match the target, need a legacy source choice, or cannot be processed. Cancelling leaves every source file unchanged.
+
+## The important rule
+
+| File type | What EC does automatically |
+| --- | --- |
+| Unicode or ASCII | May convert it |
+| Legacy text | Leaves it unchanged until you choose the original encoding |
+| Unknown or unreadable data | Leaves it unchanged |
+
+Choosing a legacy encoding answers only “how should these bytes be read?” It does not disable strict decoding, output verification, backups, or atomic installation.
+
+## What EC does
+
+```mermaid
+flowchart LR
+    A[Scan files] --> B[Decide source interpretation]
+    B --> C[Build review plan]
+    C --> D[User confirms]
+    D --> E[Strict source decode]
+    E --> F[Strict target encode]
+    F --> G[Verify identical text]
+    G --> H[Backup and metadata]
+    H --> I[Install verified output]
+```
+
+Every step after confirmation must succeed. If decoding, encoding, verification, backup creation, or installation fails, EC leaves that source file unchanged.
+
+## Plans and the command line
+
+For a cautious batch workflow, create a plan first:
+
+```powershell
+EncodingChecker.exe -BasePath "C:\Files" -Target utf-8 -Plan plan.json
+```
+
+After reviewing it, apply that exact plan:
+
+```powershell
+EncodingChecker.exe -Apply plan.json
+```
+
+The plan contains the files’ hashes and conversion settings. If a scheduled file changes after review, EC rejects the whole plan instead of applying an approval to different bytes.
+
+For a known legacy source, supply the encoding explicitly:
+
+```powershell
+EncodingChecker.exe -BasePath "C:\Files" -Target utf-8 -From windows-1252 -Backup
+```
+
+For the detailed guarantees and known limits, read [Safety and audit](SAFETY-AUDIT.md).

docs/RELEASE-NOTES-v3.8.0.md

@@ -0,0 +1,24 @@
+# EncodingChecker v3.8.0
+
+## Safer legacy conversion
+
+- Unicode and ASCII files continue to convert automatically.
+- Detected legacy text is now left unchanged until its original source encoding is
+  explicitly chosen in the GUI or supplied with `-From` on the command line.
+- An explicit source encoding still uses strict decoding, verified output, backup
+  checks, and atomic replacement; it is not a safety bypass.
+
+## Clearer review and export
+
+- The conversion review states which files are ready, which need a source encoding,
+  and which EC will leave unchanged.
+- Legacy source choices apply only to the ticked files and show their scope clearly.
+- **Export results** now offers a CSV report and, after conversion, a JSON conversion
+  history.
+
+## Reliability and maintainability
+
+- Saved plans bind approved file hashes and conversion semantics before `-Apply`.
+- Backup sidecars preserve the source codec and conversion provenance.
+- The shared Unicode detector is checked for parity across EncodingChecker,
+  LineEndingNormalizer, and CorpusTesters.

docs/SAFETY-AUDIT.md

@@ -0,0 +1,72 @@
+# EncodingChecker safety and audit
+
+This document is the technical companion to the main [README](../README.md). It explains what EC's conversion pipeline guarantees, what it does not guarantee, and how those claims are checked independently.
+
+## Conversion safety boundary
+
+For each file that EC is allowed to convert, the engine:
+
+1. strictly decodes the source encoding;
+2. strictly encodes the requested target encoding into a temporary file;
+3. strictly decodes that temporary output and compares the exact Unicode scalar sequence with the source text;
+4. if backups are enabled, creates and verifies `<file>.bak` plus recovery metadata;
+5. installs the verified temporary file atomically where the platform supports it.
+
+Any decode, encode, verification, backup, or write failure leaves the source file unchanged. No normalization, case folding, whitespace rewriting, or replacement-character fallback is used to make a conversion appear successful.
+
+The source is not rewritten in place. File attributes and timestamps are applied to the temporary output before installation. EC skips its own backups and temporary files on subsequent scans.
+
+## Source-encoding policy
+
+Encoding identification and text preservation are different questions. A sequence of legacy bytes often cannot prove which historical single-byte code page produced it.
+
+EC therefore has a simple policy:
+
+| Source interpretation | Automatic conversion |
+| --- | --- |
+| Unicode or ASCII | Allowed |
+| Legacy codec supplied explicitly by the user | Allowed, subject to all safety checks |
+| Legacy codec detected automatically | Refused; choose the source codec first |
+| Unknown or unreadable source | Not converted |
+
+`-From` and the GUI source chooser replace detection only. They do not bypass strict decoding, output verification, backup verification, or atomic installation.
+
+## Plans, confirmation, and recovery
+
+`-Plan` writes a conversion plan without changing files. The plan contains the source hashes, paths relative to its declared root, target and BOM policy, source-selection mode, backup setting, and conversion-semantics version.
+
+`-Apply` rejects changed, missing, relocated, or incompatible planned work as a whole; it does not silently apply the remaining files. EC also rechecks the source hash immediately before installation. That narrows, but cannot eliminate, a narrow concurrent-writer TOCTOU window between the final check and replacement.
+
+The GUI uses the same policy and plan model. It displays a review before writing, and a changed source while that review is open invalidates the run.
+
+With backups enabled, each conversion has a portable `<file>.ecmeta.json` sidecar. The sidecar records the source codec actually used, whether it was detected or explicitly selected, source and backup hashes, target/BOM policy, conversion timestamp, and version. Recovery verifies the backup against that metadata before restoring it.
+
+`-Journal` provides the batch-level record: EC's detected or explicit source, policy decision, planned action, actual outcome, and before/after hashes for every file—including skipped and refused ones.
+
+## Strict-codec defect fixed in v3.6.0
+
+The independent audit found that assigning `Decoder.Fallback` or `Encoder.Fallback` after calling `GetDecoder()` or `GetEncoder()` does not reliably make .NET `CodePagesEncodingProvider` codecs strict. Some malformed legacy input could be silently substituted while EC's old downstream content check still reported success.
+
+EC now constructs strict code-page encodings with exception fallbacks at `Encoding.GetEncoding(...)` construction time. Permanent regression tests cover the previously permissive decoder and encoder paths.
+
+## Independent audit
+
+[CorpusTesters](https://github.com/amrali-eg/CorpusTesters) is a separate, reproducible audit harness. It runs EC against four public corpora:
+
+- [UnicodeTestSuite](https://github.com/amrali-eg/UnicodeTestSuite)
+- [chardet test-data](https://github.com/chardet/test-data)
+- [char-dataset](https://github.com/Ousret/char-dataset)
+- [UTF-unknown](https://github.com/CharsetDetector/UTF-unknown)
+
+It operates on working copies, never source corpora. For each file with authoritative metadata, it compares the exact decoded source text against strict UTF output. It also verifies backup hashes, inventories every file, runs mutation controls, checks codec strictness, and keeps per-file CSV/JSON evidence.
+
+The audit distinguishes detection identity, text-equivalent labels, unsupported or unscored material, mapping/profile differences, and end-to-end text preservation. It does **not** treat one runtime's legacy mapping table as a universal authority: independent-oracle checks are used for a stratified sentinel set, and mapping differences remain explicitly qualified.
+
+Current raw artifacts, methodology revisions, and results are published with CorpusTesters. Historical corpus figures must be read in their recorded taxonomy and build context; they are not a substitute for the current product policy above.
+
+## Known limits
+
+- No detector can recover an author's historical legacy encoding when the same bytes admit multiple plausible readings. EC refuses automatic legacy conversion instead of guessing.
+- Some named legacy codecs have legitimate mapping/profile differences across implementations. An explicit source choice specifies the .NET profile EC will use; strict conversion still verifies that profile's text round trip.
+- The final hash check reduces concurrent-writer risk but cannot make a filesystem replacement fully race-free without holding source handles against writers for the entire operation.
+- `-Backup` is optional in the CLI for scripting. Use `-Backup` or the plan workflow when an in-place conversion must be recoverable.

sources/EncodingChecker.Tests/BackupIntegrityTests.cs

@@ -82,7 +82,7 @@ public void Backup_SuccessfulConversion_LeavesNoTempArtifactBehind()
         Assert.Equal(ConversionRowResult.Converted, Assert.Single(entries).Result);
         Assert.True(File.Exists(path + ".bak"));
         // Temp filename shape: "<name>.<guid>.bak.<TEMP_FILE_SUFFIX>".
-        Assert.Empty(Directory.GetFiles(_root, $"*.bak.{EncodingConverter.TEMP_FILE_SUFFIX}"));
+        Assert.Empty(Directory.GetFiles(_root, $"*.bak.{EncodingConverter.TempFileSuffix}"));
     }
 
     [Fact]
@@ -122,7 +122,7 @@ public void Backup_WildcardInclude_NeverScansItsOwnBakOrTempFiles()
 
         // A leftover temp-conversion artifact, as could survive a crash mid-conversion.
         File.WriteAllText(
-            Path.Combine(_root, $"other.txt.{Guid.NewGuid():N}.{EncodingConverter.TEMP_FILE_SUFFIX}"),
+            Path.Combine(_root, $"other.txt.{Guid.NewGuid():N}.{EncodingConverter.TempFileSuffix}"),
             "leftover temp file content");
 
         var options = new ScanDirectoryOptions

sources/EncodingChecker.Tests/CancellationTests.cs

@@ -29,7 +29,7 @@ public void Dispose()
     }
 
     private static string[] TempArtifacts(string root) =>
-        Directory.GetFiles(root, $"*.{EncodingConverter.TEMP_FILE_SUFFIX}");
+        Directory.GetFiles(root, $"*.{EncodingConverter.TempFileSuffix}");
 
     [Fact]
     public void Convert_PreCancelledToken_ReportsCancelled_WithoutTouchingTheFile()