diff --git a/.github/workflows/detector-parity.yml b/.github/workflows/detector-parity.yml new file mode 100644 index 0000000..29c082f --- /dev/null +++ b/.github/workflows/detector-parity.yml @@ -0,0 +1,47 @@ +name: Shared Unicode detector parity + +on: + push: + branches: [ master ] + schedule: + - cron: "17 3 * * 1" + workflow_dispatch: + +jobs: + parity: + runs-on: windows-latest + steps: + - uses: actions/checkout@v7 + with: + path: encoding-checker + + - uses: actions/checkout@v7 + with: + repository: amrali-eg/LineEndingNormalizer + ref: master + path: line-ending-normalizer + + - uses: actions/checkout@v7 + with: + repository: amrali-eg/CorpusTesters + ref: master + path: corpus-testers + + - name: Verify shared Unicode detector + shell: pwsh + run: | + $ec = Get-Content 'encoding-checker/sources/EncodingChecker/UnicodeDetector.cs' -Raw + $len = Get-Content 'line-ending-normalizer/UnicodeDetector.cs' -Raw + $corpus = Get-Content 'corpus-testers/CorpusTesting/UnicodeDetector.cs' -Raw + $ec = $ec -replace 'namespace EncodingChecker;', 'namespace SharedUnicodeDetector;' + $len = $len -replace 'namespace LineEndingNormalizer;', 'namespace SharedUnicodeDetector;' + $corpus = $corpus -replace 'namespace CorpusTesting;', 'namespace SharedUnicodeDetector;' + $ec = $ec -replace '^using System;\r?\n', '' + $len = $len -replace '^using System;\r?\n', '' + $corpus = $corpus -replace '^using System;\r?\n', '' + $ec = [regex]::Replace($ec, "`r?`n", "`n").TrimStart([char]0xFEFF) + $len = [regex]::Replace($len, "`r?`n", "`n").TrimStart([char]0xFEFF) + $corpus = [regex]::Replace($corpus, "`r?`n", "`n").TrimStart([char]0xFEFF) + if ($ec -cne $len -or $ec -cne $corpus) { + throw 'UnicodeDetector.cs differs between EncodingChecker, LineEndingNormalizer, and CorpusTesters. Keep the shared algorithm synchronized; application-specific TextEncoding wrappers may differ.' + } diff --git a/README.md b/README.md index 3e7b924..5e6849d 100644 --- a/README.md +++ b/README.md @@ -1,439 +1,118 @@ -[![CI](https://github.com/amrali-eg/EncodingChecker/actions/workflows/ci.yml/badge.svg)](https://github.com/amrali-eg/EncodingChecker/actions/workflows/ci.yml) +[![CI](https://github.com/amrali-eg/EncodingChecker/actions/workflows/ci.yml/badge.svg)](https://github.com/amrali-eg/EncodingChecker/actions/workflows/ci.yml) -# EncodingChecker v3.7.0 +# EncodingChecker v3.8.0 -File Encoding Checker detects, validates, and converts the text encoding of one or more files. It runs either as a Windows GUI app or as a command-line tool for scripting and CI, and shares one detection/conversion engine between both. +EncodingChecker is a Windows tool for finding, checking, and safely converting text-file encodings. Use the GUI for everyday work or the command line for repeatable jobs. -Requires the [.NET 10 Desktop Runtime](https://dotnet.microsoft.com/download) (Windows only). +Requires the [.NET 10 Desktop Runtime](https://dotnet.microsoft.com/download). Releases provide both a small framework-dependent build and a self-contained build. -Each [release](https://github.com/amrali-eg/EncodingChecker/releases) publishes two single-file builds: `EncodingChecker.zip` (framework-dependent, requires the .NET 10 Desktop Runtime above) and `EncodingChecker-selfcontained.zip` (larger, but runs on a machine with no .NET runtime installed). +![EncodingChecker window](./form.png) -![form image](./form.png "File Encoding Checker Form Preview") +## Start here -## Highlights +### GUI -- Layered detection: byte-order-mark and heuristic checks for Unicode encodings, [UtfUnknown](https://github.com/CharsetDetector/UTF-unknown) for legacy code pages, each candidate independently verified by strict decoding before being trusted. -- Lossless, safe conversion: every write is verified afterward by comparing a SHA-256 hash of the decoded content, so a silent encoder substitution (e.g. an unrepresentable character) is caught and reported as an error instead of corrupting the file. -- Refuses to convert files whose encoding the bytes do not determine, naming the encodings actually in conflict, with `-From` (or the GUI's source-encoding selection) to supply the answer yourself. One policy engine decides for every surface. -- The GUI confirms before writing, showing exactly what will happen to each file and carrying out that same plan rather than re-deciding. -- `-Journal` records what a run actually did — including the files it refused and why — with each file's SHA-256 before and after. -- `-Plan`/`-Apply` preflight: review what a conversion would do, then carry out exactly that — the plan is bound to the files' hashes and is refused whole if they change. -- Optional `.bak` backup before overwriting, and a `-WhatIf` dry-run mode that reports what would happen without touching any file. -- Covered by an xUnit test suite exercising the detection/conversion engine, CLI argument parsing, and CSV report formatting across multilingual content and edge cases. +1. Open the folder and select file patterns. +2. Choose **View** to inspect the files. +3. Select rows and choose **Convert**. +4. Read the review, then confirm the files that are safe to convert. -## GUI usage +Nothing is changed until the final confirmation. The GUI enables backups by default. +After a conversion, choose **Export results → Conversion history (JSON)…** to keep a +durable record of its decisions and outcomes. -Launch `EncodingChecker.exe` with no arguments. Pick a directory and file filters, choose **View** to detect encodings, **Validate** against a set of accepted charsets, or **Convert** to a target encoding. Results can be exported to CSV. +> [!TIP] +> **Safe workflow:** View → review → choose a legacy source encoding if needed → Convert → keep the resulting `.bak` and `.ecmeta.json` files. -Two options apply to **Convert**: +## How conversion works -- **Back up original files before converting (.bak)** — keeps each original as `.bak` before it is replaced. The equivalent of the CLI's `-Backup`. -- **Preview changes without modifying files** — reports which files *would* be converted without writing anything and without creating any `.bak`. Previewed rows keep their current encoding and stay selected, so you can review the result and then convert for real. The equivalent of the CLI's `-WhatIf`. +View → review → confirm → verified conversion -### Confirming a conversion +[Read the conversion workflow](docs/CONVERSION-WORKFLOW.md) -**Convert** does not write anything immediately. It first works out what would happen to -every selected file, then shows that for approval: +### Command line -``` -Convert 417 of 480 selected file(s) to utf-8 without BOM - - 386 Encoding determined by the file's own bytes Will convert. - 31 Encoding undetermined, every reading agrees on the text Will convert; the label - is a choice, the content - is not. - 22 Encoding undetermined, readings disagree on the text WILL NOT be converted. - 39 Already in the target encoding Nothing to do. - 2 Encoding could not be identified Left alone. - -Directory C:\Source -Source encoding detected per file -Backups enabled — each original kept as .bak -Guarantees strict codecs, verified output, atomic install, ambiguity refusal -``` - -The conversion that runs is the one shown. Nothing is detected a second time between the -confirmation and the writing, so the dialog cannot describe one set of conclusions while -a different set is carried out — the same property `-Apply` has. - -If the files change between the confirmation and the writing, **nothing is converted** — -the same all-or-nothing check `-Apply` makes, for the same reason: a person reading a -dialog takes time, and what they approved was the files as they were. - -When files are refused, the dialog lists them with the encodings actually in conflict and -offers the one thing that resolves it: saying which encoding they are. That selection is -the GUI's `-From`. It replaces detection for those files and nothing else — the bytes must -still decode strictly as the chosen encoding, the output is still verified to hold exactly -the same text, and a failed backup still stops the conversion. - -The choice applies only to the files you tick, and the button says how many. A batch can -easily hold refused files in different encodings — Cyrillic in koi8-r beside French in -windows-1252 — and one answer settles only the files it was given about. Imposing it on -the rest would repeat, one level up, the mistake the refusal exists to prevent. - -### The conversion journal - -`-Journal ` writes a JSON record of the run. For every file: what its encoding was -detected or declared to be, whether the bytes identified it, which encodings competed, -what EC decided, what it actually did, and the file's SHA-256 before and after. - -```json -{ - "RelativePath": "notes.txt", - "Sha256Before": "f19e0e0c…", - "Sha256After": null, - "DetectionMode": "Detected", - "DetectedEncoding": "iso-8859-1", - "SourceEncoding": "iso-8859-1", - "Ambiguity": "TextChanging", - "AmbiguityReason": "MultipleCodecsDifferentText", - "DetectionCandidates": ["cp866", "ibm852", "ibm855", "…"], - "PlannedAction": "Refuse", - "Status": "Refused", - "Reason": "The encoding could not be determined uniquely…" -} -``` - -Three things it does deliberately: - -- **Refused and skipped files are in it.** *Why was this file not converted?* gets asked - far more often than *how do I put this one back?*, and until now only the second had an - answer — in a sidecar written solely where a backup existed. -- **It records the encoding the conversion read, not the detector's raw output.** Those - differ whenever somebody named the source encoding, and `DetectedEncoding` keeps the - detector's answer beside it so the difference is visible. -- **It does not claim more than happened.** `Sha256After` is present only where a file was - actually rewritten, so the record can be checked against the disk. A `-WhatIf` run is - marked as a preview and its files are recorded as decided, not converted. - -In the GUI it is written through **Export report**, choosing *Conversion journal (\*.json)*. - -Together with the per-file `.ecmeta.json` sidecar — which exists so one conversion can be -undone — this completes the chain: what EC believed, what it decided, what was approved, -and what it wrote. - -### One policy engine - -The GUI and the CLI ask the same question of the same code: +For a folder you have not converted before, make a plan first: +```powershell +EncodingChecker.exe -BasePath "C:\Files" -Target utf-8 -Plan plan.json ``` -detection / explicit source → classification → PlannedAction → CLI, GUI, plan -``` - -[`ConversionPolicy`](sources/EncodingChecker/ConversionPolicy.cs) decides; every surface -acts on that decision rather than reaching its own. A missing classification is an -internal error, never a safe state: an entry that reaches a conversion or a plan without -one is refused or raises, rather than being treated as unambiguous. - -EC also counts how often it works out an encoding, and asserts in its test suite that a -file is never examined twice — once when scanned, and never again between a decision being -approved and carried out. Applying a plan, and confirming a GUI conversion, do no -detection at all. - -## Command-line usage - -Launch `EncodingChecker.exe` with arguments to run in console mode instead. Run `EncodingChecker.exe -?` (or `-h`, `/?`, `--help`) at any time to print this from the tool itself. - -``` -EncodingChecker.exe - -BasePath - [-Include ""] # repeatable; patterns accumulate - [-Exclude ""] # repeatable; patterns accumulate - - -Target "" # Convert mode (default); e.g. "utf-8" or "utf-8-bom" - -Validate "" # Validate mode: flag files not in this list - -DetectOnly # Read-only detection mode - - [-From ""] # Treat every file as this encoding instead of - # detecting it (Convert mode only) - [-Plan ] # Write a conversion plan; change nothing - [-Apply ] # Carry out a plan written by -Plan - [-Journal ] # Record what the conversion actually did - - [-Report ] # Also write a CSV report to this path - [-MaxParallelism ] # Default: min(logical processor count, 4) - [-WhatIf] # Convert mode: report without writing - [-Backup] # Convert mode: write ".bak" before overwriting - # (ignored under -WhatIf, which writes nothing) - [-Quiet] # Suppress per-file rows; print only a summary - [-Verbose] # Print error detail and a result breakdown - [-FailOnChanges] # Non-zero exit code if any file needs (or, under - # -Validate, fails) conversion — useful as a CI gate -``` - -`-Include`/`-Exclude` are comma-separated wildcard patterns, and both options may be repeated — patterns from every occurrence accumulate, so `-Include "*.cs" -Include "*.txt"` is equivalent to `-Include "*.cs,*.txt"`. A pattern with no `/` or `\` matches just the filename (e.g. `*.cs` matches at any depth); a pattern containing a separator matches the path relative to `-BasePath` instead (e.g. `src/*.cs` matches only under `src`, `\` and `/` behave the same way). `.git`, `.svn`, `.hg`, `.vs`, `.idea`, `bin`, `obj`, `node_modules`, `packages`, `dist`, `build`, and `target` directories are always skipped. Convert, Validate, and Detect-only are mutually exclusive modes. -`-Backup` only ever writes a `.bak` when a real conversion happens: a file that already matches the target is left alone, and under `-WhatIf` nothing is written at all, so no backup is created. +Review the summary and `plan.json`, then apply exactly that plan: -### Ambiguous encodings, and `-From` - -Some files do not identify the encoding that wrote them. A file valid in -windows-1252 is equally valid in iso-8859-1 and in koi8-r, and each reads it as -different text; nothing in the bytes decides between them. Detection still -produces an answer, and converting on that answer rewrites the file into one of -several possible readings without saying so. - -EncodingChecker refuses those conversions and names the encodings actually in -conflict: - -``` -Error: notes.txt: The encoding could not be determined uniquely from the file's -contents. iso-8859-1 and cp866, ibm852, ibm855, iso-8859-13, and 17 more all match -this file and would produce different text. No conversion was performed; specify -the source encoding explicitly to convert it. -``` - -The refusal applies only where the rival readings *disagree about the text*. A -file whose encoding is undetermined but whose candidates all decode it -identically — plain ASCII being the common case — is converted normally, because -there is nothing to protect the user from. Nor does it apply where the file's own -structure picks the encoding out: valid UTF-8, Shift_JIS or Big5 byte sequences -are not valid by accident, and codecs that accept any byte sequence are not -offering a competing reading so much as failing to object. - -`-From` supplies the answer detection could not. It replaces detection and -nothing else: the bytes must still decode strictly as the named encoding, the -output is still verified to hold exactly the same text, and a failed backup still -aborts the conversion. Naming an encoding says which one it is, not "convert it -regardless". - -### Preflight: `-Plan` and `-Apply` - -`-Plan` writes down what a conversion would do and changes nothing. For every -file the plan records the action, the source encoding, whether it was detected or -specified, whether the bytes identify it uniquely, which encodings compete for -it, and the reason behind any refusal — as JSON, alongside a summary on stdout: - -``` -Selected: 3 - -Will convert: 2 - encoding determined: 2 - same text either way: 0 -Already in target encoding: 0 -Encoding not identified: 0 -Refused, ambiguous encoding: 1 -Refused, unreadable: 0 - -Directory: C:\Source -Target: utf-8 without BOM -Source encoding: detected per file -Backups: enabled -Guarantees: strict codecs, verified output, atomic install, ambiguity refusal - -No files modified. -``` - -The two indented lines break down `Will convert`; the rest sum exactly to -`Selected`, so the totals can be checked rather than trusted. - -`-Apply` carries that plan out. It does not detect anything a second time: the -encodings, the target, and the backup setting all come from the plan, so -`-BasePath`, `-Target`, `-From`, and `-Backup` are rejected rather than silently -ignored. `plan.json` is the whole approval. - -The binding is the point of the feature, not the preview: - -- **Bound to the files.** Every scheduled file carries the SHA-256 it had when - the plan was made, and `-Apply` verifies each one before writing anything. If - any file has changed or been deleted in between, **nothing is converted** — not - even the files that still match. A plan reviewed as a whole belongs to the - directory it was reviewed against, and the files most likely to have changed - are the ones something else is actively writing. Each file is checked once - more at the moment it is installed, which narrows the window between that - verification and the write. -- **Bound to the directory.** Paths are stored relative to a recorded root, so a - plan is a document about a directory rather than about one machine. Applying a - copy of a plan converts the tree it was approved for, not whichever tree it - happens to sit in, and an entry that resolves outside that root is refused. -- **Bound to the conversion.** The plan records the target encoding, BOM policy, - backup policy, whether the source encoding was detected or specified, and a - semantics version describing the conversion behaviour it was approved under. A - plan written under different behaviour is refused rather than carried out — - what was approved was a conversion, not a list of filenames. - -The semantics version is deliberately separate from EC's version number: it moves -only when conversion or classification behaviour changes, so a release that -changes nothing about conversion does not invalidate plans and teach people to -work around the check. - -```bash -EncodingChecker.exe -BasePath . -Include "*" -Target "utf-8" -Plan plan.json +```powershell EncodingChecker.exe -Apply plan.json ``` -Exit codes: `0` clean, `1` usage/argument error (nothing was scanned), `2` `-FailOnChanges` triggered, `3` the run did not complete cleanly — one or more files failed to process, the scan itself failed, or the `-Report` file could not be written, `4` cancelled (Ctrl+C). - -These are the same codes as [LineEndingNormalizer](https://github.com/amrali-eg/LineEndingNormalizer), a companion Windows CLI tool that normalizes line endings, so a script driving both can share one exit-code mapping. It additionally returns `5` for a missing base directory and `6` for a reparse-point `-BasePath`, both of which are reported here as `1` — so no code means two different things across the two tools, and treating `1`, `5` and `6` alike handles either. +The plan is tied to the selected files and their hashes. If a scheduled source file changes after review, nothing is converted. -The CSV report (and `-DetectOnly`'s stdout) uses the columns `File,Encoding,BOM,Target,TargetBOM,Result`, where `Encoding`/`BOM` describe the original file and `Target`/`TargetBOM` the encoding and BOM state it was (or would be) converted to. +## The conversion rule -Examples: +**Unicode and ASCII files can be converted automatically.** -```bash -EncodingChecker.exe -BasePath C:\Source -Include "*.cs,*.txt" -Target "utf-8" +**Legacy text needs an explicit source encoding.** Tell EC what it is with `-From` on the command line or the source-encoding chooser in the GUI: -EncodingChecker.exe -BasePath . -Include "*.cpp,*.hpp" -Target "utf-8" -WhatIf - -EncodingChecker.exe -BasePath . -Include "*" -Validate "utf-8,utf-8-bom" -Report report.csv -FailOnChanges +```powershell +EncodingChecker.exe -BasePath "C:\Files" -Target utf-8 -From windows-1252 -Backup ``` -## Safety model - -These are the guarantees the implementation actually provides. - -- Content is decoded and re-encoded through a strict `Decoder`/`Encoder` pair: - malformed input, and content the target cannot represent, are rejected rather - than silently replaced. There is no raw-byte conversion path — every encoding, - Unicode or legacy, goes through decode/re-encode. -
Strictness is enforced by rebuilding the encoding with its fallbacks - supplied up front (`TextEncoding.Strict`). Assigning `Decoder.Fallback` or - `Encoder.Fallback` *after* `GetDecoder()`/`GetEncoder()` is silently ignored by - the `CodePagesEncodingProvider` encodings — the codec has already taken its - fallbacks from the parent `Encoding` — which is exactly the defect the - [independent audit](#independent-audit) found in v3.5.0 and earlier. -- Every write is verified before installation by re-decoding the temporary file - and comparing a SHA-256 hash of its *decoded* content and BOM state against the - source. This is a backstop behind the strict codecs, not the primary defence: - because it compares decoded source against decoded target, a decoder that - substitutes silently would produce agreeing hashes. Strict codecs are what - prevent that; the hash catches anything they cannot. -- The source file is never rewritten in place: conversion writes to a new - temporary file beside the destination, which is verified before it is - installed. -- Immediately before installation, the destination is revalidated (length - and last-write time) so a file changed elsewhere during conversion is not - silently overwritten. **This is a point-in-time race check, not a - complete elimination of every possible TOCTOU window.** -- Original file attributes and timestamps are preserved: applied to the - temporary file before installation, so the final file's metadata is - correct atomically along with its content. -- With `-Backup`, the original is copied to `.bak` *before* the main - file is replaced; if the backup fails, the main conversion is aborted and - the original is left untouched. A previously read-only `.bak` is still - replaced correctly. -
**In the GUI this is on by default** (and remembered between runs). The - audit found that conversion from a Unicode or ASCII source altered none of - 1,832 files, but roughly one in five converted from a legacy code page came - out with different text, because single-byte code pages are mutually decodable - and nothing in the bytes says which was intended. Such a conversion is almost - always reversible — but only for someone who still knows which codec was used, - and that is recorded solely in the conversion report. The CLI leaves `-Backup` - opt-in, since a script can keep the report. -- A conversion whose source encoding cannot be determined from the file's own - bytes is refused rather than guessed at, when the competing encodings would - produce different text. `-From`, and the GUI's source-encoding selection, - override the detection, not the conversion safeguards. - See [Ambiguous encodings](#ambiguous-encodings-and--from). -
That decision is made in one place for every surface, so the GUI and the - CLI cannot diverge on what is safe. They previously could, and did: the - classification ran only during a Convert-mode scan, the GUI scans in Detect - mode, and so the GUI converted the files the CLI refused. -- A plan written by `-Plan` is bound to the SHA-256 of every file it schedules, - to the directory those files are under, and to the conversion behaviour it was - approved under. `-Apply` verifies all of them before writing anything and - refuses the plan whole if any has changed, so a decision made about one set of - bytes is never applied to a different one. Under `-Apply`, each source is - re-hashed again immediately before installation. **This narrows the window - between verification and write; it does not close it** — a source rewritten - between that check and the replacement is still not detected, which would - require holding every source open against writers for the whole run. -- `-BasePath` itself is rejected if it is a symbolic link, junction, or - other reparse point. Reparse-point subdirectories are skipped during - traversal, and a file that is (or becomes) a reparse point is rejected at - the point of installation. -- `.bak` files and the tool's own abandoned temporary files — both the - conversion temp file and the `-Backup` install's own temp file — are - automatically excluded from scanning, including under a broad - `-Include "*"`, so a later run never treats its own output as input. -- Installation uses .NET's `File.Replace` where supported; a plain, - non-atomic move is used only when that platform support is genuinely - unavailable, never as a silent fallback after a real replacement - failure. -- Cleanup of the temporary file after a failure clears any inherited - ReadOnly attribute before deleting it, and a cleanup failure can never - replace or mask the actual error being reported — conversion results are - returned as structured data, not thrown, so the result is already - finalized before cleanup ever runs. -- Cancellation (Ctrl+C in the CLI) is observed between files and at - multiple points within a single file's conversion; a cancelled run never - leaves a half-written destination, because the destination is only - touched by the final install step. - -## Independent audit - -EncodingChecker's conversion is audited end to end against four public corpora — -**5,078 files** — by a separate harness: -**[CorpusTesters](https://github.com/amrali-eg/CorpusTesters)**. - -The audit answers one question per file, with no normalization of any kind and no -replacement characters permitted: +Choosing a source encoding does not bypass safety checks. The source must still decode strictly, the output must verify as the same text, and a backup failure stops the conversion. -``` -strict-decode(original bytes, reference codec + BOM) - == strict-decode(converted bytes, target codec) -``` +## Common commands -Ground truth comes from each corpus's own manifest or catalogue, never from -filenames and never from compatibility metadata. Source corpora are treated as -read-only: each is copied into a working directory and only the copy is -converted, verified after every run against the corpora's published SHA-256 -hashes. +```powershell +# Inspect detected encodings only; do not modify files. +EncodingChecker.exe -BasePath "C:\Files" -DetectOnly -### Results for v3.6.0 +# Preview selected files once; do not modify files or create a plan. +EncodingChecker.exe -BasePath "C:\Files" -Include "*.cs,*.txt" ` + -Exclude "*.g.cs,*.designer.cs" -Target utf-8 -WhatIf -Measured over the files EC actually **rewrote** — files it skipped or left -byte-identical cannot have lost anything: +# Validate in CI, save a CSV report, and fail when a file is outside the list. +EncodingChecker.exe -BasePath "C:\Files" -Validate "utf-8,utf-8-bom" ` + -Report validation.csv -FailOnChanges -Quiet -| Source | Rewritten | Text preserved | -|---|---:|---:| -| Unicode + ASCII | 1,832 | 1,832 (**100.00%**) | -| Legacy code page (.NET has a codec) | 2,021 | 1,602 (79.27%) | -| No .NET codec exists | 112 | 21 (18.75%) | +# Convert known legacy text, preserve originals, and record the run. +EncodingChecker.exe -BasePath "C:\Files" -Include "*.txt" -From windows-1252 ` + -Target utf-8 -Backup -Journal conversion.json -Verbose -Four metrics are reported separately rather than blended into one accuracy -figure, because a single number would average silent data loss against files that -merely happened to be ASCII: +# Limit work against a network or slow disk. +EncodingChecker.exe -BasePath "D:\Share" -Target utf-8 -MaxParallelism 2 +``` -| Metric | Result | -|---|---| -| Detection accuracy (exact codec identity) | 3756/4961 (75.7%) | -| Strict-decoding correctness | **5023/5023 (100%)** | -| Codec conformance | 89 divergences | -| End-to-end text preservation | 4101/4741 (86.5%) | +## Command-line reference -**Unicode and ASCII input is safe on this evidence** — not one of the 1,832 files -converted from a Unicode or ASCII source came out with different text. +| Option | Meaning | +| --- | --- | +| `-BasePath ` | Folder to scan. Required except with `-Apply`. | +| `-Target ` | Target encoding, such as `utf-8` or `utf-8-bom`. Required for conversion. | +| `-From ` | Explicit original encoding for every selected file. Use for legacy conversion. | +| `-Plan ` | Write a reviewable plan; do not modify files. | +| `-Apply ` | Execute a saved plan. Do not add `-BasePath`, `-Target`, `-From`, or `-Backup`. | +| `-WhatIf` | One-time preview; do not write files. | +| `-Backup` | Save every replaced original as `.bak`. | +| `-DetectOnly` | Report detected encodings; do not modify files. | +| `-Validate ` | Check files against an allowed encoding list; do not modify files. | +| `-Include` / `-Exclude` | Comma-separated wildcard patterns; either option may be repeated. | +| `-Report ` | Also write the CSV report to a file. | +| `-Journal ` | Write a JSON record of conversion decisions and results. | +| `-Quiet` / `-Verbose` | Show only a summary, or include details and a result breakdown. | +| `-MaxParallelism ` | Maximum simultaneous files; default is `min(CPU count, 4)`. | +| `-FailOnChanges` | Return exit code `2` if files need conversion or fail validation. Useful in CI. | -**Legacy input carries the residual risk.** The dominant part of it is -source-encoding identification: single-byte code pages are mutually decodable, so -`windows-1252` text is perfectly valid `iso-8859-1` text and nothing in the bytes -distinguishes them. Forced to the correct codec, those files convert exactly. +Patterns without a path separator match filenames at any depth, such as `*.txt`. Patterns containing `/` or `\` match paths relative to `-BasePath`, such as `src/*.cs`. Build and metadata folders including `.git`, `bin`, `obj`, and `node_modules` are skipped automatically. -Codec implementation differences and decoder strictness are separate conversion -risks rather than the same one, and the two interact — the detector can name the -right encoding and the conversion still alter text, because the implementation -behind that name differs from the reference. +Run `EncodingChecker.exe -?` for the same reference from the executable. The help aliases are `-?`, `/?`, `-h`, `/h`, and `--help`. Exit codes: `0` completed, `1` invalid command, `2` `-FailOnChanges`, `3` processing/plan/report failure, `4` cancelled. -The 89 codec divergences are known Microsoft-vs-Unicode mapping differences in -the Japanese and Chinese code pages (U+301C wave dash versus U+FF5E fullwidth -tilde, and similar) — properties of .NET's code-page tables, not of this tool. +## Safety and transparency -### What it found +Every conversion uses strict decoding and encoding, verifies the output text before installation, and writes through a temporary file rather than in place. With `-Backup`, EC verifies the backup before it replaces the source. The GUI enables backups by default. -The audit's PHASE 0 establishes what the build under test actually does before -judging any file, and that is how the strict-fallback defect fixed in v3.6.0 -([#36](https://github.com/amrali-eg/EncodingChecker/pull/36)) was found: files -whose bytes their own codec could not represent were being converted with -substituted characters and reported as `Converted`. +For the complete safety model, conversion-plan guarantees, recovery metadata, known limits, and independent corpus audit, see [Safety and audit](docs/SAFETY-AUDIT.md). The audit harness and reproducible per-file evidence live in [CorpusTesters](https://github.com/amrali-eg/CorpusTesters). -Blast radius, stated plainly: **4 files out of 5,078**. It was a latent -correctness hole, not mass corruption — it rarely fired because detection usually -picks a codec that *can* decode the bytes. Before and after the fix, across all -four corpora: **8 files changed outcome, all improvements, zero regressions.** +## Known limits -Every figure above is reproducible; the harness, its methodology and its raw -per-file evidence are documented in the CorpusTesters repository. +File bytes alone cannot always identify the original legacy encoding uniquely. EC therefore leaves detected legacy text unchanged until you choose or confirm its source encoding. Keep each `.bak` file with its matching `.ecmeta.json` record for deterministic recovery. ## Supported charsets @@ -462,9 +141,9 @@ Over forty charsets, matching what [UtfUnknown](https://github.com/CharsetDetect ## Credits -The original project [EncodingChecker](https://archive.codeplex.com/?p=encodingchecker) on CodePlex was written by [Jeevan James](https://github.com/JeevanJames). +The original [EncodingChecker](https://archive.codeplex.com/?p=encodingchecker) project was written by [Jeevan James](https://github.com/JeevanJames). -For encoding detection, File Encoding Checker uses the [UtfUnknown](https://github.com/CharsetDetector/UTF-unknown) library, a C# port of [uchardet](https://gitlab.freedesktop.org/uchardet/uchardet), itself a C++ port of the original [Mozilla Universal Charset Detector](https://dxr.mozilla.org/mozilla/source/extensions/universalchardet/). See [THIRD-PARTY-NOTICES.txt](./THIRD-PARTY-NOTICES.txt) for its license. +For detection, EncodingChecker uses [UtfUnknown](https://github.com/CharsetDetector/UTF-unknown), a C# port of uchardet. See [THIRD-PARTY-NOTICES.txt](./THIRD-PARTY-NOTICES.txt) for license details. ## License diff --git a/RELEASE-CHECKLIST.md b/RELEASE-CHECKLIST.md index 5237760..3a8cbe5 100644 --- a/RELEASE-CHECKLIST.md +++ b/RELEASE-CHECKLIST.md @@ -6,9 +6,10 @@ cannot answer. ## Automated - [ ] `dotnet test sources/EncodingChecker.Tests/EncodingChecker.Tests.csproj -c Release` — all green. -- [ ] The 1,033-file oracle sentinel set still agrees with GNU libiconv and ICU. -- [ ] Detector-drift check passes (the detector sources are duplicated across three - repositories and nothing enforces the sync; a fix in one is a fix owed to all three). +- [ ] The scheduled **Shared Unicode detector parity** workflow is green. It compares + the shared detector source in EncodingChecker, LineEndingNormalizer, and + CorpusTesters after normalizing namespace, a redundant `using System` import, + and line-ending differences. ## Manual: the GUI smoke test @@ -33,45 +34,35 @@ record the file's SHA-256 before and after: Get-FileHash -Algorithm SHA256 | Select-Object -ExpandProperty Hash ``` -### Test files - -| name | contents | encoding | expected classification | -|---|---|---|---| -| `jp.txt` | `こんにちは世界。日本語のテキストです。` | Shift_JIS | unambiguous | -| `french.txt` | `Le café était déjà prêt` | windows-1252 | text-changing | -| `russian.txt` | `Привет мир, это русский текст` | koi8-r | text-changing | -| `plain.txt` | `plain ascii, no high bytes at all` | ASCII | text-equivalent | - -### Structure it in phases, not one long sequence - -The first version of this test used one folder and one final check for the whole matrix. -That cannot work, and the reason is worth keeping: the stale-plan case **stops the entire -run**, so every "must have converted" expectation after it is unreachable by construction. -Worse, the state it leaves is byte-identical to "the tester cancelled everything", so the -result cannot say *which* protection fired. The first real run produced a FAIL that was -entirely the instrument's fault, and only inspecting the bytes by hand showed the product -had behaved correctly. - -Each phase therefore gets its own folder, its own short click sequence, and its own check, -and proves exactly one property. [`tools/gui-smoke-test.py`](tools/gui-smoke-test.py) does the setup and the -verification; it also refuses to pass a phase whose defining action was skipped — a phase -that silently tests nothing is the failure mode a manual matrix is most prone to. - -### Matrix - -| # | step | expected | -|---|---|---| -| 1 | **View** the directory | 4 files listed with their encodings | -| 2 | Tick all, **Convert** to utf-8 | confirmation appears; two files listed as needing an explicit source encoding, with competing encodings named | -| 3 | **Cancel** | nothing converted; **all four hashes unchanged**; no `.bak` files | -| 4 | Convert again; untick `russian.txt`; choose `windows-1252` | button reads "Use this encoding for 1 file(s)" | -| 5 | Confirm the re-planned conversion | `french.txt` converts and reads correctly as French | -| 6 | Check `russian.txt` | **hash unchanged**; still refused | -| 7 | Convert again; while the dialog is open, edit one selected file in another editor and save | — | -| 8 | Confirm | run stops; message names the changed file; **every hash unchanged** | -| 9 | Convert `jp.txt` alone, backups on | converts; `jp.txt.bak` and `jp.txt.ecmeta.json` present; text reads correctly | -| 10 | Create a **directory** named `.bak` beside a file, convert it | conversion refused; **source hash unchanged** | -| 11 | Export report → **Conversion journal (\*.json)** | journal written; refused files present with their competing encodings; `Sha256After` null for everything not converted | +### Core GUI smoke test + +[`tools/gui-smoke-test.py`](tools/gui-smoke-test.py) creates disposable folders on the +Desktop and verifies the resulting bytes. For every phase, set the printed folder as +**Directory to check** and choose **utf-8** in **Convert to**. Then run each short phase +with the Release build: + +```powershell +python tools/gui-smoke-test.py setup A +# perform the displayed GUI steps +python tools/gui-smoke-test.py verify A +``` + +| Phase | What the GUI check proves | +|---|---| +| A | **View** lists the prepared files; Unicode and ASCII are ready, legacy files need a source choice; **Cancel** changes no bytes and creates no recovery files. | +| B | Unicode and ASCII convert without a source choice and preserve their exact text. | +| C | A chosen legacy source encoding applies only to the ticked files; unselected legacy files stay unchanged. | + +The script verifies hashes and decoded output; status messages alone never count as evidence. + +### Accessibility spot check + +- [ ] At 100%, 125%, and 150% display scaling, the review text, source-encoding + chooser, and its confirmation button are fully visible without horizontal scrolling. +- [ ] Keyboard-only: Tab reaches the legacy-file list, source chooser, and both final + actions; Enter performs only the displayed ready conversion; Escape cancels. +- [ ] In a Windows high-contrast theme, the review outcomes and legacy warning remain + readable and distinguishable. ### Record @@ -86,50 +77,15 @@ Windows version: Date: Tester: -Phase A (refuse + cancel change nothing): PASS / FAIL -Phase B (explicit source, scoped): PASS / FAIL -Phase C (stale plan stops the whole run): PASS / FAIL -Phase D (backup + record; backup failure): PASS / FAIL +Phase A (review + cancel): PASS / FAIL +Phase B (Unicode + ASCII conversion): PASS / FAIL +Phase C (scoped legacy source choice): PASS / FAIL Cases where observed differed from expected: Result: PASS / FAIL ``` -### Run of 2026-08-27 - -```text -EC version: 3.7.0.0 -Commit: a201a08 -Windows version: Microsoft Windows NT 10.0.26200.0 -.NET version: 10.0.400 -Date: 2026-08-27 -Tester: amrali-eg - -Phase A (refuse + cancel change nothing): PASS -Phase B (explicit source, scoped): PASS -Phase C (stale plan stops the whole run): PASS -Phase D (backup + record; backup failure): PASS - -Result: PASS -``` - -Notes from that run, kept because they qualify what the phases actually establish: - -- **Phase B proves less on its own than it appears to.** Its French sample decodes - identically under windows-1252 and iso-8859-1, so "the text is preserved" cannot show - which codec was used. What settles it is the recovery record: `french.txt.ecmeta.json` - gives `DetectedCodePage: 1252`, the encoding chosen in the dialog rather than the - `iso-8859-1` that detection proposed. A future revision should use content where the two - encodings genuinely disagree, so the assertion stands without the sidecar. -- **Text-equivalent ambiguity is nearly unreachable.** Eight ASCII shapes — short strings, - digits, JSON, code, newlines — all classify as `StructurallyDetermined`, because ASCII - constrains every byte below 0x80. Only a **one-byte file** reaches `TextEquivalent`, - where no codec that decodes it at all can read it differently. The middle class of the - three-way taxonomy is far rarer in practice than the taxonomy suggests. The classifier - is right in both cases; the corpus has to be contrived to exercise it, and `tiny.txt` - exists for that reason alone. - ## Documentation - [ ] README figures match the current audit run; no stale counts. diff --git a/docs/CONVERSION-WORKFLOW.md b/docs/CONVERSION-WORKFLOW.md new file mode 100644 index 0000000..e408182 --- /dev/null +++ b/docs/CONVERSION-WORKFLOW.md @@ -0,0 +1,63 @@ +# How conversion works + +This page explains what happens after you ask EncodingChecker to convert files. It is the same safety model in the GUI, the command line, and saved conversion plans. + +## What you do + +1. **View** the folder to see what EC found. +2. Select the files you want to handle and choose **Convert**. +3. Read the review before any file is changed. +4. For legacy text, choose the source encoding if you know it. +5. Confirm the reviewed conversion. + +The review tells you which files will convert, already match the target, need a legacy source choice, or cannot be processed. Cancelling leaves every source file unchanged. + +## The important rule + +| File type | What EC does automatically | +| --- | --- | +| Unicode or ASCII | May convert it | +| Legacy text | Leaves it unchanged until you choose the original encoding | +| Unknown or unreadable data | Leaves it unchanged | + +Choosing a legacy encoding answers only “how should these bytes be read?” It does not disable strict decoding, output verification, backups, or atomic installation. + +## What EC does + +```mermaid +flowchart LR + A[Scan files] --> B[Decide source interpretation] + B --> C[Build review plan] + C --> D[User confirms] + D --> E[Strict source decode] + E --> F[Strict target encode] + F --> G[Verify identical text] + G --> H[Backup and metadata] + H --> I[Install verified output] +``` + +Every step after confirmation must succeed. If decoding, encoding, verification, backup creation, or installation fails, EC leaves that source file unchanged. + +## Plans and the command line + +For a cautious batch workflow, create a plan first: + +```powershell +EncodingChecker.exe -BasePath "C:\Files" -Target utf-8 -Plan plan.json +``` + +After reviewing it, apply that exact plan: + +```powershell +EncodingChecker.exe -Apply plan.json +``` + +The plan contains the files’ hashes and conversion settings. If a scheduled file changes after review, EC rejects the whole plan instead of applying an approval to different bytes. + +For a known legacy source, supply the encoding explicitly: + +```powershell +EncodingChecker.exe -BasePath "C:\Files" -Target utf-8 -From windows-1252 -Backup +``` + +For the detailed guarantees and known limits, read [Safety and audit](SAFETY-AUDIT.md). diff --git a/docs/RELEASE-NOTES-v3.8.0.md b/docs/RELEASE-NOTES-v3.8.0.md new file mode 100644 index 0000000..11ec379 --- /dev/null +++ b/docs/RELEASE-NOTES-v3.8.0.md @@ -0,0 +1,24 @@ +# EncodingChecker v3.8.0 + +## Safer legacy conversion + +- Unicode and ASCII files continue to convert automatically. +- Detected legacy text is now left unchanged until its original source encoding is + explicitly chosen in the GUI or supplied with `-From` on the command line. +- An explicit source encoding still uses strict decoding, verified output, backup + checks, and atomic replacement; it is not a safety bypass. + +## Clearer review and export + +- The conversion review states which files are ready, which need a source encoding, + and which EC will leave unchanged. +- Legacy source choices apply only to the ticked files and show their scope clearly. +- **Export results** now offers a CSV report and, after conversion, a JSON conversion + history. + +## Reliability and maintainability + +- Saved plans bind approved file hashes and conversion semantics before `-Apply`. +- Backup sidecars preserve the source codec and conversion provenance. +- The shared Unicode detector is checked for parity across EncodingChecker, + LineEndingNormalizer, and CorpusTesters. diff --git a/docs/SAFETY-AUDIT.md b/docs/SAFETY-AUDIT.md new file mode 100644 index 0000000..6002bc1 --- /dev/null +++ b/docs/SAFETY-AUDIT.md @@ -0,0 +1,72 @@ +# EncodingChecker safety and audit + +This document is the technical companion to the main [README](../README.md). It explains what EC's conversion pipeline guarantees, what it does not guarantee, and how those claims are checked independently. + +## Conversion safety boundary + +For each file that EC is allowed to convert, the engine: + +1. strictly decodes the source encoding; +2. strictly encodes the requested target encoding into a temporary file; +3. strictly decodes that temporary output and compares the exact Unicode scalar sequence with the source text; +4. if backups are enabled, creates and verifies `.bak` plus recovery metadata; +5. installs the verified temporary file atomically where the platform supports it. + +Any decode, encode, verification, backup, or write failure leaves the source file unchanged. No normalization, case folding, whitespace rewriting, or replacement-character fallback is used to make a conversion appear successful. + +The source is not rewritten in place. File attributes and timestamps are applied to the temporary output before installation. EC skips its own backups and temporary files on subsequent scans. + +## Source-encoding policy + +Encoding identification and text preservation are different questions. A sequence of legacy bytes often cannot prove which historical single-byte code page produced it. + +EC therefore has a simple policy: + +| Source interpretation | Automatic conversion | +| --- | --- | +| Unicode or ASCII | Allowed | +| Legacy codec supplied explicitly by the user | Allowed, subject to all safety checks | +| Legacy codec detected automatically | Refused; choose the source codec first | +| Unknown or unreadable source | Not converted | + +`-From` and the GUI source chooser replace detection only. They do not bypass strict decoding, output verification, backup verification, or atomic installation. + +## Plans, confirmation, and recovery + +`-Plan` writes a conversion plan without changing files. The plan contains the source hashes, paths relative to its declared root, target and BOM policy, source-selection mode, backup setting, and conversion-semantics version. + +`-Apply` rejects changed, missing, relocated, or incompatible planned work as a whole; it does not silently apply the remaining files. EC also rechecks the source hash immediately before installation. That narrows, but cannot eliminate, a narrow concurrent-writer TOCTOU window between the final check and replacement. + +The GUI uses the same policy and plan model. It displays a review before writing, and a changed source while that review is open invalidates the run. + +With backups enabled, each conversion has a portable `.ecmeta.json` sidecar. The sidecar records the source codec actually used, whether it was detected or explicitly selected, source and backup hashes, target/BOM policy, conversion timestamp, and version. Recovery verifies the backup against that metadata before restoring it. + +`-Journal` provides the batch-level record: EC's detected or explicit source, policy decision, planned action, actual outcome, and before/after hashes for every file—including skipped and refused ones. + +## Strict-codec defect fixed in v3.6.0 + +The independent audit found that assigning `Decoder.Fallback` or `Encoder.Fallback` after calling `GetDecoder()` or `GetEncoder()` does not reliably make .NET `CodePagesEncodingProvider` codecs strict. Some malformed legacy input could be silently substituted while EC's old downstream content check still reported success. + +EC now constructs strict code-page encodings with exception fallbacks at `Encoding.GetEncoding(...)` construction time. Permanent regression tests cover the previously permissive decoder and encoder paths. + +## Independent audit + +[CorpusTesters](https://github.com/amrali-eg/CorpusTesters) is a separate, reproducible audit harness. It runs EC against four public corpora: + +- [UnicodeTestSuite](https://github.com/amrali-eg/UnicodeTestSuite) +- [chardet test-data](https://github.com/chardet/test-data) +- [char-dataset](https://github.com/Ousret/char-dataset) +- [UTF-unknown](https://github.com/CharsetDetector/UTF-unknown) + +It operates on working copies, never source corpora. For each file with authoritative metadata, it compares the exact decoded source text against strict UTF output. It also verifies backup hashes, inventories every file, runs mutation controls, checks codec strictness, and keeps per-file CSV/JSON evidence. + +The audit distinguishes detection identity, text-equivalent labels, unsupported or unscored material, mapping/profile differences, and end-to-end text preservation. It does **not** treat one runtime's legacy mapping table as a universal authority: independent-oracle checks are used for a stratified sentinel set, and mapping differences remain explicitly qualified. + +Current raw artifacts, methodology revisions, and results are published with CorpusTesters. Historical corpus figures must be read in their recorded taxonomy and build context; they are not a substitute for the current product policy above. + +## Known limits + +- No detector can recover an author's historical legacy encoding when the same bytes admit multiple plausible readings. EC refuses automatic legacy conversion instead of guessing. +- Some named legacy codecs have legitimate mapping/profile differences across implementations. An explicit source choice specifies the .NET profile EC will use; strict conversion still verifies that profile's text round trip. +- The final hash check reduces concurrent-writer risk but cannot make a filesystem replacement fully race-free without holding source handles against writers for the entire operation. +- `-Backup` is optional in the CLI for scripting. Use `-Backup` or the plan workflow when an in-place conversion must be recoverable. diff --git a/sources/EncodingChecker.Tests/BackupIntegrityTests.cs b/sources/EncodingChecker.Tests/BackupIntegrityTests.cs index fd8dc18..826c8d6 100644 --- a/sources/EncodingChecker.Tests/BackupIntegrityTests.cs +++ b/sources/EncodingChecker.Tests/BackupIntegrityTests.cs @@ -82,7 +82,7 @@ public void Backup_SuccessfulConversion_LeavesNoTempArtifactBehind() Assert.Equal(ConversionRowResult.Converted, Assert.Single(entries).Result); Assert.True(File.Exists(path + ".bak")); // Temp filename shape: "..bak.". - Assert.Empty(Directory.GetFiles(_root, $"*.bak.{EncodingConverter.TEMP_FILE_SUFFIX}")); + Assert.Empty(Directory.GetFiles(_root, $"*.bak.{EncodingConverter.TempFileSuffix}")); } [Fact] @@ -122,7 +122,7 @@ public void Backup_WildcardInclude_NeverScansItsOwnBakOrTempFiles() // A leftover temp-conversion artifact, as could survive a crash mid-conversion. File.WriteAllText( - Path.Combine(_root, $"other.txt.{Guid.NewGuid():N}.{EncodingConverter.TEMP_FILE_SUFFIX}"), + Path.Combine(_root, $"other.txt.{Guid.NewGuid():N}.{EncodingConverter.TempFileSuffix}"), "leftover temp file content"); var options = new ScanDirectoryOptions diff --git a/sources/EncodingChecker.Tests/CancellationTests.cs b/sources/EncodingChecker.Tests/CancellationTests.cs index 5caa3c0..a397d2a 100644 --- a/sources/EncodingChecker.Tests/CancellationTests.cs +++ b/sources/EncodingChecker.Tests/CancellationTests.cs @@ -29,7 +29,7 @@ public void Dispose() } private static string[] TempArtifacts(string root) => - Directory.GetFiles(root, $"*.{EncodingConverter.TEMP_FILE_SUFFIX}"); + Directory.GetFiles(root, $"*.{EncodingConverter.TempFileSuffix}"); [Fact] public void Convert_PreCancelledToken_ReportsCancelled_WithoutTouchingTheFile() diff --git a/sources/EncodingChecker.Tests/CharsetLabelRoundTripTests.cs b/sources/EncodingChecker.Tests/CharsetLabelRoundTripTests.cs index cde0935..65432ba 100644 --- a/sources/EncodingChecker.Tests/CharsetLabelRoundTripTests.cs +++ b/sources/EncodingChecker.Tests/CharsetLabelRoundTripTests.cs @@ -63,4 +63,17 @@ public void ParseCharsetLabel_IsCaseInsensitiveForTheBomSuffix() Assert.True(hasBom); Assert.Equal("utf-8", baseCharset); } + + [Theory] + [MemberData(nameof(BomAwareCharsets))] + public void IsBomCapable_RecognizesEveryEncodingOfferedWithABom(string charset) => + Assert.True(ScanEngine.IsBomCapable(charset)); + + [Fact] + public void DescribeTarget_DoesNotInventABomForAscii() + { + Assert.False(ScanEngine.IsBomCapable("us-ascii")); + Assert.Equal("us-ascii", ScanEngine.DescribeTarget("us-ascii", hasBom: false)); + Assert.Equal("utf-8 without a BOM", ScanEngine.DescribeTarget("utf-8", hasBom: false)); + } } diff --git a/sources/EncodingChecker.Tests/ConversionConfirmationFormTests.cs b/sources/EncodingChecker.Tests/ConversionConfirmationFormTests.cs index 88b81d2..9ae1341 100644 --- a/sources/EncodingChecker.Tests/ConversionConfirmationFormTests.cs +++ b/sources/EncodingChecker.Tests/ConversionConfirmationFormTests.cs @@ -1,4 +1,4 @@ -using System.Text; +using System.Text; using System.Windows.Forms; namespace EncodingChecker.Tests; @@ -31,32 +31,6 @@ public void Dispose() } } - /// Runs on an STA thread, as WinForms requires. - private static void OnUiThread(Action body) - { - Exception? failure = null; - - var thread = new Thread(() => - { - try - { - body(); - } - catch (Exception ex) - { - failure = ex; - } - }); - - thread.SetApartmentState(ApartmentState.STA); - thread.Start(); - - Assert.True(thread.Join(TimeSpan.FromSeconds(30)), "the dialog did not finish"); - - if (failure is not null) - throw new Xunit.Sdk.XunitException($"the dialog threw: {failure}"); - } - private void Write(string name, string text, string charset) => File.WriteAllBytes( Path.Combine(_root, name), Encoding.GetEncoding(charset).GetBytes(text)); @@ -109,7 +83,7 @@ public void ItBuildsForAMixOfEveryOutcome() ConversionPlan plan = Plan(); - OnUiThread(() => + UiTest.OnStaThread(() => { using var form = new ConversionConfirmationForm(plan); @@ -125,7 +99,7 @@ public void ItBuildsWhenNothingIsRefused() ConversionPlan plan = Plan(); - OnUiThread(() => + UiTest.OnStaThread(() => { using var form = new ConversionConfirmationForm(plan); @@ -145,20 +119,20 @@ public void ItBuildsWhenEverythingIsRefused() Assert.All(plan.Files, f => Assert.Equal(PlannedAction.Refuse, f.Action)); - OnUiThread(() => + UiTest.OnStaThread(() => { using var form = new ConversionConfirmationForm(plan); string text = AllText(form); - Assert.Contains("need an explicit source encoding", text); - Assert.Contains("Nothing to convert", text); + Assert.Contains("require their source encoding to be identified or confirmed", text); + Assert.Contains("Nothing ready to convert", text); Assert.DoesNotContain("Convert 1 file", text); }); } [Fact] - public void ItNamesTheCompetingEncodingsRatherThanJustReportingLowConfidence() + public void ItNamesTheDetectedLegacyEncodingAndOffersAnExplicitChoice() { // "Could not be determined" on its own gives a user nothing to act on. The // alternatives and the way out are what make the refusal actionable. @@ -167,9 +141,7 @@ public void ItNamesTheCompetingEncodingsRatherThanJustReportingLowConfidence() ConversionPlan plan = Plan(); PlannedFile refused = Assert.Single(plan.Files); - Assert.NotEmpty(refused.CompetingEncodings); - - OnUiThread(() => + UiTest.OnStaThread(() => { using var form = new ConversionConfirmationForm(plan); @@ -183,13 +155,45 @@ .. Descendants(form) ]; Assert.Contains("ambiguous.txt", cells); - Assert.Contains(cells, c => c.Contains(refused.CompetingEncodings[0])); + Assert.Contains(refused.SourceEncoding, cells); // And the way out is offered, populated from the charsets EC supports. ComboBox chooser = Assert.Single(Descendants(form).OfType()); Assert.True(chooser.Items.Count > 1); Assert.Contains("windows-1252", chooser.Items.Cast()); + Assert.True(chooser.Width >= 235, "the full source-encoding prompt must be visible"); + }); + } + + [Fact] + public void ItExposesTheReviewControlsToKeyboardAndAssistiveTechnology() + { + Write("legacy.txt", "Le caf\u00e9 \u20ac", "windows-1252"); + + ConversionPlan plan = Plan(); + + UiTest.OnStaThread(() => + { + using var form = new ConversionConfirmationForm(plan); + + ListView list = Assert.Single(Descendants(form).OfType()); + ComboBox source = Assert.Single(Descendants(form).OfType()); + Button confirmSource = Assert.Single( + Descendants(form).OfType