TextEncoding.cs

using System.Text;
using System.Buffers;
using UtfUnknown;

namespace LineEndingNormalizer;

/// <summary>
/// Orchestrates character-encoding detection for files, streams, and byte buffers.
///
/// Controls the detection sample size and delegates encoding detection
/// to specialized byte-level detectors. Unicode encodings are detected
/// using <see cref="UnicodeDetector"/>; if no Unicode encoding is detected,
/// UtfUnknown is used to obtain a legacy encoding candidate, which is then
/// independently verified using strict decoding and text validation before
/// being accepted.
/// </summary>
internal static class TextEncoding
{
    //
    // Maximum number of bytes sampled for encoding detection.
    //
    // 64 KiB is sufficient for the encoding detectors while limiting
    // unnecessary I/O for large files.
    //
    internal const int DefaultMaxSampleBytes = 64 * 1024;

    //
    // Minimum sample size for reliably using entropy to reject binary data.
    //
    private const int MinimumEntropyProbeBytes = 512;

    //
    // Entropy threshold above which sufficiently large samples are treated
    // as likely binary, compressed, or encrypted data rather than text.
    //
    private const double BinaryEntropyThreshold = 7.4;


    /// <summary>
    /// Detects the character encoding of the specified file.
    /// </summary>
    /// <param name="filePath">Path to the file.</param>
    /// <param name="maxSampleBytes">
    /// Maximum number of bytes to examine.
    /// </param>
    /// <returns>
    /// The detected <see cref="Encoding"/>, or <see langword="null"/> if
    /// the encoding could not be detected.
    /// </returns>
    internal static Encoding? DetectFromFile(
        string filePath,
        int maxSampleBytes = DefaultMaxSampleBytes)
    {
        ArgumentNullException.ThrowIfNull(filePath);
        ArgumentOutOfRangeException.ThrowIfNegativeOrZero(
            maxSampleBytes);

        using FileStream stream = new(
            filePath,
            FileMode.Open,
            FileAccess.Read,
            FileShare.ReadWrite | FileShare.Delete,
            4096,
            FileOptions.SequentialScan);

        return DetectFromStream(stream, maxSampleBytes);
    }


    /// <summary>
    /// Detects the character encoding by reading a sample from the
    /// specified stream.
    /// </summary>
    /// <remarks>
    /// The stream must be seekable. The original stream position is
    /// restored when the method returns.
    /// </remarks>
    /// <param name="stream">
    /// Seekable stream containing the data to examine.
    /// </param>
    /// <param name="maxSampleBytes">
    /// Maximum number of bytes to examine.
    /// </param>
    /// <returns>
    /// The detected <see cref="Encoding"/>, or <see langword="null"/> if
    /// the encoding could not be detected.
    /// </returns>
    internal static Encoding? DetectFromStream(
        Stream stream,
        int maxSampleBytes = DefaultMaxSampleBytes)
    {
        ArgumentNullException.ThrowIfNull(stream);

        if (!stream.CanSeek)
        {
            throw new ArgumentException(
                "The stream must be seekable.",
                nameof(stream));
        }

        ArgumentOutOfRangeException.ThrowIfNegativeOrZero(
            maxSampleBytes);

        if (stream.Length == 0L)
            return null;

        long originalPosition = stream.Position;

        int bytesToRead = (int)Math.Min(
            stream.Length,
            maxSampleBytes);

        byte[] buffer =
            ArrayPool<byte>.Shared.Rent(bytesToRead);

        try
        {
            //
            // Always inspect the stream from the beginning.
            //
            stream.Position = 0;

            int bytesRead = stream.ReadAtLeast(
                buffer.AsSpan(0, bytesToRead),
                bytesToRead,
                throwOnEndOfStream: false);

            if (bytesRead == 0)
                return null;

            return DetectFromBuffer(
                buffer.AsSpan(0, bytesRead),
                maxSampleBytes);
        }
        finally
        {
            stream.Position = originalPosition;
            ArrayPool<byte>.Shared.Return(buffer);
        }
    }


    /// <summary>
    /// Detects the character encoding of the specified byte buffer.
    /// </summary>
    /// <param name="buffer">
    /// Buffer containing the bytes to examine.
    /// </param>
    /// <param name="maxSampleBytes">
    /// Maximum number of bytes to examine for encoding detection.
    /// </param>
    /// <returns>
    /// The detected <see cref="Encoding"/>, or <see langword="null"/> if
    /// the encoding could not be detected.
    /// </returns>
    internal static Encoding? DetectFromBuffer(
        ReadOnlySpan<byte> buffer,
        int maxSampleBytes = DefaultMaxSampleBytes)
    {
        ArgumentOutOfRangeException.ThrowIfNegativeOrZero(
            maxSampleBytes);

        int bytesToExamine = Math.Min(
            buffer.Length,
            maxSampleBytes);

        if (bytesToExamine == 0)
            return null;

        buffer = buffer[..bytesToExamine];

        // Reject high-entropy data as likely binary.
        if (buffer.Length >= MinimumEntropyProbeBytes &&
            BinaryEntropy(buffer) > BinaryEntropyThreshold)
        {
            return null;
        }

        // 1. Detect Unicode.
        Encoding? encoding =
            UnicodeDetector.DetectFromBuffer(buffer);

        if (encoding != null)
            return encoding;

        // 2. Detect a legacy encoding using UtfUnknown.
        //
        // https://github.com/CharsetDetector/UTF-unknown
        //
        byte[] bytes = [.. buffer];

        DetectionResult? result;

        try
        {
            result = CharsetDetector.DetectFromBytes(bytes);
        }
        catch (Exception)
        {
            // UtfUnknown's exception surface for malformed input isn't documented; treat
            // any failure the same as "no legacy encoding detected" rather than letting it
            // abort the caller's per-file processing.
            result = null;
        }

        DetectionDetail? detected =
            result?.Detected;

        // Get the System.Text.Encoding of the found encoding (can be null if not available)
        Encoding? legacyEncoding =
            detected?.Encoding;

        if (legacyEncoding is null)
            return null;

        // 3. Accept only legacy encodings verified safe for the raw byte-level
        // CR/LF scan. Other UtfUnknown legacy encodings are treated as undetected.
        if (!IsSafeLegacyEncoding(legacyEncoding))
            return null;

        // 4. Independently validate UtfUnknown's result.
        return TextValidation.IsValidText(
            legacyEncoding,
            buffer)
            ? legacyEncoding
            : null;
    }


    #region Helpers

    /// <summary>
    /// Returns an encoding whose decoder and encoder actually enforce strict fallback.
    /// </summary>
    /// <remarks>
    /// Assigning <see cref="Decoder.Fallback"/> or <see cref="Encoder.Fallback"/> after
    /// <see cref="Encoding.GetDecoder"/>/<see cref="Encoding.GetEncoder"/> has no effect for
    /// the encodings supplied by <see cref="System.Text.CodePagesEncodingProvider"/>: those
    /// codecs take their fallbacks from the parent <see cref="Encoding"/> when they are
    /// created, so a later assignment is silently ignored and unmappable input is replaced
    /// instead of throwing. The fallbacks must be supplied up front, to
    /// <see cref="Encoding.GetEncoding(int, EncoderFallback, DecoderFallback)"/>.
    /// <para>
    /// Only the codecs come from the returned encoding; callers that emit a preamble must
    /// keep using their original instance, which is what carries the requested BOM policy.
    /// </para>
    /// </remarks>
    internal static Encoding Strict(
        Encoding encoding)
    {
        ArgumentNullException.ThrowIfNull(encoding);

        if (encoding.DecoderFallback is DecoderExceptionFallback &&
            encoding.EncoderFallback is EncoderExceptionFallback)
        {
            return encoding;
        }

        try
        {
            return Encoding.GetEncoding(
                encoding.CodePage,
                EncoderFallback.ExceptionFallback,
                DecoderFallback.ExceptionFallback);
        }
        catch (Exception ex) when (ex is ArgumentException or NotSupportedException)
        {
            // Returning the original encoding here could silently re-enable its
            // replacement fallback. A caller that cannot obtain strict semantics must
            // refuse conversion instead.
            throw new NotSupportedException(
                $"Could not construct a strict codec for code page {encoding.CodePage}.",
                ex);
        }
    }


    /// <summary>
    /// Computes Shannon entropy (bits per byte) over the detector sample.
    /// </summary>
    private static double BinaryEntropy(
        ReadOnlySpan<byte> buffer)
    {
        if (buffer.IsEmpty)
            return 0.0;

        Span<int> histogram = stackalloc int[256];
        foreach (byte b in buffer)
        {
            histogram[b]++;
        }
        double entropy = 0.0;
        foreach (int frequency in histogram)
        {
            if (frequency == 0)
                continue;
            double probability =
                (double)frequency / buffer.Length;
            entropy -=
                probability * Math.Log2(probability);
        }
        return entropy;
    }

    #endregion


    /// <summary>
    /// Multi-byte legacy encodings verified safe for the raw byte-level
    /// CR/LF scan in <see cref="LosslessFileWriter"/>. Each was swept to
    /// confirm that 0x0D/0x0A never occur inside a multi-byte sequence.
    /// Keep this an explicit allowlist: other UtfUnknown encodings have not
    /// been verified safe for raw-byte scanning.
    /// </summary>
    private static readonly HashSet<int> SafeMultiByteLegacyCodePages =
    [
        932,   // shift_jis
        51949, // euc-kr
        51932, // euc-jp
        936,   // gb2312 / gbk
        54936, // gb18030
        950,   // big5
    ];


    /// <summary>
    /// Returns whether <paramref name="encoding"/> is safe for raw byte-level
    /// CR/LF normalization. Single-byte encodings are checked generically;
    /// multi-byte encodings require explicit verification in the allowlist.
    /// </summary>
    internal static bool IsSafeLegacyEncoding(
        Encoding encoding)
    {
        ArgumentNullException.ThrowIfNull(encoding);

        if (encoding.IsSingleByte)
            return IsSafeSingleByteEncoding(encoding);

        return SafeMultiByteLegacyCodePages.Contains(encoding.CodePage);
    }


    /// <summary>
    /// Verifies that CR and LF retain their ASCII byte values in this
    /// single-byte encoding, making raw byte-level scanning safe.
    /// </summary>
    private static bool IsSafeSingleByteEncoding(
        Encoding encoding)
    {
        string cr = encoding.GetString([0x0D]);
        string lf = encoding.GetString([0x0A]);

        return cr.Length == 1 && cr[0] == '\r' &&
               lf.Length == 1 && lf[0] == '\n';
    }


    /// <summary>
    /// Returns whether <paramref name="encoding"/> is handled by the Unicode
    /// detection path: ASCII, UTF-8, UTF-16, or UTF-32.
    /// </summary>
    /// <remarks>
    /// Unicode encodings use strict decode/re-encode normalization; legacy
    /// encodings use raw byte-level CR/LF scanning and therefore require
    /// explicit verification that 0x0D/0x0A cannot occur inside characters.
    /// </remarks>
    internal static bool IsUnicodeEncoding(
        Encoding encoding)
    {
        ArgumentNullException.ThrowIfNull(encoding);

        return encoding.CodePage is
            20127 or  // ASCII
            65001 or  // UTF-8
            1200 or   // UTF-16LE
            1201 or   // UTF-16BE
            12000 or  // UTF-32LE
            12001;    // UTF-32BE
    }
}