From a703e14a3e28c31cd3a816fa0875f79ecf0cd85c Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:10:42 +0900 Subject: [PATCH 01/16] fix: restore Lua 5.2 standard library contracts Invoke xpcall handlers before unwinding failing frames, preserve one handler result, and keep handler failures protected. Store the handler and its result in a stack slot so successful calls do not allocate extra state. Preserve cancellation and nested protected-call boundaries. Correct require caching and loader arguments, searchpath defaults and errors, rawset results, print's global tostring lookup, numeric assert messages, integer argument truncation, and multi-format file iterators. Implement load readers and string.dump with Lua 5.2 bytecode headers and representation-aware string serialization. Share numeric scanning across memory and stream readers to preserve trailing input and signed hex values. Add regression coverage and document bytecode and binary I/O compatibility. --- README.md | 4 + README_JA.md | 4 + src/Lua/CodeAnalysis/Compilation/Dump.cs | 27 +- src/Lua/CodeAnalysis/Compilation/Parser.cs | 16 +- .../Compilation/PrototypeBuilder.cs | 9 +- src/Lua/Exceptions.cs | 4 +- src/Lua/IO/NumberReaderHelper.cs | 169 ++++++------- src/Lua/Internal/Utf8Reader.cs | 142 ++--------- src/Lua/LuaFunctionExecutionContext.cs | 9 + src/Lua/LuaState.cs | 68 +++++- src/Lua/LuaStateExtensions.cs | 19 +- src/Lua/Runtime/LuaVirtualMachine.cs | 52 ++-- src/Lua/Runtime/Prototype.cs | 13 + src/Lua/Standard/BasicLibrary.cs | 206 +++++++++++----- src/Lua/Standard/DebugLibrary.cs | 33 ++- src/Lua/Standard/FileHandle.cs | 8 +- src/Lua/Standard/Internal/IOHelper.cs | 10 +- src/Lua/Standard/ModuleLibrary.cs | 117 +++++---- src/Lua/Standard/StringLibrary.cs | 7 +- tests/Lua.Tests/ByteStringIntegrationTests.cs | 6 +- tests/Lua.Tests/LoadReaderTests.cs | 157 ++++++++++++ tests/Lua.Tests/MemoryLifetimeTests.cs | 17 +- tests/Lua.Tests/ModuleContractTests.cs | 110 +++++++++ tests/Lua.Tests/NumberReadTests.cs | 48 ++++ tests/Lua.Tests/ProtectedCallTests.cs | 231 ++++++++++++++++++ .../Lua.Tests/StandardLibraryContractTests.cs | 124 ++++++++++ 26 files changed, 1208 insertions(+), 402 deletions(-) create mode 100644 tests/Lua.Tests/LoadReaderTests.cs create mode 100644 tests/Lua.Tests/ModuleContractTests.cs create mode 100644 tests/Lua.Tests/NumberReadTests.cs create mode 100644 tests/Lua.Tests/ProtectedCallTests.cs create mode 100644 tests/Lua.Tests/StandardLibraryContractTests.cs diff --git a/README.md b/README.md index 0bbf117c..4119cd7b 100644 --- a/README.md +++ b/README.md @@ -773,6 +773,10 @@ Paths and names passed to `ILuaFileSystem`, `ILuaFileLoader` and `ILuaModuleLoad Files, temporary files and redirected standard I/O preserve bytes, including a BOM read as data. Raw line reads stop at LF and retain CR; `*L` also retains the LF. Invalid UTF-8 is replaced when decoding with `DecodeUtf8`, so keep binary data in byte strings or byte arrays. Passing unconverted .NET characters above U+00FF to raw output writes `?` for those characters. +Lua's `load` accepts source strings and reader functions, including readers that return a binary chunk in pieces. `string.dump` serializes a Lua function's prototype; it does not preserve captured upvalue values. The resulting string holds raw bytes in both string modes. Use byte-mode streams when writing it from Lua, or `LuaByteString.ToBytes` when writing it from C#. UTF-16 streams encode text as UTF-8 and cannot preserve arbitrary binary strings. + +`Prototype.StringMode` records its string representation, so `ToBytecode` and `WriteBytecode` preserve byte-mode constants. `Prototype.FromBytecode(bytes, name, stringMode)` selects the representation when loading without a state; `LuaState.Load` uses the state's mode. Bytecode uses the Lua 5.2 function-header layout. Regenerate older Lua-CSharp chunks that used a different field order. + ### Garbage Collection Since Lua-CSharp is implemented in C#, it relies on .NET's garbage collector. As a result, memory management behavior differs from regular Lua. diff --git a/README_JA.md b/README_JA.md index 1b28b8f0..4aa75c47 100644 --- a/README_JA.md +++ b/README_JA.md @@ -667,6 +667,10 @@ print(l) Stringライブラリの関数はすべて文字列をUTF-16として扱う実装に変更されていることに注意してください。 +`load`は文字列とreader関数に対応し、バイナリチャンクを分割して返すreaderも利用できます。`string.dump`はLua関数のプロトタイプを保存しますが、キャプチャしたupvalueの値は保存しません。戻り値はどちらの文字列モードでも生のバイト列です。Luaからファイルに書く場合はバイトモードのストリームを、C#から書く場合は`LuaByteString.ToBytes`を使用してください。UTF-16モードのストリームはUTF-8でテキストを書き込むため、任意のバイナリ文字列を保持できません。 + +`Prototype.StringMode`は文字列表現を保持し、`ToBytecode`と`WriteBytecode`でもバイトモードの定数を保持します。stateを使わずに読み込む場合は`Prototype.FromBytecode(bytes, name, stringMode)`で表現を指定できます。`LuaState.Load`はstateのモードを使用します。バイトコードはLua 5.2の関数ヘッダー順に従います。異なる順序で生成された古いLua-CSharpのチャンクは再生成してください。 + ### ガベージコレクション Lua-CSharpはC#で実装されているため.NETのGCに依存しています。そのため、メモリ管理に関する動作が通常のLuaとは異なります。 diff --git a/src/Lua/CodeAnalysis/Compilation/Dump.cs b/src/Lua/CodeAnalysis/Compilation/Dump.cs index 4811b82b..0057b106 100644 --- a/src/Lua/CodeAnalysis/Compilation/Dump.cs +++ b/src/Lua/CodeAnalysis/Compilation/Dump.cs @@ -5,6 +5,7 @@ using System.Runtime.InteropServices; using System.Text; using Lua.Internal; +using Lua.IO; using Lua.Runtime; namespace Lua.CodeAnalysis.Compilation; @@ -96,8 +97,14 @@ public void Validate(ReadOnlySpan name) } } -unsafe ref struct DumpState(IBufferWriter writer, bool reversedEndian) +unsafe ref struct DumpState( + IBufferWriter writer, + bool reversedEndian, + LuaStringMode stringMode +) { + readonly Encoding encoding = + stringMode == LuaStringMode.Bytes ? LuaByteEncoding.Instance : Encoding.UTF8; public readonly IBufferWriter Writer = writer; Span unWritten; @@ -142,8 +149,8 @@ void DumpFunction(Prototype prototype) WriteInt(prototype.LineDefined); // 4 WriteInt(prototype.LastLineDefined); // 4 WriteByte((byte)prototype.ParameterCount); // 1 - WriteByte((byte)prototype.MaxStackSize); // 1 WriteByte((byte)(prototype.HasVariableArguments ? 1 : 0)); // 1 + WriteByte((byte)prototype.MaxStackSize); // 1 WriteIntSpanWithLength(MemoryMarshal.Cast(prototype.Code)); // 4 WriteConstants(prototype.Constants); // 4 WritePrototypes(prototype.ChildPrototypes); // 4 @@ -215,7 +222,7 @@ void WriteBool(bool v) void WriteString(string v) { - var bytes = Encoding.UTF8.GetBytes(v); + var bytes = encoding.GetBytes(v); var len = bytes.Length; if (bytes.Length != 0) { @@ -295,9 +302,12 @@ void WriteUpValues(ReadOnlySpan upValues) unsafe ref struct UndumpState( ReadOnlySpan span, ReadOnlySpan name, - StringInternPool internPool + StringInternPool internPool, + LuaStringMode stringMode ) { + readonly Encoding encoding = + stringMode == LuaStringMode.Bytes ? LuaByteEncoding.Instance : Encoding.UTF8; public ReadOnlySpan Unread = span; bool otherEndian; int pointerSize; @@ -412,8 +422,8 @@ Prototype UndumpFunction(int depth) var lineDefined = ReadInt(); // 4 var lastLineDefined = ReadInt(); // 4 var parameterCount = ReadByte(); // 1 - var maxStackSize = ReadByte(); // 1 var isVarArg = ReadBool(); // 1 + var maxStackSize = ReadByte(); // 1 var codeLength = ReadCount(sizeof(int)); var code = new Instruction[codeLength]; ReadInToIntSpan(MemoryMarshal.Cast(code)); @@ -450,7 +460,10 @@ Prototype UndumpFunction(int depth) lineInfo, localVariables, upValues - ); + ) + { + StringMode = stringMode, + }; new BytecodeVerifier(prototype, name).Verify(); return prototype; } @@ -489,7 +502,7 @@ string ReadString() len <= 128 ? stackalloc char[len] : (charArrayPooled = ArrayPool.Shared.Rent(len)); - var count = Encoding.UTF8.GetChars(bytes, chars); + var count = encoding.GetChars(bytes, chars); return internPool.Intern(chars[..count]); } finally diff --git a/src/Lua/CodeAnalysis/Compilation/Parser.cs b/src/Lua/CodeAnalysis/Compilation/Parser.cs index 734530c7..646e9d16 100644 --- a/src/Lua/CodeAnalysis/Compilation/Parser.cs +++ b/src/Lua/CodeAnalysis/Compilation/Parser.cs @@ -979,7 +979,7 @@ public static Prototype Parse(LuaState l, TextReader r, string name) f.Proto.IsVarArg = true; f.Proto.LineDefined = 0; p.MainFunction(); - var prototype = f.Proto.CreatePrototypeAndRelease(); + var prototype = f.Proto.CreatePrototypeAndRelease(l.Platform.StringMode); f.Release(); return prototype; } @@ -990,7 +990,11 @@ public static void Dump( bool useLittleEndian = true ) { - DumpState state = new(writer, useLittleEndian ^ BitConverter.IsLittleEndian); + DumpState state = new( + writer, + useLittleEndian ^ BitConverter.IsLittleEndian, + prototype.StringMode + ); state.Dump(prototype); } @@ -1001,7 +1005,11 @@ public static byte[] Dump(Prototype prototype, bool useLittleEndian = true) return writer.WrittenSpan.ToArray(); } - public static Prototype Undump(ReadOnlySpan span, ReadOnlySpan name) + public static Prototype Undump( + ReadOnlySpan span, + ReadOnlySpan name, + LuaStringMode stringMode = LuaStringMode.Utf16 + ) { if (name.Length > 0) { @@ -1014,7 +1022,7 @@ public static Prototype Undump(ReadOnlySpan span, ReadOnlySpan name) } using var internPool = new StringInternPool(4); - UndumpState state = new(span, name, internPool); + UndumpState state = new(span, name, internPool, stringMode); return state.Undump(); } } diff --git a/src/Lua/CodeAnalysis/Compilation/PrototypeBuilder.cs b/src/Lua/CodeAnalysis/Compilation/PrototypeBuilder.cs index b3421253..3ffc0482 100644 --- a/src/Lua/CodeAnalysis/Compilation/PrototypeBuilder.cs +++ b/src/Lua/CodeAnalysis/Compilation/PrototypeBuilder.cs @@ -80,13 +80,13 @@ internal void Release() pool.TryPush(this); } - public Prototype CreatePrototypeAndRelease() + public Prototype CreatePrototypeAndRelease(LuaStringMode stringMode) { var protoTypes = Prototypes.Length == 0 ? Array.Empty() : new Prototype[Prototypes.Length]; for (var i = 0; i < Prototypes.Length; i++) { - protoTypes[i] = Prototypes[i].CreatePrototypeAndRelease(); //ref + protoTypes[i] = Prototypes[i].CreatePrototypeAndRelease(stringMode); } Prototype p = new( @@ -102,7 +102,10 @@ public Prototype CreatePrototypeAndRelease() LineInfo.ToArray(), LocalVariables.ToArray(), UpValues.ToArray() - ); + ) + { + StringMode = stringMode, + }; Release(); return p; } diff --git a/src/Lua/Exceptions.cs b/src/Lua/Exceptions.cs index c3214ecd..3c7a465f 100644 --- a/src/Lua/Exceptions.cs +++ b/src/Lua/Exceptions.cs @@ -513,7 +513,9 @@ internal string MinimalMessage(LuaPlatform platform) { if (State != null) { - var callStack = State.ExceptionTrace.AsSpan(); + ReadOnlySpan callStack = State.ExceptionTrace.AsSpan(); + if (callStack.IsEmpty) + callStack = State.GetCallStackFrames(); level = Math.Min(level, callStack.Length + 1); callStack = callStack[..^(level - 1)]; if (callStack.IsEmpty) diff --git a/src/Lua/IO/NumberReaderHelper.cs b/src/Lua/IO/NumberReaderHelper.cs index ec348914..afbcfaec 100644 --- a/src/Lua/IO/NumberReaderHelper.cs +++ b/src/Lua/IO/NumberReaderHelper.cs @@ -1,4 +1,5 @@ using System.Globalization; +using System.Runtime.CompilerServices; using Lua.Internal; namespace Lua.IO; @@ -6,119 +7,97 @@ namespace Lua.IO; internal static class NumberReaderHelper { /// - /// Scans a span of characters to find the extent of a valid number. - /// Returns the length of the number portion, or 0 if no valid number is found. + /// Scans a numeric input item. Parsing validates incomplete signs, + /// hexadecimal prefixes and exponents. /// /// The span to scan - /// The length of the valid number portion + /// The number of characters consumed before the first nonnumeric character public static int ScanNumberLength(ReadOnlySpan span) { + Scanner scanner = new(); var position = 0; - - if (position >= span.Length) - { - return 0; - } - - var hasStarted = false; - var isHex = false; - var hasDecimal = false; - var lastWasE = false; - - // Check for sign - if (position < span.Length && (span[position] == '+' || span[position] == '-')) - { + while (position < span.Length && scanner.TryConsume(span[position])) position++; - hasStarted = true; - } + return position; + } - // Check for hex prefix right at the start (after optional sign) - if ( - position < span.Length - 1 - && span[position] == '0' - && (span[position + 1] == 'x' || span[position + 1] == 'X') - ) + internal struct Scanner + { + enum Part : byte { - isHex = true; - position += 2; // Skip '0x' or '0X' - hasStarted = true; + Start, + Sign, + Zero, + Integer, + Fraction, + Exponent, + ExponentDigits, } - // Scan for valid number characters - while (position < span.Length) - { - var c = span[position]; - - // Hex prefix is handled above before the loop + Part part; + bool hex; + bool hasDigits; - if (isHex) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool TryConsume(char c) + { + if (c is >= '0' and <= '9') { - // Hex digits - if (c is >= '0' and <= '9' or >= 'a' and <= 'f' or >= 'A' and <= 'F') - { - position++; - hasStarted = true; - } - // Hex decimal point - else if (c == '.' && !hasDecimal) - { - position++; - hasDecimal = true; - } - // Hex exponent (p or P) - else if (c is 'p' or 'P' && hasStarted) - { - position++; - lastWasE = true; - } - // Sign after exponent - else if (lastWasE && c is '+' or '-') - { - position++; - lastWasE = false; - } - else - { - break; - } + if (part is Part.Start or Part.Sign) + part = c == '0' ? Part.Zero : Part.Integer; + else if (part == Part.Zero) + part = Part.Integer; + else if (part == Part.Exponent) + part = Part.ExponentDigits; + hasDigits = true; + return true; } - else + if (part == Part.Start) { - // Decimal digits - if (c is >= '0' and <= '9') - { - position++; - hasStarted = true; - lastWasE = false; - } - // Decimal point - else if (c == '.' && !hasDecimal) - { - position++; - hasDecimal = true; - lastWasE = false; - } - // Exponent (e or E) - else if (c is 'e' or 'E' && hasStarted) - { - position++; - lastWasE = true; - } - // Sign after exponent - else if (lastWasE && c is '+' or '-') - { - position++; - lastWasE = false; - } - else + part = Part.Sign; + if (c is '+' or '-') + return true; + } + if (part == Part.Sign) + { + part = Part.Integer; + } + else if (part == Part.Zero) + { + part = Part.Integer; + if (c is 'x' or 'X') { - break; + hex = true; + hasDigits = false; + return true; } } - } - // Return the length of the number portion - return position; + if (part == Part.Exponent) + { + part = Part.ExponentDigits; + if (c is '+' or '-') + return true; + } + if (part == Part.ExponentDigits) + return false; + if (hex && c is >= 'a' and <= 'f' or >= 'A' and <= 'F') + { + hasDigits = true; + return true; + } + if (part == Part.Integer && c == '.') + { + part = Part.Fraction; + return true; + } + if (hasDigits && (hex ? c is 'p' or 'P' : c is 'e' or 'E')) + { + part = Part.Exponent; + return true; + } + return false; + } } public static bool TryParseToDouble(ReadOnlySpan span, out double result) diff --git a/src/Lua/Internal/Utf8Reader.cs b/src/Lua/Internal/Utf8Reader.cs index b85aed7c..0659ce63 100644 --- a/src/Lua/Internal/Utf8Reader.cs +++ b/src/Lua/Internal/Utf8Reader.cs @@ -1,4 +1,4 @@ -using System.Buffers; +using System.Buffers; using System.Text; using Lua.CodeAnalysis.Compilation; using Lua.IO; @@ -198,10 +198,7 @@ public void Clear() { var resultBuffer = ArrayPool.Shared.Rent(64); // Numbers shouldn't be too long var len = 0; - var hasStarted = false; - var isHex = false; - var hasDecimal = false; - var lastWasE = false; + NumberReaderHelper.Scanner scanner = new(); try { @@ -223,114 +220,21 @@ public void Clear() ReadByte(stream); // Consume whitespace } - // Check for hex prefix at the start - if (PeekByte(stream) == '0') - { - var nextByte = PeekByte(stream, 1); - if (nextByte == 'x' || nextByte == 'X') - { - isHex = true; - resultBuffer[len++] = '0'; - ReadByte(stream); - resultBuffer[len++] = (char)ReadByte(stream); - hasStarted = true; - } - } - - // Read number characters while (true) { var b = PeekByte(stream); - if (b == -1) - { - break; // EOF - } - - var c = (char)b; - var shouldConsume = false; - - if (!hasStarted && (c == '+' || c == '-')) - { - shouldConsume = true; - hasStarted = true; - } - else if (isHex) - { - // Hex digits - if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) - { - shouldConsume = true; - hasStarted = true; - } - // Hex decimal point - else if (c == '.' && !hasDecimal) - { - shouldConsume = true; - hasDecimal = true; - } - // Hex exponent (p or P) - else if ((c == 'p' || c == 'P') && hasStarted) - { - shouldConsume = true; - lastWasE = true; - } - // Sign after exponent - else if (lastWasE && (c == '+' || c == '-')) - { - shouldConsume = true; - lastWasE = false; - } - } - else - { - // Decimal digits - if (c >= '0' && c <= '9') - { - shouldConsume = true; - hasStarted = true; - lastWasE = false; - } - // Decimal point - else if (c == '.' && !hasDecimal) - { - shouldConsume = true; - hasDecimal = true; - lastWasE = false; - } - // Exponent (e or E) - else if ((c == 'e' || c == 'E') && hasStarted) - { - shouldConsume = true; - lastWasE = true; - } - // Sign after exponent - else if (lastWasE && (c == '+' || c == '-')) - { - shouldConsume = true; - lastWasE = false; - } - } - - if (shouldConsume) - { - if (len >= resultBuffer.Length) - { - // Number too long, expand buffer - var newBuffer = ArrayPool.Shared.Rent(resultBuffer.Length * 2); - resultBuffer.AsSpan(0, len).CopyTo(newBuffer); - ArrayPool.Shared.Return(resultBuffer); - resultBuffer = newBuffer; - } - - resultBuffer[len++] = c; - ReadByte(stream); // Consume the byte - } - else + if (b < 0 || !scanner.TryConsume((char)b)) + break; + if (len == resultBuffer.Length) { - break; // Not part of the number + var expanded = ArrayPool.Shared.Rent(resultBuffer.Length * 2); + resultBuffer.AsSpan(0, len).CopyTo(expanded); + ArrayPool.Shared.Return(resultBuffer); + resultBuffer = expanded; } + resultBuffer[len++] = (char)b; + ReadByte(stream); } - return len == 0 ? null : NumberReaderHelper.ParseNumber(resultBuffer.AsSpan(0, len)); } finally @@ -344,28 +248,16 @@ public bool IsEndOfStream(Stream stream) return PeekByte(stream) < 0; } - int PeekByte(Stream stream, int offset = 0) + int PeekByte(Stream stream) { - // Ensure we have enough data in buffer - while (bufPos + offset >= bufLen) + if (bufPos >= bufLen) { - if (bufLen == 0 || bufPos == bufLen) - { - bufLen = stream.Read(buffer, 0, buffer.Length); - bufPos = 0; - if (bufLen == 0) - { - return -1; // EOF - } - } - else - { - // We need more data but buffer has some - this shouldn't happen with small offsets + bufLen = stream.Read(buffer, 0, buffer.Length); + bufPos = 0; + if (bufLen == 0) return -1; - } } - - return buffer[bufPos + offset]; + return buffer[bufPos]; } public void Dispose() diff --git a/src/Lua/LuaFunctionExecutionContext.cs b/src/Lua/LuaFunctionExecutionContext.cs index bd6036c6..1eacf58e 100644 --- a/src/Lua/LuaFunctionExecutionContext.cs +++ b/src/Lua/LuaFunctionExecutionContext.cs @@ -84,6 +84,15 @@ internal string GetStringArgument(int index) return null!; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal int GetIntegerArgument(int index) + { + var number = Math.Truncate(GetArgument(index)); + if (double.IsNaN(number) || number < int.MinValue || number > int.MaxValue) + LuaRuntimeException.BadArgumentNumberIsNotInteger(State, index + 1); + return (int)number; + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal T GetArgumentOrDefault(int index, T defaultValue = default!) { diff --git a/src/Lua/LuaState.cs b/src/Lua/LuaState.cs index 54a9801d..ce01fe47 100644 --- a/src/Lua/LuaState.cs +++ b/src/Lua/LuaState.cs @@ -146,6 +146,8 @@ public ValueTask YieldAsync( CancellationToken cancellationToken = default ) { + if (isInErrorHandler) + throw new LuaRuntimeException(this, "attempt to yield across a C-call boundary"); if (coroutine is not null) { return coroutine.YieldAsyncCore( @@ -162,6 +164,8 @@ public ValueTask YieldAsync( public ValueTask YieldAsync(LuaStack stack, CancellationToken cancellationToken = default) { + if (isInErrorHandler) + throw new LuaRuntimeException(this, "attempt to yield across a C-call boundary"); if (coroutine is not null) { return coroutine.YieldAsyncCore(stack, stack.Count, 0, null, cancellationToken); @@ -218,6 +222,8 @@ public void Release() internal int LastPc; internal ILuaTracebackBuildable? CurrentException; + internal int ErrorHandlerIndex; + bool isInErrorHandler; internal readonly ReversedStack ExceptionTrace = new(); internal Prototype? LastCallerPrototype; @@ -598,6 +604,13 @@ public async ValueTask RunAsync( return await function.Func(context, cancellationToken); } + catch (Exception exception) + when (ErrorHandlerIndex != 0 && exception is not OperationCanceledException) + { + var handled = await HandleErrorAsync(exception, cancellationToken); + Stack.PopUntil(returnBase); + throw handled; + } catch { Stack.PopUntil(returnBase); @@ -611,6 +624,59 @@ public async ValueTask RunAsync( } } + [AsyncMethodBuilder(typeof(LightAsyncValueTaskMethodBuilder<>))] + internal async ValueTask HandleErrorAsync( + Exception exception, + CancellationToken cancellationToken + ) + { + if (ErrorHandlerIndex == 0 || exception is OperationCanceledException) + return exception; + + // Consume this protected call's handler before entering it. Nested protected + // calls can install their own handler without handling the same error twice. + var handlerIndex = ErrorHandlerIndex - 1; + var handler = Stack.Get(handlerIndex); + ErrorHandlerIndex = 0; + LuaValue error; + if (exception is LuaRuntimeException luaException) + { + error = + luaException.InnerException is { } inner ? Platform.GetErrorMessage(inner) + : luaException.ErrorObject.Type == LuaValueType.String + ? luaException.MinimalMessage(Platform) + : luaException.ErrorObject; + luaException.Forget(); + } + else + error = Platform.GetErrorMessage(exception); + + var top = Stack.Count; + var previousInHandler = isInErrorHandler; + isInErrorHandler = true; + try + { + Stack.PushRange(handler, error); + var count = await LuaVirtualMachine.Call(this, top, top, cancellationToken); + error = count > 0 ? Stack.Get(top) : LuaValue.Nil; + } + catch (Exception handlerException) + { + if (handlerException is OperationCanceledException) + return handlerException; + if (handlerException is LuaRuntimeException luaHandlerException) + luaHandlerException.Forget(); + error = "error in error handling"; + } + finally + { + isInErrorHandler = previousInHandler; + Stack.PopUntil(top); + } + Stack.Get(handlerIndex) = error; + return exception; + } + /// /// Loads .NET source text and a .NET chunk name. In byte-string mode, text is /// encoded as UTF-8. Use the byte overload for source that is already encoded. @@ -700,7 +766,7 @@ static bool AllowsMode(string mode, char chunkMode) throw new Exception("attempt to load a binary chunk (mode is 't')"); } - return new(this, Parser.Undump(chunk, chunkName), environment); + return new(this, Parser.Undump(chunk, chunkName, Platform.StringMode), environment); } } diff --git a/src/Lua/LuaStateExtensions.cs b/src/Lua/LuaStateExtensions.cs index cf9836b4..005655ab 100644 --- a/src/Lua/LuaStateExtensions.cs +++ b/src/Lua/LuaStateExtensions.cs @@ -1,5 +1,6 @@ using System.Buffers; using System.Runtime.CompilerServices; +using Lua.CodeAnalysis.Compilation; using Lua.Internal; using Lua.Internal.CompilerServices; using Lua.IO; @@ -42,7 +43,7 @@ CancellationToken cancellationToken if (stream is ILuaByteStream byteStream) { var firstByte = await byteStream.ReadByteAsync(cancellationToken); - if (firstByte != '\e' && !mode.Contains('t')) + if (firstByte is not ('\e' or '#' or 0xEF) && !mode.Contains('t')) { throw new Exception("attempt to load a text chunk (mode is 'b')"); } @@ -57,7 +58,21 @@ CancellationToken cancellationToken source.GetSpan(1)[0] = (byte)firstByte; source.Advance(1); await byteStream.ReadBytesAsync(source, cancellationToken); - closure = state.LoadCore(source.WrittenSpan, name, mode, environment); + var bytes = source.WrittenSpan; + var chunk = BomUtility.SkipUtf8Bom(bytes); + if (!chunk.IsEmpty && chunk[0] == '#') + { + var newline = chunk.IndexOf((byte)'\n'); + chunk = newline < 0 ? default : chunk[(newline + 1)..]; + } + // Text keeps its initial comment for line numbers. Binary chunks + // begin after the optional BOM and first-line comment. + closure = state.LoadCore( + !chunk.IsEmpty && chunk[0] == '\e' ? chunk : bytes, + name, + mode, + environment + ); } } else if (!mode.Contains('t')) diff --git a/src/Lua/Runtime/LuaVirtualMachine.cs b/src/Lua/Runtime/LuaVirtualMachine.cs index c019be60..652cae7a 100644 --- a/src/Lua/Runtime/LuaVirtualMachine.cs +++ b/src/Lua/Runtime/LuaVirtualMachine.cs @@ -311,30 +311,34 @@ or PostOperationType.DontPop return State.Stack.Count - returnFrameBase; } catch (Exception e) + when (State.ErrorHandlerIndex != 0 && e is not OperationCanceledException) { - // FrameBase follows nested calls; all discarded frames need closing. + var exception = await State.HandleErrorAsync(e, CancellationToken); State.CloseUpValues(InitialArgumentBase); - if (toCatchFlag) + PopOnTopCallStackFrames(); + throw exception; + } + catch (Exception e) + { + Exception exception = e; + if ( + toCatchFlag + && e is not (LuaRuntimeException or LuaCanceledException) + && PostOperation != PostOperationType.Return + ) { - if ( - e is not (LuaRuntimeException or LuaCanceledException) - && PostOperation != PostOperationType.Return - ) - { - Exception newException = e is OperationCanceledException cancelled - ? new LuaCanceledException( - State, - State.GetCancellationOrigin(cancelled.CancellationToken), - e - ) - : new LuaRuntimeException(State, e); - PopOnTopCallStackFrames(); - throw newException; - } - - PopOnTopCallStackFrames(); + exception = e is OperationCanceledException cancelled + ? new LuaCanceledException( + State, + State.GetCancellationOrigin(cancelled.CancellationToken), + e + ) + : new LuaRuntimeException(State, e); } - + State.CloseUpValues(InitialArgumentBase); + PopOnTopCallStackFrames(); + if (!ReferenceEquals(exception, e)) + throw exception; throw; } finally @@ -1123,7 +1127,6 @@ static void VarArg(VirtualMachineExecutionContext context) } catch (Exception e) { - context.State.CloseUpValues(context.InitialArgumentBase); if ( e is not (LuaRuntimeException or LuaCanceledException) && context.PostOperation != PostOperationType.Return @@ -1136,11 +1139,9 @@ e is not (LuaRuntimeException or LuaCanceledException) e ) : new LuaRuntimeException(context.State, e); - context.PopOnTopCallStackFrames(); throw newException; } - context.PopOnTopCallStackFrames(); throw; } } @@ -1617,6 +1618,11 @@ CancellationToken cancellationToken throw; } + catch (Exception exception) when (state.ErrorHandlerIndex != 0) + { + var handled = await state.HandleErrorAsync(exception, cancellationToken); + throw handled; + } finally { state.PopCallStackFrame(); diff --git a/src/Lua/Runtime/Prototype.cs b/src/Lua/Runtime/Prototype.cs index 6d4ee32b..7da250d6 100644 --- a/src/Lua/Runtime/Prototype.cs +++ b/src/Lua/Runtime/Prototype.cs @@ -19,6 +19,9 @@ public sealed class Prototype( UpValueDesc[] upValues ) { + /// The representation of strings in this prototype and its bytecode. + public LuaStringMode StringMode { get; init; } + public ReadOnlySpan Constants => constants; public ReadOnlySpan Code => code; @@ -55,6 +58,16 @@ public static Prototype FromBytecode(ReadOnlySpan span, ReadOnlySpan return Parser.Undump(span, name); } + /// Loads bytecode using the selected representation for its strings. + public static Prototype FromBytecode( + ReadOnlySpan span, + ReadOnlySpan name, + LuaStringMode stringMode + ) + { + return Parser.Undump(span, name, stringMode); + } + /// /// Converts a Prototype object to a Lua bytecode. /// diff --git a/src/Lua/Standard/BasicLibrary.cs b/src/Lua/Standard/BasicLibrary.cs index 7df436ef..5dfc5460 100644 --- a/src/Lua/Standard/BasicLibrary.cs +++ b/src/Lua/Standard/BasicLibrary.cs @@ -1,7 +1,9 @@ using System.Globalization; using System.Runtime.CompilerServices; +using Lua.CodeAnalysis.Compilation; using Lua.Internal; using Lua.Internal.CompilerServices; +using Lua.IO; using Lua.Runtime; // ReSharper disable MethodHasAsyncOverloadWithCancellation @@ -93,7 +95,7 @@ CancellationToken cancellationToken var message = "assertion failed!"; if (context.HasArgument(1)) { - message = context.GetArgument(1); + message = context.GetStringArgument(1); } throw new LuaAssertionException(context.State, message); @@ -146,7 +148,7 @@ CancellationToken cancellationToken ) { var value = context.ArgumentCount == 0 ? LuaValue.Nil : context.Arguments[0]; - var level = context.HasArgument(1) ? context.GetArgument(1) : 1; + var level = context.HasArgument(1) ? context.GetIntegerArgument(1) : 1; throw new LuaRuntimeException(context.State, value, level); } @@ -244,7 +246,8 @@ CancellationToken cancellationToken } } - public ValueTask Load( + [AsyncMethodBuilder(typeof(LightAsyncValueTaskMethodBuilder<>))] + public async ValueTask Load( LuaFunctionExecutionContext context, CancellationToken cancellationToken ) @@ -257,47 +260,92 @@ CancellationToken cancellationToken var hasEnvironment = context.ArgumentCount > 3; var environment = hasEnvironment ? context.GetArgument(3) : LuaValue.Nil; + if (arg0.Type is not (LuaValueType.String or LuaValueType.Number or LuaValueType.Function)) + context.ThrowBadArgument(1, "function expected"); + var previousHandler = context.State.ErrorHandlerIndex; + context.State.ErrorHandlerIndex = 0; // do not use LuaState.DoFileAsync as it uses the newExecutionContext try { - if (arg0.TryRead(out var str)) + LuaClosure closure; + if (arg0.Type is LuaValueType.String or LuaValueType.Number) { - if (!mode.Contains('t')) - { - throw new Exception("attempt to load a text chunk (mode is 'b')"); - } - - var closure = context.State.LoadCore(str, name ?? str, null); - if (hasEnvironment) - { - closure.SetEnvironment(environment); - } - - return new(context.Return(closure)); + var source = context.GetStringArgument(0); + closure = LoadSource(context.State, source, name ?? source, mode); } - else if (arg0.TryRead(out var function)) + else { - // TODO: - throw new NotImplementedException(); + var function = arg0.Read(); + using var source = new PooledList(256); + var stack = context.State.Stack; + var top = stack.Count; + while (true) + { + var count = await context.State.RunAsync(function, 0, top, cancellationToken); + var piece = count == 0 ? LuaValue.Nil : stack.Get(top); + if (piece.Type == LuaValueType.Nil) + { + stack.PopUntil(top); + break; + } + if (piece.Type is not (LuaValueType.String or LuaValueType.Number)) + throw new LuaRuntimeException( + context.State, + "reader function must return a string" + ); + var text = piece.ToLuaString(context.State.Platform.NumberFormat); + stack.PopUntil(top); + if (text.Length == 0) + break; + source.AddRange(text); + } + closure = LoadSource(context.State, source.AsSpan(), name ?? "=(load)", mode); } - else + if (hasEnvironment) + closure.SetEnvironment(environment); + return context.Return(closure); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + if (ex is LuaRuntimeException luaException) { - LuaRuntimeException.BadArgument( - context.State, - 1, - ["string", "function,binary data"], - arg0.TypeToString() - ); - return default; // dummy + var error = + luaException.InnerException == null + && luaException.ErrorObject.Type != LuaValueType.String + ? luaException.ErrorObject + : luaException.MinimalMessage(context.State.Platform); + luaException.Forget(); + return context.Return(LuaValue.Nil, error); } + return context.Return(LuaValue.Nil, context.State.Platform.GetErrorMessage(ex)); } - catch (Exception ex) + finally { - return new(context.Return(LuaValue.Nil, context.State.Platform.GetErrorMessage(ex))); + context.State.ErrorHandlerIndex = previousHandler; } } + static LuaClosure LoadSource( + LuaState state, + ReadOnlySpan source, + string name, + string mode + ) + { + if (!source.IsEmpty && source[0] == '\e') + { + if (!mode.Contains('b')) + throw new Exception("attempt to load a binary chunk (mode is 't')"); + using PooledArray bytes = new(source.Length); + LuaByteEncoding.Instance.GetBytes(source, bytes.AsSpan()); + return new(state, Parser.Undump(bytes.AsSpan(), name, state.Platform.StringMode)); + } + if (!mode.Contains('t')) + throw new Exception("attempt to load a text chunk (mode is 'b')"); + return state.LoadCore(source, name); + } + public ValueTask Next( LuaFunctionExecutionContext context, CancellationToken cancellationToken @@ -359,7 +407,10 @@ public async ValueTask PCall( CancellationToken cancellationToken ) { + context.GetArgument(0); var frameCount = context.State.CallStackFrameCount; + var previousHandler = context.State.ErrorHandlerIndex; + context.State.ErrorHandlerIndex = 0; try { var count = await LuaVirtualMachine.Call( @@ -395,6 +446,10 @@ CancellationToken cancellationToken return context.Return(false, context.State.Platform.GetErrorMessage(ex)); } } + finally + { + context.State.ErrorHandlerIndex = previousHandler; + } } [AsyncMethodBuilder(typeof(LightAsyncValueTaskMethodBuilder<>))] @@ -404,11 +459,24 @@ CancellationToken cancellationToken ) { var stdout = context.GlobalState.Platform.StandardIO.Output; + var toString = context.State.Environment["tostring"]; + var argumentBase = context.FrameBase; for (var i = 0; i < context.ArgumentCount; i++) { - await context.Arguments[i].CallToStringAsync(context, cancellationToken); - await stdout.WriteAsync(context.State.Stack.Pop().Read(), cancellationToken); + var stack = context.State.Stack; + var top = stack.Count; + stack.PushRange(toString, stack.Get(argumentBase + i)); + var count = await LuaVirtualMachine.Call(context.State, top, top, cancellationToken); + var value = count > 0 ? stack.Get(top) : LuaValue.Nil; + if (value.Type is not (LuaValueType.String or LuaValueType.Number)) + throw new LuaRuntimeException( + context.State, + "'tostring' must return a string to 'print'" + ); + var text = value.ToLuaString(context.State.Platform.NumberFormat); + stack.PopUntil(top); + await stdout.WriteAsync(text, cancellationToken); if (i < context.ArgumentCount - 1) { await stdout.WriteAsync("\t", cancellationToken); @@ -478,7 +546,7 @@ CancellationToken cancellationToken var arg2 = context.GetArgument(2); arg0[arg1] = arg2; - return new(context.Return()); + return new(context.Return(arg0)); } public ValueTask Select( @@ -648,54 +716,62 @@ public async ValueTask XPCall( CancellationToken cancellationToken ) { - var frameCount = context.State.CallStackFrameCount; + var state = context.State; + var frameCount = state.CallStackFrameCount; var arg0 = context.GetArgument(0); - var arg1 = context.GetArgument(1); + var arg1 = context.GetArgument(1); + var previousHandler = state.ErrorHandlerIndex; + var functionBase = context.FrameBase + 1; + // Keep the handler below the forwarded arguments and the result range. + var handlerIndex = Math.Min(context.ReturnFrameBase, context.FrameBase); + state.Stack.Get(handlerIndex) = arg1; + state.ErrorHandlerIndex = handlerIndex + 1; try { var stack = context.State.Stack; - stack.Get(context.FrameBase + 1) = arg0; + stack.Get(functionBase) = arg0; var count = await LuaVirtualMachine.Call( context.State, - context.FrameBase + 1, + functionBase, context.ReturnFrameBase + 1, cancellationToken ); - context.State.Stack.Get(context.ReturnFrameBase) = true; + stack.Get(context.ReturnFrameBase) = true; return count + 1; } catch (Exception ex) + when (state.ErrorHandlerIndex != 0 && ex is not OperationCanceledException) { - var state = context.State; - state.PopCallStackFrameUntil(frameCount); - cancellationToken.ThrowIfCancellationRequested(); - - if (ex is LuaRuntimeException luaEx) - { - var error = - state.Platform.StringMode == LuaStringMode.Bytes - && luaEx.InnerException is { } inner - ? state.Platform.GetErrorMessage(inner) - : luaEx.ErrorObject; - luaEx.Forget(); - state.Push(error); - } - else - { - state.Push(state.Platform.GetErrorMessage(ex)); - } - - // invoke error handler - var count = await state.RunAsync( - arg1, - 1, - context.ReturnFrameBase + 1, - cancellationToken - ); - context.State.Stack.Get(context.ReturnFrameBase) = false; - return count + 1; + ex = await state.HandleErrorAsync(ex, cancellationToken); + return FinishXPCall(context, ex, frameCount, handlerIndex, cancellationToken); } + catch (Exception ex) + { + return FinishXPCall(context, ex, frameCount, handlerIndex, cancellationToken); + } + finally + { + state.ErrorHandlerIndex = previousHandler; + } + } + + static int FinishXPCall( + LuaFunctionExecutionContext context, + Exception exception, + int frameCount, + int handlerIndex, + CancellationToken cancellationToken + ) + { + context.State.PopCallStackFrameUntil(frameCount); + if (exception is LuaCanceledException) + throw exception; + if (exception is OperationCanceledException) + throw new LuaCanceledException(context.State, cancellationToken, exception); + if (exception is LuaRuntimeException luaException) + luaException.Forget(); + return context.Return(false, context.State.Stack.Get(handlerIndex)); } } diff --git a/src/Lua/Standard/DebugLibrary.cs b/src/Lua/Standard/DebugLibrary.cs index e2f5184a..fcbc3f8a 100644 --- a/src/Lua/Standard/DebugLibrary.cs +++ b/src/Lua/Standard/DebugLibrary.cs @@ -160,13 +160,13 @@ static LuaValue GetParam(LuaFunction function, int index) var state = GetLuaThread(context, out var argOffset); - var index = context.GetArgument(argOffset + 1); + var index = context.GetIntegerArgument(argOffset + 1); if (context.GetArgument(argOffset).TryReadFunction(out var f)) { return new(context.Return(GetParam(f, index - 1))); } - var level = context.GetArgument(argOffset); + var level = context.GetIntegerArgument(argOffset); if (level < 0 || level >= state.GetCallStackFrames().Length) { @@ -190,8 +190,8 @@ CancellationToken cancellationToken var state = GetLuaThread(context, out var argOffset); var value = context.GetArgument(argOffset + 2); - var index = context.GetArgument(argOffset + 1); - var level = context.GetArgument(argOffset); + var index = context.GetIntegerArgument(argOffset + 1); + var level = context.GetIntegerArgument(argOffset); if (level < 0 || level >= state.GetCallStackFrames().Length) { @@ -214,7 +214,7 @@ CancellationToken cancellationToken ) { var func = context.GetArgument(0); - var index = context.GetArgument(1) - 1; + var index = context.GetIntegerArgument(1) - 1; if (func is not LuaClosure closure) { if (func is CSharpClosure csClosure) @@ -250,7 +250,7 @@ CancellationToken cancellationToken ) { var func = context.GetArgument(0); - var index = context.GetArgument(1) - 1; + var index = context.GetIntegerArgument(1) - 1; var value = context.GetArgument(2); if (func is not LuaClosure closure) { @@ -371,7 +371,10 @@ CancellationToken cancellationToken var state = GetLuaThread(context, out var argOffset); var message = context.GetArgumentOrDefault(argOffset); - var level = context.GetArgumentOrDefault(argOffset + 1, argOffset == 0 ? 1 : 0); + var level = + context.HasArgument(argOffset + 1) ? context.GetIntegerArgument(argOffset + 1) + : argOffset == 0 ? 1 + : 0; if (message.Type is not (LuaValueType.Nil or LuaValueType.String or LuaValueType.Number)) { @@ -421,7 +424,7 @@ public ValueTask UpValueId( CancellationToken cancellationToken ) { - var n1 = context.GetArgument(1); + var n1 = context.GetIntegerArgument(1); var f1 = context.GetArgument(0); if (f1 is not LuaClosure closure) @@ -444,9 +447,9 @@ public ValueTask UpValueJoin( CancellationToken cancellationToken ) { - var n2 = context.GetArgument(3); + var n2 = context.GetIntegerArgument(3); var f2 = context.GetArgument(2); - var n1 = context.GetArgument(1); + var n1 = context.GetIntegerArgument(1); var f1 = context.GetArgument(0); if (f1 is not LuaClosure closure1 || f2 is not LuaClosure closure2) @@ -478,8 +481,12 @@ CancellationToken cancellationToken { var state = GetLuaThread(context, out var argOffset); var hook = context.GetArgumentOrDefault(argOffset); - var mask = context.GetArgumentOrDefault(argOffset + 1) ?? ""; - var count = context.GetArgumentOrDefault(argOffset + 2); + var mask = context.HasArgument(argOffset + 1) + ? context.GetStringArgument(argOffset + 1) + : ""; + var count = context.HasArgument(argOffset + 2) + ? context.GetIntegerArgument(argOffset + 2) + : 0; state.SetHookFromLua(hook, mask, count); if (hook is null) { @@ -562,7 +569,7 @@ CancellationToken cancellationToken } else if (arg1.TryReadNumber(out _)) { - var level = context.GetArgument(argOffset) + 1; + var level = context.GetIntegerArgument(argOffset) + 1; var callStack = state.GetCallStackFrames(); diff --git a/src/Lua/Standard/FileHandle.cs b/src/Lua/Standard/FileHandle.cs index d8886628..115d22dc 100644 --- a/src/Lua/Standard/FileHandle.cs +++ b/src/Lua/Standard/FileHandle.cs @@ -190,13 +190,15 @@ public async ValueTask Close(CancellationToken cancellationToken) (context, cancellationToken) => { var file = context.GetArgument(0); - var format = context.HasArgument(1) ? context.Arguments[1] : "*l"; + var upValues = new LuaValue[context.ArgumentCount]; + upValues[0] = new(file); + context.Arguments[1..].CopyTo(upValues.AsSpan(1)); return new( context.Return( new CSharpClosure( "iterator", - [new(file), format], + upValues, static async (context, cancellationToken) => { var upValues = context.GetCsClosure()!.UpValues.AsMemory(); @@ -278,7 +280,7 @@ static async (context, cancellationToken) => { var file = context.GetArgument(0); var mode = context.GetStringArgument(1); - var size = context.HasArgument(2) ? context.GetArgument(2) : -1; + var size = context.HasArgument(2) ? context.GetIntegerArgument(2) : -1; file.SetVBuf(mode, size); diff --git a/src/Lua/Standard/Internal/IOHelper.cs b/src/Lua/Standard/Internal/IOHelper.cs index 4f55a61f..ae07e615 100644 --- a/src/Lua/Standard/Internal/IOHelper.cs +++ b/src/Lua/Standard/Internal/IOHelper.cs @@ -176,8 +176,16 @@ CancellationToken cancellationToken break; } } - else if (format.TryRead(out var count)) + else if (format.Type == LuaValueType.Number) { + var length = Math.Truncate(format.UnsafeReadDouble()); + if (double.IsNaN(length) || length < 0 || length > int.MaxValue) + LuaRuntimeException.BadArgument( + state, + startArgumentIndex + i + 1, + "invalid count" + ); + var count = (int)length; var ret = await file.ReadStringAsync(count, cancellationToken); if (ret == null) { diff --git a/src/Lua/Standard/ModuleLibrary.cs b/src/Lua/Standard/ModuleLibrary.cs index dbf9f9cf..8ecf691f 100644 --- a/src/Lua/Standard/ModuleLibrary.cs +++ b/src/Lua/Standard/ModuleLibrary.cs @@ -1,4 +1,5 @@ using System.Runtime.CompilerServices; +using System.Text; using Lua.Internal.CompilerServices; namespace Lua.Standard; @@ -27,9 +28,11 @@ CancellationToken cancellationToken var arg0 = context.GetStringArgument(0); var loaded = context.GlobalState.LoadedModules; - if (!loaded.TryGetValue(arg0, out var loadedTable)) + var loadedTable = loaded[arg0]; + if (!loadedTable.ToBoolean()) { LuaFunction loader; + LuaValue data = LuaValue.Nil; var moduleLoader = context.GlobalState.ModuleLoader; var hostName = moduleLoader is null ? arg0 : context.State.Platform.DecodeText(arg0); if (moduleLoader != null && moduleLoader.Exists(hostName)) @@ -39,19 +42,31 @@ CancellationToken cancellationToken } else { - loader = await FindLoader(context.State, arg0, cancellationToken); + (loader, data) = await FindLoader(context.State, arg0, cancellationToken); } - await context.State.RunAsync(loader, 0, context.ReturnFrameBase, cancellationToken); - loadedTable = context.State.Stack.Get(context.ReturnFrameBase); - loaded[arg0] = loadedTable; + context.State.Stack.PushRange(arg0, data); + var count = await context.State.RunAsync( + loader, + 2, + context.ReturnFrameBase, + cancellationToken + ); + if ( + count > 0 + && context.State.Stack.Get(context.ReturnFrameBase).Type != LuaValueType.Nil + ) + loaded[arg0] = context.State.Stack.Get(context.ReturnFrameBase); + loadedTable = loaded[arg0]; + if (loadedTable.Type == LuaValueType.Nil) + loaded[arg0] = loadedTable = true; } return context.Return(loadedTable); } [AsyncMethodBuilder(typeof(LightAsyncValueTaskMethodBuilder<>))] - internal static async ValueTask FindFile( + internal static async ValueTask<(string? FileName, string? Error)> FindFile( LuaState state, string name, string pName, @@ -61,12 +76,18 @@ string dirSeparator var globalState = state.GlobalState; var package = globalState.Environment["package"]; var p = await state.GetTableAsync(package, pName); - if (!p.TryReadString(out var path)) + if (p.Type is not (LuaValueType.String or LuaValueType.Number)) { throw new LuaRuntimeException(state, $"package.{pName} must be a string"); } - return SearchPath(state, name, path, ".", dirSeparator); + return SearchPath( + state, + name, + p.ToLuaString(state.Platform.NumberFormat), + ".", + dirSeparator + ); } public ValueTask SearchPath( @@ -76,13 +97,17 @@ CancellationToken cancellationToken { var name = context.GetStringArgument(0); var path = context.GetStringArgument(1); - var separator = context.GetStringArgument(2); - var dirSeparator = context.GetStringArgument(3); - var fileName = SearchPath(context.State, name, path, separator, dirSeparator); - return new(context.Return(fileName ?? LuaValue.Nil)); + var separator = context.HasArgument(2) ? context.GetStringArgument(2) : "."; + var dirSeparator = context.HasArgument(3) + ? context.GetStringArgument(3) + : context.State.Platform.FileSystem.DirectorySeparator; + var (fileName, error) = SearchPath(context.State, name, path, separator, dirSeparator); + return new( + fileName != null ? context.Return(fileName) : context.Return(LuaValue.Nil, error!) + ); } - internal static string? SearchPath( + internal static (string? FileName, string? Error) SearchPath( LuaState state, string name, string path, @@ -96,39 +121,28 @@ string dirSeparator } var pathSpan = path.AsSpan(); - var nextIndex = pathSpan.IndexOf(';'); - if (nextIndex == -1) + StringBuilder? errors = null; + while (!pathSpan.IsEmpty) { - nextIndex = pathSpan.Length; - } - - do - { - path = pathSpan[..nextIndex].ToString(); - var fileName = path.Replace("?", name); - if (state.Platform.FileSystem.IsReadable(state.Platform.DecodeText(fileName))) - { - return fileName; - } - - if (pathSpan.Length <= nextIndex) - { - break; - } - - pathSpan = pathSpan[(nextIndex + 1)..]; - nextIndex = pathSpan.IndexOf(';'); - if (nextIndex == -1) - { + var nextIndex = pathSpan.IndexOf(';'); + if (nextIndex < 0) nextIndex = pathSpan.Length; + var template = pathSpan[..nextIndex]; + if (!template.IsEmpty) + { + var fileName = template.ToString().Replace("?", name); + if (state.Platform.FileSystem.IsReadable(state.Platform.DecodeText(fileName))) + return (fileName, null); + (errors ??= new()).Append("\n\tno file '").Append(fileName).Append('\''); } - } while (nextIndex != -1); + pathSpan = nextIndex == pathSpan.Length ? default : pathSpan[(nextIndex + 1)..]; + } - return null; + return (null, errors?.ToString() ?? ""); } [AsyncMethodBuilder(typeof(LightAsyncValueTaskMethodBuilder<>))] - internal static async ValueTask FindLoader( + internal static async ValueTask<(LuaFunction Function, LuaValue Data)> FindLoader( LuaState state, string name, CancellationToken cancellationToken @@ -136,13 +150,12 @@ CancellationToken cancellationToken { var package = state.GlobalState.Environment["package"].Read(); var searchers = package["searchers"].Read(); - for (var i = 0; i < searchers.GetArrayReadOnlySpan().Length; i++) + StringBuilder? errors = null; + for (var i = 1; ; i++) { - var searcher = searchers.GetArrayReadOnlySpan()[i]; + var searcher = searchers[i]; if (searcher.Type == LuaValueType.Nil) - { - continue; - } + break; var loader = searcher; var top = state.Stack.Count; @@ -154,15 +167,18 @@ CancellationToken cancellationToken var result = state.Stack.Get(top); if (result.Type == LuaValueType.Function) { + var data = resultCount > 1 ? state.Stack.Get(top + 1) : LuaValue.Nil; state.Stack.SetTop(top); - return result.Read(); + return (result.Read(), data); } + if (result.Type is LuaValueType.String or LuaValueType.Number) + (errors ??= new()).Append(result.ToLuaString(state.Platform.NumberFormat)); } state.Stack.SetTop(top); } - throw new LuaRuntimeException(state, $"Module '{name}' not found"); + throw new LuaRuntimeException(state, $"module '{name}' not found:{errors}"); } public ValueTask SearcherPreload( @@ -174,7 +190,7 @@ CancellationToken cancellationToken var preload = context.GlobalState.PreloadModules[name]; if (preload == LuaValue.Nil) { - return new(context.Return()); + return new(context.Return($"\n\tno field package.preload['{name}']")); } return new(context.Return(preload)); @@ -186,7 +202,7 @@ CancellationToken cancellationToken ) { var name = context.GetStringArgument(0); - var fileName = await FindFile( + var (fileName, error) = await FindFile( context.State, name, "path", @@ -194,7 +210,7 @@ CancellationToken cancellationToken ); if (fileName == null) { - return context.Return(LuaValue.Nil); + return context.Return(error!); } return context.Return( @@ -203,7 +219,8 @@ await context.State.LoadFileAsync( "bt", null, cancellationToken - ) + ), + fileName ); } } diff --git a/src/Lua/Standard/StringLibrary.cs b/src/Lua/Standard/StringLibrary.cs index 3d4aef1c..f85567b9 100644 --- a/src/Lua/Standard/StringLibrary.cs +++ b/src/Lua/Standard/StringLibrary.cs @@ -95,7 +95,12 @@ public ValueTask Dump( CancellationToken cancellationToken ) { - throw new NotSupportedException("stirng.dump is not supported"); + var function = context.GetArgument(0); + if (function is not LuaClosure closure) + throw new LuaRuntimeException(context.State, "unable to dump given function"); + using var writer = new ArrayPoolBufferWriter(); + closure.Proto.WriteBytecode(writer); + return new(context.Return(LuaByteString.FromBytes(writer.WrittenSpan))); } public ValueTask Find( diff --git a/tests/Lua.Tests/ByteStringIntegrationTests.cs b/tests/Lua.Tests/ByteStringIntegrationTests.cs index 428d8b28..91e4c471 100644 --- a/tests/Lua.Tests/ByteStringIntegrationTests.cs +++ b/tests/Lua.Tests/ByteStringIntegrationTests.cs @@ -48,7 +48,7 @@ public void ExistingConstructorSignaturesRemainAvailable() [TestCase(LuaStringMode.Utf16)] [TestCase(LuaStringMode.Bytes)] - public async Task XPCallHostErrorsPreserveDefaultBehaviorAndEncodeByteMode( + public async Task XPCallHostErrorsPreserveTheirMessageInBothStringModes( LuaStringMode stringMode ) { @@ -65,7 +65,9 @@ LuaStringMode stringMode Assert.That( result[1], Is.EqualTo( - stringMode == LuaStringMode.Bytes ? LuaByteString.EncodeUtf8(message) : LuaValue.Nil + new LuaValue( + stringMode == LuaStringMode.Bytes ? LuaByteString.EncodeUtf8(message) : message + ) ) ); } diff --git a/tests/Lua.Tests/LoadReaderTests.cs b/tests/Lua.Tests/LoadReaderTests.cs new file mode 100644 index 00000000..d28d1bea --- /dev/null +++ b/tests/Lua.Tests/LoadReaderTests.cs @@ -0,0 +1,157 @@ +using Lua.Platforms; +using Lua.Runtime; +using Lua.Standard; + +namespace Lua.Tests; + +[TestFixture(LuaStringMode.Utf16)] +[TestFixture(LuaStringMode.Bytes)] +public class LoadReaderTests(LuaStringMode stringMode) +{ + LuaState CreateState() + { + var state = LuaState.Create( + stringMode == LuaStringMode.Bytes ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default + ); + state.OpenStandardLibraries(); + return state; + } + + [Test] + public async Task ReaderLoadsPiecesWithAnEnvironmentAndChunkName() + { + using var state = CreateState(); + await state.DoStringAsync( + """ + local pieces={'return value+',42} + local index=0 + local f=assert(load(function() + index=index+1 + return pieces[index],'ignored' + end,'reader-name','t',{value=1})) + assert(index==3 and f()==43) + assert(debug.getinfo(f).source=='reader-name') + """ + ); + } + + [TestCase("nil")] + [TestCase("''")] + public async Task ReaderStopsAtTheFirstEmptyPiece(string ending) + { + using var state = CreateState(); + await state.DoStringAsync( + "local calls=0;local f=assert(load(function() calls=calls+1;return " + + ending + + " end));assert(calls==1 and select('#',f())==0)" + ); + } + + [Test] + public async Task ReaderErrorsAreReturnedAndDoNotLeakFrames() + { + using var state = CreateState(); + await state.DoStringAsync( + """ + local payload={} + local f,e=load(function() error(payload) end) + assert(f==nil and e==payload) + f,e=load(function() return true end) + assert(f==nil and e:find('reader function must return a string',1,true)) + assert(not pcall(load,{})) + """ + ); + Assert.That(state.Stack.Count, Is.Zero); + Assert.That(state.CallStackFrameCount, Is.Zero); + } + + [Test] + public async Task ReaderCanSuspendAndCancellationIsNotConvertedToALoadError() + { + using var state = CreateState(); + var suspension = new ManualSuspension(); + var calls = 0; + state.Environment["reader"] = new LuaFunction( + async (context, _) => + { + if (calls++ > 0) + return context.Return(); + await suspension; + return context.Return("return 42"); + } + ); + var operation = state.DoStringAsync("return assert(load(reader))()").AsTask(); + suspension.Resume(); + Assert.That(await operation, Is.EqualTo(new LuaValue[] { 42 })); + using var source = new CancellationTokenSource(); + state.Environment["reader"] = new LuaFunction( + (context, token) => + { + source.Cancel(); + token.ThrowIfCancellationRequested(); + return new(context.Return()); + } + ); + Assert.ThrowsAsync(async () => + await state.DoStringAsync("return load(reader)", cancellationToken: source.Token) + ); + } + + [TestCase(false)] + [TestCase(true)] + public async Task DumpedFunctionsLoadDirectlyAndFromOneCharacterPieces(bool reader) + { + using var state = CreateState(); + state.Environment["readerMode"] = reader; + await state.DoStringAsync( + """ + local captured=99 + local function original(x,...) + local function nested(y) return x+y end + return nested(2),captured,'ế\0\255',... + end + local binary=string.dump(original) + local source=binary + if readerMode then + local i=0 + source=function() i=i+1;return binary:sub(i,i) end + end + local loaded=assert(load(source,'dump-name','b',41)) + local a,b,text,c=loaded(40,43) + assert(a==42 and b==41 and text=='ế\0\255' and c==43) + assert(captured==99) + local ok,message=pcall(string.dump,print) + assert(not ok and message:find('unable to dump',1,true)) + local f,message=load(binary,nil,'t') + assert(not f and message:find('attempt to load a binary chunk',1,true)) + f,message=load('return 42',nil,'b') + assert(not f and message:find('attempt to load a text chunk',1,true)) + """ + ); + } + + [Test] + public async Task NativeLua52BytecodeUsesTheSameFunctionHeader() + { + // Lua 5.2.4: string.dump(function(a, ...) return a + 2, ... end), 64-bit size_t. + var bytes = Convert.FromBase64String( + "G0x1YVIAAQQIBAgAGZMNChoKAQAAAAEAAAABAQMEAAAATQBAAKYAAABfAAAAHwCAAAEAAAADAAAAAAAAAEAAAAAAAAAAAAgAAAAAAAAAQG5hdGl2ZQAEAAAAAQAAAAEAAAABAAAAAQAAAAEAAAACAAAAAAAAAGEAAAAAAAQAAAAAAAAA" + ); + using var state = CreateState(); + var loaded = new LuaClosure(state, Prototype.FromBytecode(bytes, "native", stringMode)); + state.Stack.Push(40); + state.Stack.Push(43); + var count = await state.RunAsync(loaded, 2); + Assert.That(count, Is.EqualTo(2)); + Assert.That(state.Stack.Pop(), Is.EqualTo(new LuaValue(43))); + Assert.That(state.Stack.Pop(), Is.EqualTo(new LuaValue(42))); + + var dump = ( + await state.DoStringAsync("return string.dump(function(a,...) return a+2,... end)") + )[0] + .Read(); + Assert.That(dump[26], Is.EqualTo((char)1)); // parameter count + Assert.That(dump[27], Is.EqualTo((char)1)); // varargs + Assert.That(dump[28], Is.GreaterThanOrEqualTo((char)2)); // register window + } +} diff --git a/tests/Lua.Tests/MemoryLifetimeTests.cs b/tests/Lua.Tests/MemoryLifetimeTests.cs index 086a23d3..7b3fc563 100644 --- a/tests/Lua.Tests/MemoryLifetimeTests.cs +++ b/tests/Lua.Tests/MemoryLifetimeTests.cs @@ -74,13 +74,16 @@ public void ClearedExceptionTraceDoesNotRootDiscardedFrames() GC.KeepAlive(state); } - [TestCase(false)] - [TestCase(true)] - public void HandledErrorDoesNotRootPayloadWhileStateIsIdle(bool text) + [Test] + [Combinatorial] + public void HandledErrorDoesNotRootPayloadWhileStateIsIdle( + [Values(false, true)] bool text, + [Values(false, true)] bool messageHandler + ) { using var state = LuaState.Create(); state.OpenStandardLibraries(); - AssertCollected(HandleError(state, text)); + AssertCollected(HandleError(state, text, messageHandler)); GC.KeepAlive(state); } @@ -202,13 +205,15 @@ static WeakReference RunTailCall(LuaState state, bool asynchronous) } [MethodImpl(MethodImplOptions.NoInlining)] - static WeakReference HandleError(LuaState state, bool text = false) + static WeakReference HandleError(LuaState state, bool text = false, bool messageHandler = false) { object payload = text ? new string('x', 65536) : new object(); state.Environment["payload"] = LuaValue.FromObject(payload); state .DoStringAsync( - "local p=payload; payload=nil; local ok=pcall(function() error(p) end); return ok" + messageHandler + ? "local p=payload;payload=nil;local ok=xpcall(function() error(p) end,function(e) return e,p end);return ok" + : "local p=payload;payload=nil;local ok=pcall(function() error(p) end);return ok" ) .AsTask() .GetAwaiter() diff --git a/tests/Lua.Tests/ModuleContractTests.cs b/tests/Lua.Tests/ModuleContractTests.cs new file mode 100644 index 00000000..64b9880b --- /dev/null +++ b/tests/Lua.Tests/ModuleContractTests.cs @@ -0,0 +1,110 @@ +using Lua.IO; +using Lua.Platforms; +using Lua.Standard; +using Lua.Tests.Helpers; + +namespace Lua.Tests; + +[TestFixture(LuaStringMode.Utf16)] +[TestFixture(LuaStringMode.Bytes)] +public class ModuleContractTests(LuaStringMode stringMode) +{ + [TestCase("return", "true", 1)] + [TestCase("return nil", "true", 1)] + [TestCase("return false", "false", 2)] + [TestCase("package.loaded.test=42", "42", 1)] + [TestCase("package.loaded.test=false", "false", 2)] + [TestCase("package.loaded.test=42;return 43", "43", 1)] + public async Task RequireUsesTheLoaderResultAndPackageLoaded( + string loader, + string expected, + int expectedCalls + ) + { + using var state = CreateState(); + var result = await state.DoStringAsync( + $$""" + local calls=0 + package.preload.test=function() calls=calls+1;{{loader}} end + local first=require('test') + local second=require('test') + return first=={{expected}},second=={{expected}},package.loaded.test=={{expected}},calls + """ + ); + Assert.That(result, Is.EqualTo(new LuaValue[] { true, true, true, expectedCalls })); + } + + [Test] + public async Task RequirePassesSearcherDataAndStopsAtTheFirstNilSearcher() + { + using var state = CreateState(); + await state.DoStringAsync( + """ + local data={} + local loader=function(name,searchData) + assert(name=='test' and searchData==data) + return 42,99 + end + package.searchers={function(name) return loader,data end} + assert(require('test')==42) + package.searchers={[2]=function() error('must not run') end} + local ok,e=pcall(require,'missing') + assert(not ok and e:find("module 'missing' not found:",1,true)) + """ + ); + } + + [Test] + public async Task FileLoaderReceivesItsNameAndPath() + { + using var state = CreateState(); + state.Platform = state.Platform with { FileSystem = new ModuleFileSystem() }; + var result = await state.DoStringAsync("return require('test')"); + Assert.That(result, Is.EqualTo(new LuaValue[] { true })); + } + + [Test] + public async Task SearchPathUsesDefaultsAndReturnsFailureDetails() + { + using var state = CreateState(); + state.Platform = state.Platform with { FileSystem = new ModuleFileSystem() }; + await state.DoStringAsync( + """ + assert(package.searchpath('test',';;?.lua;;')=='test.lua') + assert(package.searchpath('test','?.lua',nil,nil)=='test.lua') + local path,message=package.searchpath('missing',';;?.lua;?/init.lua;;') + assert(path==nil and message=="\n\tno file 'missing.lua'\n\tno file 'missing/init.lua'") + local ok,e=pcall(require,'missing') + assert(not ok and e:find("no field package.preload['missing']",1,true)) + assert(e:find("no file 'missing.lua'",1,true)) + """ + ); + } + + LuaState CreateState() + { + var state = LuaState.Create( + stringMode == LuaStringMode.Bytes ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default + ); + state.OpenStandardLibraries(); + return state; + } + + sealed class ModuleFileSystem : NotImplementedExceptionFileSystemBase + { + public override string DirectorySeparator => "/"; + + public override bool IsReadable(string path) => path == "test.lua"; + + public override ValueTask Open( + string path, + LuaFileOpenMode mode, + CancellationToken cancellationToken = default + ) => + new( + ILuaStream.CreateFromString( + "local name,path=...;assert(name=='test' and path=='test.lua');return true" + ) + ); + } +} diff --git a/tests/Lua.Tests/NumberReadTests.cs b/tests/Lua.Tests/NumberReadTests.cs new file mode 100644 index 00000000..3c658564 --- /dev/null +++ b/tests/Lua.Tests/NumberReadTests.cs @@ -0,0 +1,48 @@ +using System.Text; +using Lua.IO; + +namespace Lua.Tests; + +[TestFixture("string")] +[TestFixture("memory")] +[TestFixture("utf8")] +[TestFixture("bytes")] +public class NumberReadTests(string backend) +{ + [TestCase("1e2e3x", 100, "e3x")] + [TestCase("1e2.3", 100, ".3")] + [TestCase("-0x10z", -16, "z")] + [TestCase("+0X1.Ap+2z", 6.5, "z")] + [TestCase("0x1p2p3z", 4, "p3z")] + [TestCase("0x1p2az", 4, "az")] + [TestCase("0x1p2+3", 4, "+3")] + [TestCase(" .5e-1x", 0.05, "x")] + public async Task NumericReadsPreserveTheFirstNonNumericCharacter( + string input, + double expected, + string remaining + ) + { + using var stream = CreateStream(input); + Assert.That(await stream.ReadNumberAsync(default), Is.EqualTo(expected)); + Assert.That(await stream.ReadAllAsync(default), Is.EqualTo(remaining)); + } + + ILuaStream CreateStream(string text) => + backend switch + { + "string" => ILuaStream.CreateFromString(text), + "memory" => ILuaStream.CreateFromMemory(text.AsMemory()), + _ => ILuaStream.CreateFromStream( + new ShortReadStream(Encoding.ASCII.GetBytes(text)), + LuaFileOpenMode.Read, + backend == "bytes" ? LuaStringMode.Bytes : LuaStringMode.Utf16 + ), + }; + + sealed class ShortReadStream(byte[] contents) : MemoryStream(contents) + { + public override int Read(byte[] buffer, int offset, int count) => + base.Read(buffer, offset, Math.Min(count, 1)); + } +} diff --git a/tests/Lua.Tests/ProtectedCallTests.cs b/tests/Lua.Tests/ProtectedCallTests.cs new file mode 100644 index 00000000..9769e813 --- /dev/null +++ b/tests/Lua.Tests/ProtectedCallTests.cs @@ -0,0 +1,231 @@ +using Lua.Platforms; +using Lua.Standard; + +namespace Lua.Tests; + +[TestFixture(LuaStringMode.Utf16)] +[TestFixture(LuaStringMode.Bytes)] +public class ProtectedCallTests(LuaStringMode stringMode) +{ + LuaState CreateState() + { + var state = LuaState.Create( + stringMode == LuaStringMode.Bytes ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default + ); + state.OpenStandardLibraries(); + return state; + } + + [TestCase("return 'handled',99", "handled")] + [TestCase("return", null)] + [TestCase("error('handler failed',0)", "error in error handling")] + public async Task XpcallReturnsExactlyOneHandlerResult(string handler, string? expected) + { + using var state = CreateState(); + var result = await state.DoStringAsync( + "return xpcall(function() error('fail',0) end,function(e) " + handler + " end)" + ); + Assert.That( + result, + Is.EqualTo(new LuaValue[] { false, expected == null ? LuaValue.Nil : expected }) + ); + Assert.That(state.Stack.Count, Is.Zero); + Assert.That(state.CallStackFrameCount, Is.Zero); + Assert.That(state.ErrorHandlerIndex, Is.Zero); + } + + [Test] + public async Task HandlerIsOnlyValidatedWhenAnErrorOccurs() + { + using var state = CreateState(); + await state.DoStringAsync( + """ + local ok,a,b=xpcall(function(a,b) return a,b end,nil,12,34) + assert(ok and a==12 and b==34) + local ok,e=xpcall(function() error('failure') end,nil) + assert(not ok and e=='error in error handling') + local handler=setmetatable({}, {__call=function(self,e) return e end}) + local ok,e=xpcall(function() error(42) end,handler) + assert(not ok and e==42) + """ + ); + } + + [TestCase(0, false)] + [TestCase(0, true)] + [TestCase(2, false)] + [TestCase(2, true)] + [TestCase(8, false)] + [TestCase(8, true)] + public async Task XpcallSupportsOverlappingAndForwardReturnBuffers(int returnBase, bool failure) + { + using var state = CreateState(); + state.Stack.Push( + new LuaFunction( + (context, _) => + { + if (failure) + throw new LuaRuntimeException(context.State, "failure"); + return new(context.Return(context.GetArgument(0), 43)); + } + ) + ); + state.Stack.Push(new LuaFunction((context, _) => new(context.Return("handled")))); + state.Stack.Push(42); + var function = state.Environment["xpcall"].Read(); + var count = await state.RunAsync(function, 3, returnBase); + Assert.That(count, Is.EqualTo(failure ? 2 : 3)); + Assert.That( + state.Stack.GetBuffer().Slice(returnBase, count).ToArray(), + Is.EqualTo( + failure ? new LuaValue[] { false, "handled" } : new LuaValue[] { true, 42, 43 } + ) + ); + } + + [TestCase(false)] + [TestCase(true)] + public async Task HandlerCanInspectFailingFramesAndLocals(bool asynchronous) + { + using var state = CreateState(); + var suspension = new ManualSuspension(); + state.Environment["pause"] = new LuaFunction( + async (context, _) => + { + if (asynchronous) + await suspension; + return context.Return(); + } + ); + var operation = state + .DoStringAsync( + """ + local escaped + local function namedFailure() + local value=41 + escaped=function() return value end + pause() + error('fail') + end + local ok,message=xpcall(function() namedFailure() end,function(e) + local trace=debug.traceback(e) + assert(trace:find('namedFailure',1,true)) + for level=1,10 do + local info=debug.getinfo(level,'f') + if info and info.func==namedFailure then + for index=1,10 do + local name,value=debug.getlocal(level,index) + if name=='value' then + assert(value==41) + debug.setlocal(level,index,42) + return 'handled' + end + end + end + end + error('missing failing local') + end) + return ok,message,escaped() + """, + "@protected.lua" + ) + .AsTask(); + if (asynchronous) + suspension.Resume(); + Assert.That(await operation, Is.EqualTo(new LuaValue[] { false, "handled", 42 })); + Assert.That(state.CallStackFrameCount, Is.Zero); + Assert.That(state.OpenUpValues.Length, Is.Zero); + } + + [Test] + public async Task NestedProtectedCallsRestoreTheOuterHandler() + { + using var state = CreateState(); + await state.DoStringAsync( + """ + local hits=0 + local ok,e=xpcall(function() + assert(not pcall(error,'caught')) + local f,message=load(function() error('reader failed') end) + assert(not f and message:find('reader failed',1,true)) + local ok,value=xpcall(function() error('inner',0) end,function(e) return e end) + assert(not ok and value=='inner') + error('outer',0) + end,function(e) hits=hits+1;return e end) + assert(not ok and e=='outer' and hits==1) + """ + ); + Assert.That(state.ErrorHandlerIndex, Is.Zero); + } + + [Test] + public async Task HostErrorHandlerCanSuspendWithoutLosingTheErrorOrFrames() + { + using var state = CreateState(); + var suspension = new ManualSuspension(); + const string message = "host failure ế"; + state.Environment["fail"] = new LuaFunction((_, _) => throw new IOException(message)); + state.Environment["handler"] = new LuaFunction( + async (context, _) => + { + var error = context.GetArgument(0); + Assert.That(context.State.CallStackFrameCount, Is.GreaterThan(2)); + await suspension; + return context.Return(error, 99); + } + ); + var operation = state.DoStringAsync("return xpcall(fail,handler)").AsTask(); + suspension.Resume(); + Assert.That( + await operation, + Is.EqualTo(new LuaValue[] { false, state.Platform.EncodeText(message) }) + ); + Assert.That(state.ErrorHandlerIndex, Is.Zero); + Assert.That(state.Stack.Count, Is.Zero); + Assert.That(state.CallStackFrameCount, Is.Zero); + } + + [Test] + public async Task ErrorHandlerCannotYield() + { + using var state = CreateState(); + var result = await state.DoStringAsync( + """ + local co=coroutine.create(function() + return xpcall(function() error('fail') end,coroutine.yield) + end) + local ok,status,message=coroutine.resume(co) + return ok,status,message,coroutine.status(co) + """ + ); + Assert.That( + result, + Is.EqualTo(new LuaValue[] { true, false, "error in error handling", "dead" }) + ); + } + + [TestCase(false)] + [TestCase(true)] + public async Task CancellationStillEscapesXpcall(bool cancelInHandler) + { + using var state = CreateState(); + using var source = new CancellationTokenSource(); + state.Environment["cancel"] = new LuaFunction( + (context, _) => + { + source.Cancel(); + return new(context.Return()); + } + ); + Assert.ThrowsAsync(async () => + await state.DoStringAsync( + cancelInHandler + ? "return xpcall(function() error('fail') end,cancel)" + : "return xpcall(cancel,function() error('handler must not run') end)", + cancellationToken: source.Token + ) + ); + Assert.That(state.ErrorHandlerIndex, Is.Zero); + Assert.That((await state.DoStringAsync("return 42"))[0], Is.EqualTo(new LuaValue(42))); + } +} diff --git a/tests/Lua.Tests/StandardLibraryContractTests.cs b/tests/Lua.Tests/StandardLibraryContractTests.cs new file mode 100644 index 00000000..fc4875fc --- /dev/null +++ b/tests/Lua.Tests/StandardLibraryContractTests.cs @@ -0,0 +1,124 @@ +using Lua.IO; +using Lua.Platforms; +using Lua.Standard; +using Lua.Tests.Helpers; + +namespace Lua.Tests; + +[TestFixture(LuaStringMode.Utf16)] +[TestFixture(LuaStringMode.Bytes)] +public class StandardLibraryContractTests(LuaStringMode stringMode) +{ + LuaState CreateState() + { + var state = LuaState.Create( + stringMode == LuaStringMode.Bytes ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default + ); + state.OpenStandardLibraries(); + return state; + } + + [Test] + public async Task RawSetReturnsTheOriginalTable() + { + using var state = CreateState(); + await state.DoStringAsync( + """ + local t=setmetatable({}, {__newindex=function() error('must not run') end}) + assert(rawset(t,'x',7)==t and t.x==7) + assert(select('#',rawset(t,'x',nil))==1 and t.x==nil) + """ + ); + } + + [Test] + public async Task AssertCoercesNumericMessagesButRejectsTables() + { + using var state = CreateState(); + var result = await state.DoStringAsync("return pcall(assert,false,42)"); + Assert.That(result, Is.EqualTo(new LuaValue[] { false, "42" })); + await state.DoStringAsync( + """ + local ok,e=pcall(assert,false,{}) + assert(not ok and e:find('string expected',1,true)) + assert(select('#',assert(true,42,nil))==3) + """ + ); + } + + [Test] + public async Task PrintUsesOneSnapshotOfGlobalTostring() + { + using var state = CreateState(); + var output = new List(); + var io = new TestStandardIO(stringMode) + { + Output = new BufferedOutputStream(text => output.Add(text.ToString())), + }; + state.Platform = state.Platform with { StandardIO = io }; + await state.DoStringAsync( + """ + local hits=0 + tostring=function(value) + hits=hits+1 + _G.tostring=function() error('replacement must not run') end + return value*2,'extra' + end + print(1,2) + assert(hits==2) + """ + ); + Assert.That(output, Is.EqualTo(new[] { "2\t4\n" })); + } + + [TestCase("nil")] + [TestCase("{}")] + public async Task PrintRejectsInvalidTostringResults(string result) + { + using var state = CreateState(); + await state.DoStringAsync( + "tostring=function() return " + + result + + " end;local ok,e=pcall(print,1);assert(not ok and e:find('must return a string',1,true))" + ); + } + + [Test] + public async Task FileLinesRetainsEveryReadFormat() + { + using var state = CreateState(); + using var stream = ILuaStream.CreateFromString("12 abc\n34 def\n"); + state.Environment["file"] = new LuaValue(new FileHandle(stream)); + var result = await state.DoStringAsync( + "local it=file:lines('*n','*l');local a,b=it();local c,d=it();return a,b,c,d,it(),io.type(file)" + ); + Assert.That( + result, + Is.EqualTo(new LuaValue[] { 12, " abc", 34, " def", LuaValue.Nil, "file" }) + ); + } + + [Test] + public async Task LuaIntegerArgumentsTruncateWithoutChangingTypedCSharpArguments() + { + using var state = CreateState(); + using var stream = ILuaStream.CreateFromString("abc"); + state.Environment["file"] = new LuaValue(new FileHandle(stream)); + state.Environment["typed"] = new LuaFunction( + (context, _) => new(context.Return(context.GetArgument(0))) + ); + await state.DoStringAsync( + """ + assert(file:read(1.9)=='a') + local value=42 + local function f() return value end + local name,upvalue=debug.getupvalue(f,1.9) + assert(name=='value' and upvalue==42) + assert(debug.setupvalue(f,1.9,43)=='value' and f()==43) + local ok,e=pcall(function() error('fail',0.9) end) + assert(not ok and e=='fail') + assert(not pcall(typed,1.9)) + """ + ); + } +} From e7077f263f38d0e03886541f309594ace65c42ce Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:10:42 +0900 Subject: [PATCH 02/16] test: restore Lua function and binary file coverage Enable calls.lua and exercise load readers in files.lua directly. Run the binary dump and file-loading sections in byte mode, retaining UTF-16 text I/O adaptations. Replace hard-coded line patches with source section markers. --- tests/Lua.Tests/LuaTests.cs | 49 ++++++------------------------------- 1 file changed, 8 insertions(+), 41 deletions(-) diff --git a/tests/Lua.Tests/LuaTests.cs b/tests/Lua.Tests/LuaTests.cs index 08483a8c..703860f5 100644 --- a/tests/Lua.Tests/LuaTests.cs +++ b/tests/Lua.Tests/LuaTests.cs @@ -12,49 +12,16 @@ public class LuaTests(LuaStringMode stringMode) { static string PatchFilesLuaSource(string source, LuaStringMode stringMode) { - const char utf8Bom = '\uFEFF'; - var lines = source.Replace("\r\n", "\n").Split('\n'); - lines[246] = - "do local __f = assert(io.open(file)); local __src = assert(__f:read('*a')); __f:close(); assert(load(__src, nil, nil, t))() end -- Lua-CSharp test harness uses string load because load(reader) is unsupported."; - lines[292] = - "do local __f = assert(io.open(file)); local __src = assert(__f:read('*a')); __f:close(); assert(load(__src))() end -- Lua-CSharp test harness uses string load because load(reader) is unsupported."; - lines[294] = - "do local __f = assert(io.open(file)); local __src = assert(__f:read('*a')); __f:close(); assert(load(__src))() end -- Lua-CSharp test harness uses string load because load(reader) is unsupported."; - lines[296] = - "do local __f = assert(io.open(file)); local __src = assert(__f:read('*a')); __f:close(); assert(load(__src))() end -- Lua-CSharp test harness uses string load because load(reader) is unsupported."; + source = source.Replace("\r\n", "\n"); if (stringMode != LuaStringMode.Bytes) { - lines[330] = - $"testloadfile(\"{utf8Bom}# some comment\\nreturn 234\", 234) -- Lua-CSharp test harness uses a real BOM code point because Lua strings are UTF-16 text, not raw bytes."; - lines[331] = - $"testloadfile(\"{utf8Bom}return 239\", 239) -- Lua-CSharp test harness uses a real BOM code point because Lua strings are UTF-16 text, not raw bytes."; - lines[332] = - $"testloadfile(\"{utf8Bom}\", nil) -- Lua-CSharp test harness uses a real BOM code point because Lua strings are UTF-16 text, not raw bytes."; + source = source.Replace("\\xEF\\xBB\\xBF", "\uFEFF"); + // UTF-16 platforms use text I/O. Raw binary file coverage runs in byte mode. + var start = source.IndexOf("-- loading binary file\n", StringComparison.Ordinal); + var end = source.IndexOf("-- 'loadfile' with 'env'", start, StringComparison.Ordinal); + source = source.Remove(start, end - start); } - for (var i = 341; i <= 346; i++) - { - lines[i] = - "-- Lua-CSharp test harness skips binary string.dump/loadfile coverage because string.dump is unsupported."; - } - - lines[348] = - "do -- Lua-CSharp test harness skips binary string.dump/loadfile coverage because string.dump is unsupported."; - for (var i = 349; i <= 355; i++) - { - lines[i] = - " -- Lua-CSharp test harness skips binary string.dump/loadfile coverage because string.dump is unsupported."; - } - - lines[356] = "end"; - for (var i = 359; i <= 365; i++) - { - lines[i] = - "-- Lua-CSharp test harness skips binary string.dump/loadfile coverage because string.dump is unsupported."; - } - - lines[503] = - "_noposix = true; if not _noposix then -- Lua-CSharp test harness skips io.popen/os.execute because process command execution is unsupported."; - return string.Join("\n", lines); + return source.Replace("if not _noposix then", "_noposix = true; if not _noposix then"); } [Test] @@ -66,7 +33,7 @@ static string PatchFilesLuaSource(string source, LuaStringMode stringMode) [TestCase("tests-lua/literals.lua")] [TestCase("tests-lua/pm.lua")] [TestCase("tests-lua/sort.lua")] - //[TestCase("tests-lua/calls.lua")] // string.dump and reader function for load chunk is not implemented + [TestCase("tests-lua/calls.lua")] [TestCase("tests-lua/files.lua")] [TestCase("tests-lua/closure.lua")] [TestCase("tests-lua/errors.lua")] From fe0b2663192c6a8acbf4bf7d7febc46af44c6dd4 Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:10:42 +0900 Subject: [PATCH 03/16] test: make coroutine entry lifetime assertions deterministic Drive suspension manually and return from a non-inlined execution helper before collecting. Check release once while keeping the completed coroutine and Lua state alive, without scheduler yields or GC polling. --- .../Lua.Tests/CoroutineEntryLifetimeTests.cs | 109 ++++++++++-------- 1 file changed, 62 insertions(+), 47 deletions(-) diff --git a/tests/Lua.Tests/CoroutineEntryLifetimeTests.cs b/tests/Lua.Tests/CoroutineEntryLifetimeTests.cs index a6028d78..4a966d0a 100644 --- a/tests/Lua.Tests/CoroutineEntryLifetimeTests.cs +++ b/tests/Lua.Tests/CoroutineEntryLifetimeTests.cs @@ -11,7 +11,7 @@ public class CoroutineEntryLifetimeTests(LuaStringMode mode) { [Test] [Combinatorial] - public async Task CompletedCoroutineReleasesEntryCaptures( + public void CompletedCoroutineReleasesEntryCaptures( [Values(false, true)] bool wrap, [Values(false, true)] bool asynchronous, [Values(false, true)] bool yieldFirst @@ -21,47 +21,16 @@ public async Task CompletedCoroutineReleasesEntryCaptures( mode == LuaStringMode.Bytes ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default ); state.OpenStandardLibraries(); - var reference = InstallPayload(state); - state.Environment["wrap"] = wrap; - state.Environment["yieldFirst"] = yieldFirst; - state.Environment["pause"] = new LuaFunction( - async (context, _) => - { - if (asynchronous) - await Task.Yield(); - return context.Return(); - } - ); - var result = await state.DoStringAsync( - """ - local function make(value) - return function() - thread=coroutine.running() - pause() - if yieldFirst then coroutine.yield(7) end - return value and 42 - end - end - local entry=make(payload) - payload=nil - if wrap then - run=coroutine.wrap(entry) - else - local co=coroutine.create(entry) - run=function() return coroutine.resume(co) end - end - entry=nil - if yieldFirst then run() end - return run() - """ - ); - await Task.Yield(); - Assert.That(result, Is.EqualTo(wrap ? new LuaValue[] { 42 } : new LuaValue[] { true, 42 })); + var reference = RunToCompletion(state, wrap, asynchronous, yieldFirst); + Assert.That(IsAliveAfterCollection(reference), Is.False); var thread = state.Environment["thread"].Read(); Assert.That(thread.GetStatus(), Is.EqualTo(LuaThreadStatus.Dead)); - Assert.That(() => IsAliveAfterCollection(reference), Is.False.After(1000, 10)); - var again = await state.DoStringAsync(wrap ? "return pcall(run)" : "return run()"); + var again = state + .DoStringAsync(wrap ? "return pcall(run)" : "return run()") + .AsTask() + .GetAwaiter() + .GetResult(); Assert.That(again[0], Is.EqualTo(new LuaValue(false))); Assert.That(again[1].Read(), Does.Contain("cannot resume dead coroutine")); Assert.That(thread.GetStackValues().Length, Is.Zero); @@ -72,6 +41,60 @@ local function make(value) GC.KeepAlive(state); } + [MethodImpl(MethodImplOptions.NoInlining)] + static WeakReference RunToCompletion( + LuaState state, + bool wrap, + bool asynchronous, + bool yieldFirst + ) + { + var payload = new object(); + state.Environment["payload"] = LuaValue.FromObject(payload); + var reference = new WeakReference(payload); + var suspension = new ManualSuspension(); + state.Environment["wrap"] = wrap; + state.Environment["yieldFirst"] = yieldFirst; + state.Environment["pause"] = new LuaFunction( + async (context, _) => + { + if (asynchronous) + await suspension; + return context.Return(); + } + ); + var operation = state + .DoStringAsync( + """ + local function make(value) + return function() + thread=coroutine.running() + pause() + if yieldFirst then coroutine.yield(7) end + return value and 42 + end + end + local entry=make(payload) + payload=nil + if wrap then + run=coroutine.wrap(entry) + else + local co=coroutine.create(entry) + run=function() return coroutine.resume(co) end + end + entry=nil + if yieldFirst then run() end + return run() + """ + ) + .AsTask(); + if (asynchronous) + suspension.Resume(); + var result = operation.GetAwaiter().GetResult(); + Assert.That(result, Is.EqualTo(wrap ? new LuaValue[] { 42 } : new LuaValue[] { true, 42 })); + return reference; + } + static bool IsAliveAfterCollection(WeakReference reference) { GC.Collect(); @@ -79,12 +102,4 @@ static bool IsAliveAfterCollection(WeakReference reference) GC.Collect(); return reference.IsAlive; } - - [MethodImpl(MethodImplOptions.NoInlining)] - static WeakReference InstallPayload(LuaState state) - { - var payload = new object(); - state.Environment["payload"] = LuaValue.FromObject(payload); - return new WeakReference(payload); - } } From c3356a08bfe034a974efe4c1b322296290c6b3f6 Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:32:45 +0900 Subject: [PATCH 04/16] fix: encode Lua 5.2 bytecode constants correctly Serialize Lua's constant tags independently of LuaValueType, and include the terminating byte in every string length, including empty strings. This lets native Lua load string constants without interpreting them as null strings. Add wire-format regressions and live NuLua Lua 5.2 interoperability checks to the CI verification command. Exchange dumps in both directions across the interpreter/compiler workloads, captured closures, varargs and binary strings. --- .../Benchmark/BytecodeInteropVerification.cs | 120 ++++++++++++++++++ sandbox/Benchmark/Program.cs | 1 + src/Lua/CodeAnalysis/Compilation/Dump.cs | 34 ++--- tests/Lua.Tests/BytecodeValidationTests.cs | 40 ++++++ 4 files changed, 179 insertions(+), 16 deletions(-) create mode 100644 sandbox/Benchmark/BytecodeInteropVerification.cs diff --git a/sandbox/Benchmark/BytecodeInteropVerification.cs b/sandbox/Benchmark/BytecodeInteropVerification.cs new file mode 100644 index 00000000..24186244 --- /dev/null +++ b/sandbox/Benchmark/BytecodeInteropVerification.cs @@ -0,0 +1,120 @@ +using Lua; +using Lua.Platforms; +using Lua.Standard; +using NuLua; +using NuLua.Lua52; +using LuaFunction = Lua.LuaFunction; +using LuaTable = Lua.LuaTable; + +static class BytecodeInteropVerification +{ + static readonly ComparisonCase Closures = new( + "dump-closures", + "Bytecode", + """ + local value=10 + local function make(step) + return function(...) + value=value+step + local first=... + return value+select('#',...)+first + end + end + local a,b=make(3),make(5) + assert(a(4)==18 and b(7,8)==27) + return a(20) + """, + 42 + ); + + static readonly ComparisonCase BinaryStrings = new( + "dump-binary-strings", + "Bytecode", + """ + local bytes='\0\128\255' + local utf8='\225\186\191' + local function check() + assert(#bytes==3 and bytes:byte(1)==0 and bytes:byte(2)==128 and bytes:byte(3)==255) + assert(#utf8==3 and utf8:byte(1)==225 and utf8:byte(2)==186 and utf8:byte(3)==191) + assert(''==string.sub(bytes,1,0)) + assert(true and not false and nil==nil) + return 42 + end + return check() + """, + 42 + ); + + public static async Task VerifyAsync() + { + var buffer = new byte[1024 * 1024]; + foreach ( + var workload in ComparisonWorkloads + .Cases.Concat(ComparisonWorkloads.CompilationCases) + .Append(Closures) + .Append(BinaryStrings) + .DistinctBy(c => c.Name) + ) + { + using var native = Lua52State.Create(); + native.OpenLibraries(); + native.RegisterFunction( + "host_add", + (vm, args) => + { + vm.PushNumber(args[0].Read() + args[1].Read()); + return 1; + } + ); + native.LoadString(workload.Source, "@bytecode"); + if (!native.TryDump(1, false, buffer, out var written)) + throw new InvalidOperationException( + $"Native dump exceeds verification buffer: {workload.Name}" + ); + native.SetTop(0); + + foreach (var mode in new[] { LuaStringMode.Utf16, LuaStringMode.Bytes }) + { + // Arbitrary bytes have the same semantics as C Lua in byte mode. + if (workload == BinaryStrings && mode != LuaStringMode.Bytes) + continue; + using var state = LuaState.Create( + mode == LuaStringMode.Bytes + ? LuaPlatform.DefaultByteStrings + : LuaPlatform.Default + ); + state.OpenStandardLibraries(); + state.Environment["host_add"] = new LuaFunction( + (context, _) => + new( + context.Return( + context.GetArgument(0) + context.GetArgument(1) + ) + ) + ); + + var loaded = state.Load(buffer.AsSpan(0, written), "native", "b"); + var results = await state.ExecuteAsync(loaded); + ComparisonWorkloads.Check( + workload, + $"NuLua dump -> Lua-CSharp {mode}", + results.Single().Read() + ); + + var original = state.Load(workload.Source, "@bytecode"); + var dump = state.Environment["string"].Read()["dump"].Read(); + var dumped = (await state.CallAsync(dump, [original])).Single().Read(); + native.LoadString("return assert(load(...,'managed','b'))()", "@bytecode"); + native.PushString(LuaByteString.ToBytes(dumped)); + native.Call(1, 1); + ComparisonWorkloads.Check( + workload, + $"Lua-CSharp {mode} dump -> NuLua", + native.ToNumber(-1) + ); + native.SetTop(0); + } + Console.WriteLine($"PASS Bytecode {workload.Name, -20} NuLua <-> Lua-CSharp"); + } + } +} diff --git a/sandbox/Benchmark/Program.cs b/sandbox/Benchmark/Program.cs index bcfce358..856edff5 100644 --- a/sandbox/Benchmark/Program.cs +++ b/sandbox/Benchmark/Program.cs @@ -16,6 +16,7 @@ $"NuLua Lua52 assembly: {typeof(NuLua.Lua52.Lua52State).Assembly.GetName().Version}" ); InterpreterComparison.Verify(); + await BytecodeInteropVerification.VerifyAsync(); return; } diff --git a/src/Lua/CodeAnalysis/Compilation/Dump.cs b/src/Lua/CodeAnalysis/Compilation/Dump.cs index 0057b106..4a22e24e 100644 --- a/src/Lua/CodeAnalysis/Compilation/Dump.cs +++ b/src/Lua/CodeAnalysis/Compilation/Dump.cs @@ -223,11 +223,7 @@ void WriteBool(bool v) void WriteString(string v) { var bytes = encoding.GetBytes(v); - var len = bytes.Length; - if (bytes.Length != 0) - { - len++; - } + var len = bytes.Length + 1; if (sizeof(IntPtr) == 8) { @@ -238,11 +234,8 @@ void WriteString(string v) WriteInt(len); } - if (len != 0) - { - Write(bytes); - WriteByte(0); - } + Write(bytes); + WriteByte(0); } void WriteConstants(ReadOnlySpan constants) @@ -250,20 +243,29 @@ void WriteConstants(ReadOnlySpan constants) WriteInt(constants.Length); foreach (var c in constants) { - WriteByte((byte)c.Type); + // Lua bytecode tags differ from LuaValueType for strings. switch (c.Type) { case LuaValueType.Nil: + WriteByte(0); break; case LuaValueType.Boolean: + WriteByte(1); WriteBool(c.UnsafeReadDouble() != 0); break; case LuaValueType.Number: + WriteByte(3); WriteDouble(c.UnsafeReadDouble()); break; case LuaValueType.String: + WriteByte(4); WriteString(c.UnsafeRead()); break; + default: + throw new ArgumentException( + "Unsupported bytecode constant type", + nameof(constants) + ); } } } @@ -520,18 +522,18 @@ LuaValue[] ReadConstants() var constants = new LuaValue[count]; for (var i = 0; i < count; i++) { - var type = (LuaValueType)ReadByte(); + var type = ReadByte(); switch (type) { - case LuaValueType.Nil: + case 0: break; - case LuaValueType.Boolean: + case 1: constants[i] = ReadBool(); break; - case LuaValueType.Number: + case 3: constants[i] = ReadDouble(); break; - case LuaValueType.String: + case 4: constants[i] = ReadString(); break; default: diff --git a/tests/Lua.Tests/BytecodeValidationTests.cs b/tests/Lua.Tests/BytecodeValidationTests.cs index 5623e64c..c39a2882 100644 --- a/tests/Lua.Tests/BytecodeValidationTests.cs +++ b/tests/Lua.Tests/BytecodeValidationTests.cs @@ -168,6 +168,46 @@ public void RejectsUnknownConstantTags() Assert.Throws(() => Prototype.FromBytecode(bytes, "constant")); } + [TestCase(LuaValueType.Nil, 0)] + [TestCase(LuaValueType.Boolean, 1)] + [TestCase(LuaValueType.Number, 3)] + [TestCase(LuaValueType.String, 4)] + public void ConstantTagsFollowLua52RatherThanTheInternalValueEnum(LuaValueType type, int tag) + { + LuaValue value = type switch + { + LuaValueType.Nil => LuaValue.Nil, + LuaValueType.Boolean => true, + LuaValueType.Number => 42, + _ => "hello", + }; + var bytes = Create([Instruction.Return(0, 1)], constants: [value]).ToBytecode(); + // Header, function fields, one instruction, and the constant count. + Assert.That(bytes[41], Is.EqualTo(tag)); + Assert.That(Prototype.FromBytecode(bytes, "constant").Constants[0], Is.EqualTo(value)); + bytes[41] = 2; // Lua's light userdata tag cannot occur in a serialized constant. + Assert.Throws(() => Prototype.FromBytecode(bytes, "constant")); + } + + [TestCase("", 1)] + [TestCase("a", 2)] + [TestCase("\0", 2)] + public void StringConstantLengthsIncludeTheirTerminator(string value, int length) + { + var bytes = Create([Instruction.Return(0, 1)], constants: [value]).ToBytecode(); + const int lengthOffset = 42; // Immediately after the first constant's tag. + var serializedLength = + IntPtr.Size == 8 + ? BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(lengthOffset)) + : BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(lengthOffset)); + Assert.That(serializedLength, Is.EqualTo(length)); + Assert.That(bytes[lengthOffset + IntPtr.Size + length - 1], Is.Zero); + Assert.That( + Prototype.FromBytecode(bytes, "constant").Constants[0], + Is.EqualTo(new LuaValue(value)) + ); + } + [Test] public void RejectsExcessivelyNestedPrototypes() { From 9f2b0f7b8703711ab254139e321af72b1ee40010 Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Fri, 30 Oct 2026 18:23:26 +0900 Subject: [PATCH 05/16] fix: preserve host error boundaries and Lua reader contracts Keep an outer xpcall handler pending while a C# CallAsync caller can catch the exception. Let Lua call sites handle errors that propagate back into Lua, and avoid dispatching handlers from RunAsync for host exceptions. Keep module searchers on the internal Lua call path so error handlers can still inspect failing Lua frames and locals. Check load(reader) modes after the first piece and reject Lua yields across reader calls. Use a nesting counter shared with error handlers, restore it on errors and cancellation, and retain support for asynchronous host reads. Keep source buffering and the synchronous compiler unchanged. Resolve print's tostring through the global metatable when the raw global is absent, while preserving its common direct lookup and per-call snapshot. Add regression coverage for synchronous and suspended host calls, nested reader yield boundaries, cancellation cleanup, first-piece mode rejection, searcher frame inspection, and suspended global tostring lookup. Known limitation: accepted reader input is still buffered to EOF before parsing. Early syntax errors and complete binary chunks do not stop reader callbacks as early as native Lua 5.2; a later reader error can take precedence. Validation: Release solution build; 2,861 runtime tests on each of .NET 8 and .NET 10; 73 source-generator tests; formatting; native interpreter and bytecode interoperability verification. Paired .NET 10 benchmarks show no additional allocations and ordinary reader, print, and public call timings within 1.5% of the preceding commit. --- src/Lua/LuaState.cs | 18 ++-- src/Lua/LuaStateExtensions.cs | 35 +++++++- src/Lua/Standard/BasicLibrary.cs | 64 ++++++++----- src/Lua/Standard/ModuleLibrary.cs | 3 +- tests/Lua.Tests/LoadReaderTests.cs | 79 ++++++++++++++++ tests/Lua.Tests/ProtectedCallTests.cs | 90 +++++++++++++++++-- .../Lua.Tests/StandardLibraryContractTests.cs | 44 +++++++++ 7 files changed, 292 insertions(+), 41 deletions(-) diff --git a/src/Lua/LuaState.cs b/src/Lua/LuaState.cs index ce01fe47..2d827952 100644 --- a/src/Lua/LuaState.cs +++ b/src/Lua/LuaState.cs @@ -146,7 +146,7 @@ public ValueTask YieldAsync( CancellationToken cancellationToken = default ) { - if (isInErrorHandler) + if (NonYieldableCallCount != 0) throw new LuaRuntimeException(this, "attempt to yield across a C-call boundary"); if (coroutine is not null) { @@ -164,7 +164,7 @@ public ValueTask YieldAsync( public ValueTask YieldAsync(LuaStack stack, CancellationToken cancellationToken = default) { - if (isInErrorHandler) + if (NonYieldableCallCount != 0) throw new LuaRuntimeException(this, "attempt to yield across a C-call boundary"); if (coroutine is not null) { @@ -223,7 +223,7 @@ public void Release() internal ILuaTracebackBuildable? CurrentException; internal int ErrorHandlerIndex; - bool isInErrorHandler; + internal int NonYieldableCallCount; internal readonly ReversedStack ExceptionTrace = new(); internal Prototype? LastCallerPrototype; @@ -604,13 +604,6 @@ public async ValueTask RunAsync( return await function.Func(context, cancellationToken); } - catch (Exception exception) - when (ErrorHandlerIndex != 0 && exception is not OperationCanceledException) - { - var handled = await HandleErrorAsync(exception, cancellationToken); - Stack.PopUntil(returnBase); - throw handled; - } catch { Stack.PopUntil(returnBase); @@ -652,8 +645,7 @@ CancellationToken cancellationToken error = Platform.GetErrorMessage(exception); var top = Stack.Count; - var previousInHandler = isInErrorHandler; - isInErrorHandler = true; + NonYieldableCallCount++; try { Stack.PushRange(handler, error); @@ -670,7 +662,7 @@ CancellationToken cancellationToken } finally { - isInErrorHandler = previousInHandler; + NonYieldableCallCount--; Stack.PopUntil(top); } Stack.Get(handlerIndex) = error; diff --git a/src/Lua/LuaStateExtensions.cs b/src/Lua/LuaStateExtensions.cs index 005655ab..7448c49e 100644 --- a/src/Lua/LuaStateExtensions.cs +++ b/src/Lua/LuaStateExtensions.cs @@ -546,7 +546,7 @@ public static ValueTask CallAsync( CancellationToken cancellationToken = default ) { - return LuaVirtualMachine.Call(state, funcIndex, funcIndex, cancellationToken); + return state.CallAsync(funcIndex, funcIndex, cancellationToken); } public static ValueTask CallAsync( @@ -556,7 +556,35 @@ public static ValueTask CallAsync( CancellationToken cancellationToken = default ) { - return LuaVirtualMachine.Call(state, funcIndex, returnBase, cancellationToken); + if (state.ErrorHandlerIndex == 0) + return LuaVirtualMachine.Call(state, funcIndex, returnBase, cancellationToken); + return Impl(state, funcIndex, returnBase, cancellationToken); + + [AsyncMethodBuilder(typeof(LightAsyncValueTaskMethodBuilder<>))] + static async ValueTask Impl( + LuaState state, + int funcIndex, + int returnBase, + CancellationToken cancellationToken + ) + { + // Host callers can catch the error before it propagates back into Lua. + var previousHandler = state.ErrorHandlerIndex; + state.ErrorHandlerIndex = 0; + try + { + return await LuaVirtualMachine.Call( + state, + funcIndex, + returnBase, + cancellationToken + ); + } + finally + { + state.ErrorHandlerIndex = previousHandler; + } + } } public static ValueTask CallAsync( @@ -577,6 +605,8 @@ static async ValueTask Impl( CancellationToken cancellationToken ) { + var previousHandler = state.ErrorHandlerIndex; + state.ErrorHandlerIndex = 0; try { await LuaVirtualMachine.Call(state, funcIndex, funcIndex, cancellationToken); @@ -584,6 +614,7 @@ CancellationToken cancellationToken } finally { + state.ErrorHandlerIndex = previousHandler; state.Stack.PopUntil(funcIndex); } } diff --git a/src/Lua/Standard/BasicLibrary.cs b/src/Lua/Standard/BasicLibrary.cs index 5dfc5460..c8862de5 100644 --- a/src/Lua/Standard/BasicLibrary.cs +++ b/src/Lua/Standard/BasicLibrary.cs @@ -280,25 +280,37 @@ CancellationToken cancellationToken using var source = new PooledList(256); var stack = context.State.Stack; var top = stack.Count; - while (true) + context.State.NonYieldableCallCount++; + try { - var count = await context.State.RunAsync(function, 0, top, cancellationToken); - var piece = count == 0 ? LuaValue.Nil : stack.Get(top); - if (piece.Type == LuaValueType.Nil) + while (true) { + var count = await context.State.RunAsync( + function, + 0, + top, + cancellationToken + ); + var piece = count == 0 ? LuaValue.Nil : stack.Get(top); stack.PopUntil(top); - break; + if (piece.Type == LuaValueType.Nil) + break; + if (piece.Type is not (LuaValueType.String or LuaValueType.Number)) + throw new LuaRuntimeException( + context.State, + "reader function must return a string" + ); + var text = piece.ToLuaString(context.State.Platform.NumberFormat); + if (text.Length == 0) + break; + if (source.Count == 0) + CheckLoadMode(text[0] == '\e', mode); + source.AddRange(text); } - if (piece.Type is not (LuaValueType.String or LuaValueType.Number)) - throw new LuaRuntimeException( - context.State, - "reader function must return a string" - ); - var text = piece.ToLuaString(context.State.Platform.NumberFormat); - stack.PopUntil(top); - if (text.Length == 0) - break; - source.AddRange(text); + } + finally + { + context.State.NonYieldableCallCount--; } closure = LoadSource(context.State, source.AsSpan(), name ?? "=(load)", mode); } @@ -326,6 +338,14 @@ CancellationToken cancellationToken } } + static void CheckLoadMode(bool binary, string mode) + { + if (!mode.Contains(binary ? 'b' : 't')) + throw new Exception( + $"attempt to load a {(binary ? "binary" : "text")} chunk (mode is '{mode}')" + ); + } + static LuaClosure LoadSource( LuaState state, ReadOnlySpan source, @@ -333,16 +353,13 @@ static LuaClosure LoadSource( string mode ) { + CheckLoadMode(!source.IsEmpty && source[0] == '\e', mode); if (!source.IsEmpty && source[0] == '\e') { - if (!mode.Contains('b')) - throw new Exception("attempt to load a binary chunk (mode is 't')"); using PooledArray bytes = new(source.Length); LuaByteEncoding.Instance.GetBytes(source, bytes.AsSpan()); return new(state, Parser.Undump(bytes.AsSpan(), name, state.Platform.StringMode)); } - if (!mode.Contains('t')) - throw new Exception("attempt to load a text chunk (mode is 'b')"); return state.LoadCore(source, name); } @@ -459,8 +476,15 @@ CancellationToken cancellationToken ) { var stdout = context.GlobalState.Platform.StandardIO.Output; - var toString = context.State.Environment["tostring"]; var argumentBase = context.FrameBase; + var environment = context.State.Environment; + if (!environment.TryGetValue("tostring", out var toString)) + toString = await LuaVirtualMachine.ExecuteGetTableSlowPath( + context.State, + environment, + "tostring", + cancellationToken + ); for (var i = 0; i < context.ArgumentCount; i++) { diff --git a/src/Lua/Standard/ModuleLibrary.cs b/src/Lua/Standard/ModuleLibrary.cs index 8ecf691f..f3532619 100644 --- a/src/Lua/Standard/ModuleLibrary.cs +++ b/src/Lua/Standard/ModuleLibrary.cs @@ -1,6 +1,7 @@ using System.Runtime.CompilerServices; using System.Text; using Lua.Internal.CompilerServices; +using Lua.Runtime; namespace Lua.Standard; @@ -161,7 +162,7 @@ CancellationToken cancellationToken var top = state.Stack.Count; state.Stack.Push(loader); state.Stack.Push(name); - var resultCount = await state.CallAsync(top, top, cancellationToken); + var resultCount = await LuaVirtualMachine.Call(state, top, top, cancellationToken); if (0 < resultCount) { var result = state.Stack.Get(top); diff --git a/tests/Lua.Tests/LoadReaderTests.cs b/tests/Lua.Tests/LoadReaderTests.cs index d28d1bea..ccd6d030 100644 --- a/tests/Lua.Tests/LoadReaderTests.cs +++ b/tests/Lua.Tests/LoadReaderTests.cs @@ -65,6 +65,81 @@ await state.DoStringAsync( Assert.That(state.CallStackFrameCount, Is.Zero); } + [TestCase("return 42", "b", "text", false)] + [TestCase("return 42", "b", "text", true)] + [TestCase("\u001bLua", "t", "binary", false)] + [TestCase("\u001bLua", "t", "binary", true)] + [TestCase("", "b", "text", false)] + [TestCase(null, "b", "text", false)] + public async Task ReaderChecksModeBeforeRequestingAnotherPiece( + string? piece, + string mode, + string kind, + bool asynchronous + ) + { + using var state = CreateState(); + var suspension = new ManualSuspension(); + var calls = 0; + state.Environment["reader"] = new LuaFunction( + async (context, _) => + { + if (++calls > 1) + throw new InvalidOperationException("reader must not be called again"); + if (asynchronous) + await suspension; + return context.Return(piece == null ? LuaValue.Nil : piece); + } + ); + state.Environment["mode"] = mode; + var operation = state.DoStringAsync("return load(reader,nil,mode)").AsTask(); + if (asynchronous) + suspension.Resume(); + var result = await operation; + Assert.That(result, Has.Length.EqualTo(2)); + Assert.That(result[0], Is.EqualTo(LuaValue.Nil)); + Assert.That(result[1].Read(), Does.Contain($"attempt to load a {kind} chunk")); + Assert.That(calls, Is.EqualTo(1)); + Assert.That(state.Stack.Count, Is.Zero); + Assert.That(state.CallStackFrameCount, Is.Zero); + Assert.That(state.NonYieldableCallCount, Is.Zero); + } + + [TestCase(0, false)] + [TestCase(1, false)] + [TestCase(1, true)] + public async Task ReaderCannotYieldAndRestoresYieldability(int piecesBeforeYield, bool nested) + { + using var state = CreateState(); + state.Environment["piecesBeforeYield"] = piecesBeforeYield; + state.Environment["nested"] = nested; + await state.DoStringAsync( + """ + local co=coroutine.create(function() + local calls=0 + local f,e=load(function() + calls=calls+1 + if calls<=piecesBeforeYield then return 'return 42' end + if nested then + local inner,message=load(function() coroutine.yield('inner reader') end) + assert(inner==nil and message:find('attempt to yield across a C-call boundary',1,true)) + end + coroutine.yield('reader') + end) + assert(f==nil and e:find('attempt to yield across a C-call boundary',1,true)) + coroutine.yield('after load') + return 'done' + end) + local ok,value=coroutine.resume(co) + assert(ok and value=='after load' and coroutine.status(co)=='suspended') + ok,value=coroutine.resume(co) + assert(ok and value=='done' and coroutine.status(co)=='dead') + """ + ); + Assert.That(state.Stack.Count, Is.Zero); + Assert.That(state.CallStackFrameCount, Is.Zero); + } + [Test] public async Task ReaderCanSuspendAndCancellationIsNotConvertedToALoadError() { @@ -83,6 +158,7 @@ public async Task ReaderCanSuspendAndCancellationIsNotConvertedToALoadError() var operation = state.DoStringAsync("return assert(load(reader))()").AsTask(); suspension.Resume(); Assert.That(await operation, Is.EqualTo(new LuaValue[] { 42 })); + Assert.That(state.NonYieldableCallCount, Is.Zero); using var source = new CancellationTokenSource(); state.Environment["reader"] = new LuaFunction( (context, token) => @@ -95,6 +171,9 @@ public async Task ReaderCanSuspendAndCancellationIsNotConvertedToALoadError() Assert.ThrowsAsync(async () => await state.DoStringAsync("return load(reader)", cancellationToken: source.Token) ); + Assert.That(state.NonYieldableCallCount, Is.Zero); + Assert.That(state.Stack.Count, Is.Zero); + Assert.That(state.CallStackFrameCount, Is.Zero); } [TestCase(false)] diff --git a/tests/Lua.Tests/ProtectedCallTests.cs b/tests/Lua.Tests/ProtectedCallTests.cs index 9769e813..ebf61a46 100644 --- a/tests/Lua.Tests/ProtectedCallTests.cs +++ b/tests/Lua.Tests/ProtectedCallTests.cs @@ -83,12 +83,15 @@ public async Task XpcallSupportsOverlappingAndForwardReturnBuffers(int returnBas ); } - [TestCase(false)] - [TestCase(true)] - public async Task HandlerCanInspectFailingFramesAndLocals(bool asynchronous) + [TestCase(false, false)] + [TestCase(true, false)] + [TestCase(false, true)] + [TestCase(true, true)] + public async Task HandlerCanInspectFailingFramesAndLocals(bool asynchronous, bool searcher) { using var state = CreateState(); var suspension = new ManualSuspension(); + state.Environment["searcher"] = searcher; state.Environment["pause"] = new LuaFunction( async (context, _) => { @@ -107,9 +110,12 @@ local function namedFailure() pause() error('fail') end - local ok,message=xpcall(function() namedFailure() end,function(e) + if searcher then package.searchers={namedFailure} end + local ok,message=xpcall(function() + if searcher then require('missing') else namedFailure() end + end,function(e) local trace=debug.traceback(e) - assert(trace:find('namedFailure',1,true)) + if not searcher then assert(trace:find('namedFailure',1,true)) end for level=1,10 do local info=debug.getinfo(level,'f') if info and info.func==namedFailure then @@ -158,6 +164,80 @@ await state.DoStringAsync( Assert.That(state.ErrorHandlerIndex, Is.Zero); } + [TestCase(false, "CallAsync")] + [TestCase(true, "CallAsync")] + [TestCase(false, "CallStackAsync")] + [TestCase(true, "CallStackAsync")] + [TestCase(false, "RunAsync")] + [TestCase(true, "RunAsync")] + public async Task CaughtHostExceptionsDoNotConsumeTheOuterHandler(bool asynchronous, string api) + { + using var state = CreateState(); + var suspension = new ManualSuspension(); + var fail = new LuaFunction( + async (_, _) => + { + if (asynchronous) + await suspension; + throw new InvalidOperationException("caught host failure"); + } + ); + state.Environment["recover"] = new LuaFunction( + async (context, token) => + { + try + { + if (api == "RunAsync") + await context.State.RunAsync(fail, token); + else if (api == "CallStackAsync") + { + var top = context.State.Stack.Count; + context.State.Stack.Push(fail); + try + { + await context.State.CallAsync(top, token); + } + finally + { + context.State.Stack.PopUntil(top); + } + } + else + await context.State.CallAsync(fail, [], token); + } + catch (InvalidOperationException) { } + return context.Return(); + } + ); + var operation = state + .DoStringAsync( + """ + local seen={} + local ok,value=xpcall(function() + recover() + return 42 + end,function(e) seen[#seen+1]=e;return e end) + assert(ok and value==42 and #seen==0) + local ok,value=xpcall(function() + recover() + error('real failure',0) + end,function(e) seen[#seen+1]=e;return e end) + return ok,value,#seen,seen[1] + """ + ) + .AsTask(); + if (asynchronous) + { + suspension.Resume(); + suspension.Resume(); + } + Assert.That( + await operation, + Is.EqualTo(new LuaValue[] { false, "real failure", 1, "real failure" }) + ); + Assert.That(state.ErrorHandlerIndex, Is.Zero); + } + [Test] public async Task HostErrorHandlerCanSuspendWithoutLosingTheErrorOrFrames() { diff --git a/tests/Lua.Tests/StandardLibraryContractTests.cs b/tests/Lua.Tests/StandardLibraryContractTests.cs index fc4875fc..a2839e01 100644 --- a/tests/Lua.Tests/StandardLibraryContractTests.cs +++ b/tests/Lua.Tests/StandardLibraryContractTests.cs @@ -83,6 +83,50 @@ await state.DoStringAsync( ); } + [TestCase(false)] + [TestCase(true)] + public async Task PrintResolvesTostringThroughTheGlobalMetatable(bool asynchronous) + { + using var state = CreateState(); + var suspension = new ManualSuspension(); + state.Environment["pause"] = new LuaFunction( + async (context, _) => + { + if (asynchronous) + await suspension; + return context.Return(); + } + ); + var output = new List(); + state.Platform = state.Platform with + { + StandardIO = new TestStandardIO(stringMode) + { + Output = new BufferedOutputStream(value => output.Add(value.ToString())), + }, + }; + var operation = state + .DoStringAsync( + """ + local calls=0 + _G.tostring=nil + setmetatable(_G,{__index=function(t,key) + pause() + assert(key=='tostring') + calls=calls+1 + return function(value) return value*2 end + end}) + print(1,2) + assert(calls==1) + """ + ) + .AsTask(); + if (asynchronous) + suspension.Resume(); + await operation; + Assert.That(output, Is.EqualTo(new[] { "2\t4\n" })); + } + [Test] public async Task FileLinesRetainsEveryReadFormat() { From c94476aa06ca93ad98729dc9c8b6d04166e00a77 Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:11:23 +0900 Subject: [PATCH 06/16] fix: preserve Lua 5.2 execution and module boundaries Keep enclosing xpcall handlers pending while C# callers can catch Lua exceptions from RunAsync, ExecuteAsync, and DoStringAsync. Share the runner with internal library calls, which retain the active handler so it can inspect failing frames before unwinding. Keep the ordinary public runner direct when no outer handler is installed. Reject Lua yields from print callbacks and module-loading callbacks while preserving asynchronous host awaits. Restore the nesting counter on errors and cancellation, and leave already-cached module returns outside the slow loading path. Honor package.loaded reads and writes through its metatable, including the default true result for loaders that return nil. Honor package and preload field lookups as well. Capture the package table in require and the Lua file searcher through existing CSharpClosure upvalues so global reassignment and library reopening do not change saved functions or their debug upvalues. Require a mask when installing a debug hook, and ignore unused mask/count arguments when disabling one. Add regression coverage for all five issues in both string modes, including suspended host calls, failing-frame inspection, package replacement and reopening, direct searcher calls, metatable callbacks, yield restoration, and cancellation cleanup. Validation: Release solution build across all targets; 2,925 runtime tests on each of .NET 8 and .NET 10 plus focused direct-searcher regressions; 73 source-generator tests; formatting; native Lua 5.2 behavior comparisons and interpreter/bytecode interoperability. Paired .NET 10 benchmarks add no steady-state allocations. Ordinary print, reader, and public call paths remain near baseline, with no cached-require slowdown in the final comparison. A synthetic repeatedly uncached preload loop is about 8% slower, and protected RunAsync reentry adds about 12 ns per call for preserving the enclosing host exception boundary. --- src/Lua/Internal/CoroutineCore.cs | 7 +- src/Lua/LuaState.cs | 49 ++++- src/Lua/Standard/BasicLibrary.cs | 74 ++++--- src/Lua/Standard/DebugLibrary.cs | 13 +- src/Lua/Standard/ModuleLibrary.cs | 206 ++++++++++++------ src/Lua/Standard/OpenLibsExtensions.cs | 8 +- src/Lua/Standard/StringLibrary.cs | 3 +- src/Lua/Standard/TableLibrary.cs | 2 +- tests/Lua.Tests/HookArgumentTests.cs | 44 ++++ tests/Lua.Tests/ModuleContractTests.cs | 148 +++++++++++++ tests/Lua.Tests/ProtectedCallTests.cs | 101 ++++++++- .../Lua.Tests/StandardLibraryContractTests.cs | 59 +++++ 12 files changed, 602 insertions(+), 112 deletions(-) diff --git a/src/Lua/Internal/CoroutineCore.cs b/src/Lua/Internal/CoroutineCore.cs index 52722715..3015d12e 100644 --- a/src/Lua/Internal/CoroutineCore.cs +++ b/src/Lua/Internal/CoroutineCore.cs @@ -166,7 +166,12 @@ phase is Completed or Closed or Cancelling if (cancelled) executionSource!.Cancel(); isFirstCall = false; - functionTask = Thread.RunAsync(Function!, Thread.Stack.Count, executionToken); + functionTask = Thread.RunCoreAsync( + Function!, + Thread.Stack.Count, + 0, + executionToken + ); if (functionTask.IsCompleted) OnFunctionCompleted(); else diff --git a/src/Lua/LuaState.cs b/src/Lua/LuaState.cs index 2d827952..9b132047 100644 --- a/src/Lua/LuaState.cs +++ b/src/Lua/LuaState.cs @@ -568,13 +568,51 @@ public ValueTask RunAsync( return RunAsync(function, argumentCount, Stack.Count - argumentCount, cancellationToken); } - [AsyncMethodBuilder(typeof(LightAsyncValueTaskMethodBuilder<>))] - public async ValueTask RunAsync( + public ValueTask RunAsync( LuaFunction function, int argumentCount, int returnBase, CancellationToken cancellationToken = default ) + { + if (ErrorHandlerIndex == 0) + return RunCoreAsync(function, argumentCount, returnBase, cancellationToken); + return RunProtectedAsync(this, function, argumentCount, returnBase, cancellationToken); + + [AsyncMethodBuilder(typeof(LightAsyncValueTaskMethodBuilder<>))] + static async ValueTask RunProtectedAsync( + LuaState state, + LuaFunction function, + int argumentCount, + int returnBase, + CancellationToken cancellationToken + ) + { + var previousHandler = state.ErrorHandlerIndex; + state.ErrorHandlerIndex = 0; + try + { + return await state.RunCoreAsync( + function, + argumentCount, + returnBase, + cancellationToken + ); + } + finally + { + state.ErrorHandlerIndex = previousHandler; + } + } + } + + [AsyncMethodBuilder(typeof(LightAsyncValueTaskMethodBuilder<>))] + internal async ValueTask RunCoreAsync( + LuaFunction function, + int argumentCount, + int returnBase, + CancellationToken cancellationToken + ) { if (function == null) { @@ -604,6 +642,13 @@ public async ValueTask RunAsync( return await function.Func(context, cancellationToken); } + catch (Exception exception) + when (ErrorHandlerIndex != 0 && exception is not OperationCanceledException) + { + var handled = await HandleErrorAsync(exception, cancellationToken); + Stack.PopUntil(returnBase); + throw handled; + } catch { Stack.PopUntil(returnBase); diff --git a/src/Lua/Standard/BasicLibrary.cs b/src/Lua/Standard/BasicLibrary.cs index c8862de5..2da70a0c 100644 --- a/src/Lua/Standard/BasicLibrary.cs +++ b/src/Lua/Standard/BasicLibrary.cs @@ -139,7 +139,12 @@ CancellationToken cancellationToken null, cancellationToken ); - return await context.State.RunAsync(closure, cancellationToken); + return await context.State.RunCoreAsync( + closure, + 0, + context.State.Stack.Count, + cancellationToken + ); } public ValueTask Error( @@ -285,7 +290,7 @@ CancellationToken cancellationToken { while (true) { - var count = await context.State.RunAsync( + var count = await context.State.RunCoreAsync( function, 0, top, @@ -478,38 +483,51 @@ CancellationToken cancellationToken var stdout = context.GlobalState.Platform.StandardIO.Output; var argumentBase = context.FrameBase; var environment = context.State.Environment; - if (!environment.TryGetValue("tostring", out var toString)) - toString = await LuaVirtualMachine.ExecuteGetTableSlowPath( - context.State, - environment, - "tostring", - cancellationToken - ); - - for (var i = 0; i < context.ArgumentCount; i++) + context.State.NonYieldableCallCount++; + try { - var stack = context.State.Stack; - var top = stack.Count; - stack.PushRange(toString, stack.Get(argumentBase + i)); - var count = await LuaVirtualMachine.Call(context.State, top, top, cancellationToken); - var value = count > 0 ? stack.Get(top) : LuaValue.Nil; - if (value.Type is not (LuaValueType.String or LuaValueType.Number)) - throw new LuaRuntimeException( + if (!environment.TryGetValue("tostring", out var toString)) + toString = await LuaVirtualMachine.ExecuteGetTableSlowPath( context.State, - "'tostring' must return a string to 'print'" + environment, + "tostring", + cancellationToken ); - var text = value.ToLuaString(context.State.Platform.NumberFormat); - stack.PopUntil(top); - await stdout.WriteAsync(text, cancellationToken); - if (i < context.ArgumentCount - 1) + + for (var i = 0; i < context.ArgumentCount; i++) { - await stdout.WriteAsync("\t", cancellationToken); + var stack = context.State.Stack; + var top = stack.Count; + stack.PushRange(toString, stack.Get(argumentBase + i)); + var count = await LuaVirtualMachine.Call( + context.State, + top, + top, + cancellationToken + ); + var value = count > 0 ? stack.Get(top) : LuaValue.Nil; + if (value.Type is not (LuaValueType.String or LuaValueType.Number)) + throw new LuaRuntimeException( + context.State, + "'tostring' must return a string to 'print'" + ); + var text = value.ToLuaString(context.State.Platform.NumberFormat); + stack.PopUntil(top); + await stdout.WriteAsync(text, cancellationToken); + if (i < context.ArgumentCount - 1) + { + await stdout.WriteAsync("\t", cancellationToken); + } } - } - await stdout.WriteAsync("\n", cancellationToken); - await stdout.FlushAsync(cancellationToken); - return context.Return(); + await stdout.WriteAsync("\n", cancellationToken); + await stdout.FlushAsync(cancellationToken); + return context.Return(); + } + finally + { + context.State.NonYieldableCallCount--; + } } public ValueTask RawEqual( diff --git a/src/Lua/Standard/DebugLibrary.cs b/src/Lua/Standard/DebugLibrary.cs index fcbc3f8a..38939d27 100644 --- a/src/Lua/Standard/DebugLibrary.cs +++ b/src/Lua/Standard/DebugLibrary.cs @@ -481,17 +481,16 @@ CancellationToken cancellationToken { var state = GetLuaThread(context, out var argOffset); var hook = context.GetArgumentOrDefault(argOffset); - var mask = context.HasArgument(argOffset + 1) - ? context.GetStringArgument(argOffset + 1) - : ""; - var count = context.HasArgument(argOffset + 2) - ? context.GetIntegerArgument(argOffset + 2) - : 0; - state.SetHookFromLua(hook, mask, count); if (hook is null) { + state.SetHookFromLua(null, "", 0); return context.Return(); } + var mask = context.GetStringArgument(argOffset + 1); + var count = context.HasArgument(argOffset + 2) + ? context.GetIntegerArgument(argOffset + 2) + : 0; + state.SetHookFromLua(hook, mask, count); // Re-arming a hook must not dispatch another hook while its callback is running. if (state.IsReturnHookEnabled && context.State == state && !state.IsInHook) diff --git a/src/Lua/Standard/ModuleLibrary.cs b/src/Lua/Standard/ModuleLibrary.cs index f3532619..f156fde5 100644 --- a/src/Lua/Standard/ModuleLibrary.cs +++ b/src/Lua/Standard/ModuleLibrary.cs @@ -20,63 +20,111 @@ public ModuleLibrary() public readonly LuaFunction RequireFunction; public readonly LuaFunction SearchPathFunction; - [AsyncMethodBuilder(typeof(LightAsyncValueTaskMethodBuilder<>))] - public async ValueTask Require( + public ValueTask Require( LuaFunctionExecutionContext context, CancellationToken cancellationToken ) { var arg0 = context.GetStringArgument(0); var loaded = context.GlobalState.LoadedModules; - var loadedTable = loaded[arg0]; - if (!loadedTable.ToBoolean()) + if (loadedTable.ToBoolean()) + return new(context.Return(loadedTable)); + return RequireAsync(context, arg0, loaded, loadedTable, cancellationToken); + } + + [AsyncMethodBuilder(typeof(LightAsyncValueTaskMethodBuilder<>))] + static async ValueTask RequireAsync( + LuaFunctionExecutionContext context, + string arg0, + LuaTable loaded, + LuaValue loadedTable, + CancellationToken cancellationToken + ) + { + var state = context.State; + state.NonYieldableCallCount++; + try { - LuaFunction loader; - LuaValue data = LuaValue.Nil; - var moduleLoader = context.GlobalState.ModuleLoader; - var hostName = moduleLoader is null ? arg0 : context.State.Platform.DecodeText(arg0); - if (moduleLoader != null && moduleLoader.Exists(hostName)) - { - var module = await moduleLoader.LoadAsync(hostName, cancellationToken); - loader = context.State.Load(module); - } - else + if (loadedTable.Type == LuaValueType.Nil && loaded.Metatable != null) + loadedTable = await LuaVirtualMachine.ExecuteGetTableSlowPath( + state, + loaded, + arg0, + cancellationToken + ); + if (!loadedTable.ToBoolean()) { - (loader, data) = await FindLoader(context.State, arg0, cancellationToken); - } + LuaFunction loader; + LuaValue data = LuaValue.Nil; + var moduleLoader = context.GlobalState.ModuleLoader; + var hostName = moduleLoader is null ? arg0 : state.Platform.DecodeText(arg0); + if (moduleLoader != null && moduleLoader.Exists(hostName)) + { + var module = await moduleLoader.LoadAsync(hostName, cancellationToken); + loader = state.Load(module); + } + else + { + (loader, data) = await FindLoader( + state, + GetPackage(context), + arg0, + cancellationToken + ); + } - context.State.Stack.PushRange(arg0, data); - var count = await context.State.RunAsync( - loader, - 2, - context.ReturnFrameBase, - cancellationToken - ); - if ( - count > 0 - && context.State.Stack.Get(context.ReturnFrameBase).Type != LuaValueType.Nil - ) - loaded[arg0] = context.State.Stack.Get(context.ReturnFrameBase); - loadedTable = loaded[arg0]; - if (loadedTable.Type == LuaValueType.Nil) - loaded[arg0] = loadedTable = true; + state.Stack.PushRange(arg0, data); + var count = await state.RunCoreAsync( + loader, + 2, + context.ReturnFrameBase, + cancellationToken + ); + if (count > 0 && state.Stack.Get(context.ReturnFrameBase).Type != LuaValueType.Nil) + await state.SetTableAsync( + loaded, + arg0, + state.Stack.Get(context.ReturnFrameBase), + cancellationToken + ); + if (!loaded.TryGetValue(arg0, out loadedTable) && loaded.Metatable != null) + loadedTable = await LuaVirtualMachine.ExecuteGetTableSlowPath( + state, + loaded, + arg0, + cancellationToken + ); + if (loadedTable.Type == LuaValueType.Nil) + { + loadedTable = true; + await state.SetTableAsync(loaded, arg0, loadedTable, cancellationToken); + } + } + return context.Return(loadedTable); + } + finally + { + state.NonYieldableCallCount--; } - - return context.Return(loadedTable); } + static LuaValue GetPackage(LuaFunctionExecutionContext context) => + context.GetCsClosure() is { } closure + ? closure.UpValues[0] + : context.GlobalState.Environment["package"]; + [AsyncMethodBuilder(typeof(LightAsyncValueTaskMethodBuilder<>))] internal static async ValueTask<(string? FileName, string? Error)> FindFile( LuaState state, + LuaValue package, string name, string pName, - string dirSeparator + string dirSeparator, + CancellationToken cancellationToken ) { - var globalState = state.GlobalState; - var package = globalState.Environment["package"]; - var p = await state.GetTableAsync(package, pName); + var p = await state.GetTableAsync(package, pName, cancellationToken); if (p.Type is not (LuaValueType.String or LuaValueType.Number)) { throw new LuaRuntimeException(state, $"package.{pName} must be a string"); @@ -145,12 +193,20 @@ string dirSeparator [AsyncMethodBuilder(typeof(LightAsyncValueTaskMethodBuilder<>))] internal static async ValueTask<(LuaFunction Function, LuaValue Data)> FindLoader( LuaState state, + LuaValue package, string name, CancellationToken cancellationToken ) { - var package = state.GlobalState.Environment["package"].Read(); - var searchers = package["searchers"].Read(); + if (!package.TryReadTable(out var table) || !table.TryGetValue("searchers", out var value)) + value = await LuaVirtualMachine.ExecuteGetTableSlowPath( + state, + package, + "searchers", + cancellationToken + ); + if (!value.TryReadTable(out var searchers)) + throw new LuaRuntimeException(state, "package.searchers must be a table"); StringBuilder? errors = null; for (var i = 1; ; i++) { @@ -182,19 +238,37 @@ CancellationToken cancellationToken throw new LuaRuntimeException(state, $"module '{name}' not found:{errors}"); } - public ValueTask SearcherPreload( + [AsyncMethodBuilder(typeof(LightAsyncValueTaskMethodBuilder<>))] + public async ValueTask SearcherPreload( LuaFunctionExecutionContext context, CancellationToken cancellationToken ) { var name = context.GetStringArgument(0); - var preload = context.GlobalState.PreloadModules[name]; + var table = context.GlobalState.PreloadModules; + if (!table.TryGetValue(name, out var preload) && table.Metatable != null) + { + context.State.NonYieldableCallCount++; + try + { + preload = await LuaVirtualMachine.ExecuteGetTableSlowPath( + context.State, + table, + name, + cancellationToken + ); + } + finally + { + context.State.NonYieldableCallCount--; + } + } if (preload == LuaValue.Nil) { - return new(context.Return($"\n\tno field package.preload['{name}']")); + return context.Return($"\n\tno field package.preload['{name}']"); } - return new(context.Return(preload)); + return context.Return(preload); } public async ValueTask SearcherLua( @@ -203,25 +277,35 @@ CancellationToken cancellationToken ) { var name = context.GetStringArgument(0); - var (fileName, error) = await FindFile( - context.State, - name, - "path", - context.GlobalState.Platform.FileSystem.DirectorySeparator - ); - if (fileName == null) + context.State.NonYieldableCallCount++; + try { - return context.Return(error!); - } - - return context.Return( - await context.State.LoadFileAsync( - context.State.Platform.DecodeText(fileName), - "bt", - null, + var (fileName, error) = await FindFile( + context.State, + GetPackage(context), + name, + "path", + context.GlobalState.Platform.FileSystem.DirectorySeparator, cancellationToken - ), - fileName - ); + ); + if (fileName == null) + { + return context.Return(error!); + } + + return context.Return( + await context.State.LoadFileAsync( + context.State.Platform.DecodeText(fileName), + "bt", + null, + cancellationToken + ), + fileName + ); + } + finally + { + context.State.NonYieldableCallCount--; + } } } diff --git a/src/Lua/Standard/OpenLibsExtensions.cs b/src/Lua/Standard/OpenLibsExtensions.cs index e987e917..de07aef7 100644 --- a/src/Lua/Standard/OpenLibsExtensions.cs +++ b/src/Lua/Standard/OpenLibsExtensions.cs @@ -95,13 +95,17 @@ public static void OpenModuleLibrary(this LuaState state) var moduleLibrary = ModuleLibrary.Instance; LuaTable searchers = new(); searchers[1] = new LuaFunction("preload", moduleLibrary.SearcherPreload); - searchers[2] = new LuaFunction("searcher_Lua", moduleLibrary.SearcherLua); + searchers[2] = new CSharpClosure("searcher_Lua", [package], moduleLibrary.SearcherLua); package["searchers"] = searchers; package["path"] = "?.lua"; package["searchpath"] = moduleLibrary.SearchPathFunction; package["config"] = $"{Path.DirectorySeparatorChar}\n;\n?\n!\n-"; globalState.Environment["package"] = package; - globalState.Environment["require"] = moduleLibrary.RequireFunction; + globalState.Environment["require"] = new CSharpClosure( + "require", + [package], + moduleLibrary.Require + ); } public static void OpenOperatingSystemLibrary(this LuaState state) diff --git a/src/Lua/Standard/StringLibrary.cs b/src/Lua/Standard/StringLibrary.cs index f85567b9..44af181a 100644 --- a/src/Lua/Standard/StringLibrary.cs +++ b/src/Lua/Standard/StringLibrary.cs @@ -706,9 +706,10 @@ or LuaValueType.Function { stack.Push(matchState.GetCapture(k)); } - var retCount = await context.State.RunAsync( + var retCount = await context.State.RunCoreAsync( func, matchState.Level, + stack.Count - matchState.Level, cancellationToken ); using var results = context.State.ReadStack(retCount); diff --git a/src/Lua/Standard/TableLibrary.cs b/src/Lua/Standard/TableLibrary.cs index 739193b2..b9fbe482 100644 --- a/src/Lua/Standard/TableLibrary.cs +++ b/src/Lua/Standard/TableLibrary.cs @@ -430,7 +430,7 @@ CancellationToken cancellationToken stack.Push(left); stack.Push(right); - var runTask = state.RunAsync(comparer, 2, cancellationToken); + var runTask = state.RunCoreAsync(comparer, 2, top, cancellationToken); if (runTask.IsCompletedSuccessfully) { var returnCount = runTask.GetAwaiter().GetResult(); diff --git a/tests/Lua.Tests/HookArgumentTests.cs b/tests/Lua.Tests/HookArgumentTests.cs index c052f591..a478bc7e 100644 --- a/tests/Lua.Tests/HookArgumentTests.cs +++ b/tests/Lua.Tests/HookArgumentTests.cs @@ -18,6 +18,50 @@ LuaState CreateState() return state; } + [TestCase(false)] + [TestCase(true)] + public async Task DisablingHooksIgnoresUnusedArguments(bool thread) + { + using var state = CreateState(); + state.Environment["thread"] = thread; + await state.DoStringAsync( + """ + local co=coroutine.create(function() end) + local function hook() end + if thread then + debug.sethook(co,hook,'c') + assert(debug.gethook(co)==hook) + debug.sethook(co,nil,{}, {}) + assert(debug.gethook(co)==nil) + else + debug.sethook(hook,'c') + assert(debug.gethook()==hook) + debug.sethook(nil,{}, {}) + assert(debug.gethook()==nil) + end + """ + ); + Assert.That(state.Stack.Count, Is.EqualTo(4)); + Assert.That(state.CallStackFrameCount, Is.Zero); + } + + [TestCase("")] + [TestCase(",nil")] + public async Task InstallingHooksRequiresAMask(string mask) + { + using var state = CreateState(); + await state.DoStringAsync( + $$""" + local original=function() end + debug.sethook(original,'c') + local ok,e=pcall(function() debug.sethook(function() end{{mask}}) end) + assert(not ok and type(e)=='string') + assert(debug.gethook()==original) + debug.sethook() + """ + ); + } + [TestCase(0, "c")] [TestCase(1, "c")] [TestCase(2, "c")] diff --git a/tests/Lua.Tests/ModuleContractTests.cs b/tests/Lua.Tests/ModuleContractTests.cs index 64b9880b..064085ba 100644 --- a/tests/Lua.Tests/ModuleContractTests.cs +++ b/tests/Lua.Tests/ModuleContractTests.cs @@ -54,6 +54,154 @@ await state.DoStringAsync( ); } + [TestCase(false)] + [TestCase(true)] + public async Task RequireHonorsLoadedModuleMetamethods(bool asynchronous) + { + using var state = CreateState(); + var suspension = new ManualSuspension(); + state.Environment["pause"] = new LuaFunction( + async (context, _) => + { + if (asynchronous) + await suspension; + return context.Return(); + } + ); + var operation = state + .DoStringAsync( + """ + local writes=0 + setmetatable(package.loaded,{ + __index=function(t,key) + if key=='cached' then pause();return 42 end + end, + __newindex=function(t,key,value) + writes=writes+1 + if key=='loaded' then + pause() + rawset(t,key,value+1) + else + assert(key=='empty' and value==true) + rawset(t,key,43) + end + end + }) + package.preload.loaded=function() return 41 end + package.preload.empty=function() end + assert(require('cached')==42) + assert(require('loaded')==42) + assert(require('empty')==true and require('empty')==43) + assert(writes==2) + """ + ) + .AsTask(); + if (asynchronous) + { + suspension.Resume(); + suspension.Resume(); + } + await operation; + Assert.That(state.Stack.Count, Is.Zero); + Assert.That(state.CallStackFrameCount, Is.Zero); + Assert.That(state.NonYieldableCallCount, Is.Zero); + } + + [TestCase(false)] + [TestCase(true)] + public async Task RequireAndFileSearcherRetainTheirPackage(bool reopen) + { + using var state = CreateState(); + state.Platform = state.Platform with { FileSystem = new ModuleFileSystem() }; + await state.DoStringAsync( + "savedRequire=require;savedSearcher=package.searchers[2];savedPackage=package" + ); + if (reopen) + state.OpenModuleLibrary(); + else + state.Environment["package"] = new LuaTable(); + await state.DoStringAsync( + """ + package.path='missing/?.lua' + savedPackage.preload.preloaded=function() return 42 end + local _,value=debug.getupvalue(savedRequire,1) + assert(value==savedPackage) + _,value=debug.getupvalue(savedSearcher,1) + assert(value==savedPackage) + assert(savedRequire('preloaded')==42) + local f,path=savedSearcher('test') + assert(type(f)=='function' and path=='test.lua' and f('test',path)==true) + assert(savedRequire('test')==true) + """ + ); + } + + [Test] + public async Task PackageAndPreloadFieldsUseNormalTableAccess() + { + using var state = CreateState(); + state.Platform = state.Platform with { FileSystem = new ModuleFileSystem() }; + await state.DoStringAsync( + """ + local searchers=package.searchers + package.searchers=nil + package.path=nil + setmetatable(package,{__index=function(t,key) + if key=='searchers' then return searchers end + if key=='path' then return '?.lua' end + end}) + setmetatable(package.preload,{__index=function(t,key) + if key=='virtual' then return function(name) assert(name==key);return 42 end end + end}) + assert(require('virtual')==42) + assert(require('test')==true) + """ + ); + } + + [TestCase("package.preload.test=function() coroutine.yield('loader') end", "require")] + [TestCase("package.searchers={function() coroutine.yield('searcher') end}", "require")] + [TestCase( + "setmetatable(package.loaded,{__index=function() coroutine.yield('index') end})", + "require" + )] + [TestCase( + "package.preload.test=function() return 42 end;setmetatable(package.loaded,{__newindex=function() coroutine.yield('newindex') end})", + "require" + )] + [TestCase( + "setmetatable(package.preload,{__index=function() coroutine.yield('preload') end})", + "package.searchers[1]" + )] + [TestCase( + "package.path=nil;setmetatable(package,{__index=function() coroutine.yield('path') end})", + "package.searchers[2]" + )] + public async Task ModuleCallbacksCannotYieldAndRestoreYieldability( + string setup, + string function + ) + { + using var state = CreateState(); + await state.DoStringAsync( + setup + + $$""" + ;local co=coroutine.create(function() + local ok,e=pcall({{function}},'test') + assert(not ok and e:find('attempt to yield across a C-call boundary',1,true)) + coroutine.yield('after require') + return 42 + end) + local ok,value=coroutine.resume(co) + assert(ok and value=='after require') + ok,value=coroutine.resume(co) + assert(ok and value==42 and coroutine.status(co)=='dead') + """ + ); + Assert.That(state.Stack.Count, Is.Zero); + Assert.That(state.CallStackFrameCount, Is.Zero); + } + [Test] public async Task FileLoaderReceivesItsNameAndPath() { diff --git a/tests/Lua.Tests/ProtectedCallTests.cs b/tests/Lua.Tests/ProtectedCallTests.cs index ebf61a46..7822da46 100644 --- a/tests/Lua.Tests/ProtectedCallTests.cs +++ b/tests/Lua.Tests/ProtectedCallTests.cs @@ -83,15 +83,21 @@ public async Task XpcallSupportsOverlappingAndForwardReturnBuffers(int returnBas ); } - [TestCase(false, false)] - [TestCase(true, false)] - [TestCase(false, true)] - [TestCase(true, true)] - public async Task HandlerCanInspectFailingFramesAndLocals(bool asynchronous, bool searcher) + [TestCase(false, "direct")] + [TestCase(true, "direct")] + [TestCase(false, "searcher")] + [TestCase(true, "searcher")] + [TestCase(false, "loader")] + [TestCase(true, "loader")] + [TestCase(false, "sort")] + [TestCase(true, "sort")] + [TestCase(false, "gsub")] + [TestCase(true, "gsub")] + public async Task HandlerCanInspectFailingFramesAndLocals(bool asynchronous, string call) { using var state = CreateState(); var suspension = new ManualSuspension(); - state.Environment["searcher"] = searcher; + state.Environment["call"] = call; state.Environment["pause"] = new LuaFunction( async (context, _) => { @@ -110,12 +116,16 @@ local function namedFailure() pause() error('fail') end - if searcher then package.searchers={namedFailure} end + if call=='searcher' then package.searchers={namedFailure} end + if call=='loader' then package.preload.test=namedFailure end local ok,message=xpcall(function() - if searcher then require('missing') else namedFailure() end + if call=='searcher' or call=='loader' then require('test') + elseif call=='sort' then table.sort({2,1},namedFailure) + elseif call=='gsub' then string.gsub('a','.',namedFailure) + else namedFailure() end end,function(e) local trace=debug.traceback(e) - if not searcher then assert(trace:find('namedFailure',1,true)) end + if call=='direct' then assert(trace:find('namedFailure',1,true)) end for level=1,10 do local info=debug.getinfo(level,'f') if info and info.func==namedFailure then @@ -238,6 +248,79 @@ public async Task CaughtHostExceptionsDoNotConsumeTheOuterHandler(bool asynchron Assert.That(state.ErrorHandlerIndex, Is.Zero); } + [TestCase(false, "RunAsync")] + [TestCase(true, "RunAsync")] + [TestCase(false, "ExecuteAsync")] + [TestCase(true, "ExecuteAsync")] + [TestCase(false, "ExecuteBufferedAsync")] + [TestCase(true, "ExecuteBufferedAsync")] + [TestCase(false, "DoStringAsync")] + [TestCase(true, "DoStringAsync")] + public async Task CaughtLuaExceptionsDoNotConsumeTheOuterHandler(bool asynchronous, string api) + { + using var state = CreateState(); + var suspension = new ManualSuspension(); + state.Environment["pause"] = new LuaFunction( + async (context, _) => + { + if (asynchronous) + await suspension; + return context.Return(); + } + ); + const string source = "pause();error('caught Lua failure',0)"; + var fail = state.Load(source, "@inner"); + state.Environment["recover"] = new LuaFunction( + async (context, token) => + { + try + { + switch (api) + { + case "RunAsync": + await context.State.RunAsync(fail, token); + break; + case "ExecuteAsync": + await context.State.ExecuteAsync(fail, token); + break; + case "ExecuteBufferedAsync": + await context.State.ExecuteAsync(fail, Memory.Empty, token); + break; + case "DoStringAsync": + await context.State.DoStringAsync(source, cancellationToken: token); + break; + } + } + catch (LuaRuntimeException) { } + return context.Return(); + } + ); + var operation = state + .DoStringAsync( + """ + local seen={} + local handler=function(e) seen[#seen+1]=e;return e end + local ok,value=xpcall(function() recover();return 42 end,handler) + assert(ok and value==42 and #seen==0) + ok,value=xpcall(function() recover();error('real failure',0) end,handler) + return ok,value,#seen,seen[1] + """ + ) + .AsTask(); + if (asynchronous) + { + suspension.Resume(); + suspension.Resume(); + } + Assert.That( + await operation, + Is.EqualTo(new LuaValue[] { false, "real failure", 1, "real failure" }) + ); + Assert.That(state.ErrorHandlerIndex, Is.Zero); + Assert.That(state.Stack.Count, Is.Zero); + Assert.That(state.CallStackFrameCount, Is.Zero); + } + [Test] public async Task HostErrorHandlerCanSuspendWithoutLosingTheErrorOrFrames() { diff --git a/tests/Lua.Tests/StandardLibraryContractTests.cs b/tests/Lua.Tests/StandardLibraryContractTests.cs index a2839e01..5aa3414a 100644 --- a/tests/Lua.Tests/StandardLibraryContractTests.cs +++ b/tests/Lua.Tests/StandardLibraryContractTests.cs @@ -127,6 +127,65 @@ public async Task PrintResolvesTostringThroughTheGlobalMetatable(bool asynchrono Assert.That(output, Is.EqualTo(new[] { "2\t4\n" })); } + [TestCase("lookup")] + [TestCase("function")] + [TestCase("metamethod")] + public async Task PrintCannotYieldAndRestoresYieldability(string callback) + { + using var state = CreateState(); + state.Environment["callback"] = callback; + await state.DoStringAsync( + """ + local value=42 + if callback=='lookup' then + tostring=nil + setmetatable(_G,{__index=function(t,key) + assert(key=='tostring') + coroutine.yield('lookup') + end}) + elseif callback=='function' then + tostring=function() coroutine.yield('tostring') end + else + value=setmetatable({},{__tostring=function() coroutine.yield('metamethod') end}) + end + local co=coroutine.create(function() + local ok,e=pcall(print,value) + assert(not ok and e:find('attempt to yield across a C-call boundary',1,true)) + coroutine.yield('after print') + return 42 + end) + local ok,value=coroutine.resume(co) + assert(ok and value=='after print') + ok,value=coroutine.resume(co) + assert(ok and value==42 and coroutine.status(co)=='dead') + """ + ); + Assert.That(state.Stack.Count, Is.Zero); + Assert.That(state.CallStackFrameCount, Is.Zero); + } + + [TestCase("tostring=function() cancel() end;print(42)")] + [TestCase("setmetatable(package.loaded,{__index=function() cancel() end});require('test')")] + public async Task NonYieldableLibraryCallsRestoreStateAfterCancellation(string script) + { + using var state = CreateState(); + using var source = new CancellationTokenSource(); + state.Environment["cancel"] = new LuaFunction( + (context, token) => + { + source.Cancel(); + token.ThrowIfCancellationRequested(); + return new(context.Return()); + } + ); + Assert.ThrowsAsync(async () => + await state.DoStringAsync(script, cancellationToken: source.Token) + ); + Assert.That(state.NonYieldableCallCount, Is.Zero); + Assert.That(state.Stack.Count, Is.Zero); + Assert.That(state.CallStackFrameCount, Is.Zero); + } + [Test] public async Task FileLinesRetainsEveryReadFormat() { From e3773b3709ff6d96e8272129a1c61a8c4155efc3 Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Fri, 30 Oct 2026 20:09:42 +0900 Subject: [PATCH 07/16] fix: preserve error boundaries across metamethod callbacks Keep an enclosing xpcall handler pending while C# callers can catch errors from public indexing, assignment, arithmetic, comparison, and concatenation APIs. Apply the host boundary in the shared callback helpers, leaving the primitive fast paths and VM instruction dispatch unchanged. Keep internal module table access and default sort comparisons on the Lua call path. Dispatch active handlers before shared callback helpers remove failing managed frames, including concatenation callbacks. Cancellation still escapes without invoking a Lua handler. Add synchronous and suspended regressions for every affected public operation, caught and propagated failures, and handler frame inspection through module caches, print, sorting, gsub, and concatenation. Validation: Release build across all targets, full .NET 8/.NET 10 test suite, native Lua 5.2 comparisons, formatting, and native interoperability checks. Paired benchmarks add no steady-state allocations. Primitive operations, numeric loops, cached require, and sorting show no slowdown; host setter callbacks cost approximately 3-4 ns more per call. --- src/Lua/LuaStateExtensions.cs | 32 ++- src/Lua/Runtime/LuaVirtualMachine.cs | 76 +++++- src/Lua/Standard/ModuleLibrary.cs | 12 +- src/Lua/Standard/TableLibrary.cs | 2 +- .../Lua.Tests/MetamethodProtectedCallTests.cs | 236 ++++++++++++++++++ 5 files changed, 336 insertions(+), 22 deletions(-) create mode 100644 tests/Lua.Tests/MetamethodProtectedCallTests.cs diff --git a/src/Lua/LuaStateExtensions.cs b/src/Lua/LuaStateExtensions.cs index 7448c49e..bbccff74 100644 --- a/src/Lua/LuaStateExtensions.cs +++ b/src/Lua/LuaStateExtensions.cs @@ -392,6 +392,14 @@ public static ValueTask LessThanAsync( LuaValue x, LuaValue y, CancellationToken cancellationToken = default + ) => LessThanCoreAsync(state, x, y, cancellationToken, isHostCall: true); + + internal static ValueTask LessThanCoreAsync( + this LuaState state, + LuaValue x, + LuaValue y, + CancellationToken cancellationToken, + bool isHostCall = false ) { if (x.TryReadNumber(out var numX) && y.TryReadNumber(out var numY)) @@ -410,7 +418,8 @@ public static ValueTask LessThanAsync( x, y, OpCode.Lt, - cancellationToken + cancellationToken, + isHostCall ); } @@ -437,7 +446,8 @@ public static ValueTask LessThanOrEqualsAsync( x, y, OpCode.Le, - cancellationToken + cancellationToken, + isHostCall: true ); } @@ -458,7 +468,8 @@ public static ValueTask EqualsAsync( x, y, OpCode.Eq, - cancellationToken + cancellationToken, + isHostCall: true ); } @@ -482,7 +493,8 @@ public static async ValueTask GetTableAsync( state, table, key, - cancellationToken + cancellationToken, + isHostCall: true ); } @@ -492,6 +504,15 @@ public static ValueTask SetTableAsync( LuaValue key, LuaValue value, CancellationToken cancellationToken = default + ) => SetTableCoreAsync(state, table, key, value, cancellationToken, isHostCall: true); + + internal static ValueTask SetTableCoreAsync( + this LuaState state, + LuaValue table, + LuaValue key, + LuaValue value, + CancellationToken cancellationToken, + bool isHostCall = false ) { if (key.TryReadNumber(out var numB)) @@ -517,7 +538,8 @@ public static ValueTask SetTableAsync( table, key, value, - cancellationToken + cancellationToken, + isHostCall ); } diff --git a/src/Lua/Runtime/LuaVirtualMachine.cs b/src/Lua/Runtime/LuaVirtualMachine.cs index 652cae7a..85dee537 100644 --- a/src/Lua/Runtime/LuaVirtualMachine.cs +++ b/src/Lua/Runtime/LuaVirtualMachine.cs @@ -1428,6 +1428,11 @@ OpCode opCode context.ThrowIfCancellationRequested(); return; } + catch (Exception exception) + when (state.ErrorHandlerIndex != 0 && exception is not OperationCanceledException) + { + throw await state.HandleErrorAsync(exception, context.CancellationToken); + } finally { state.PopCallStackFrame(); @@ -2104,7 +2109,8 @@ internal static ValueTask ExecuteGetTableSlowPath( LuaState state, LuaValue table, LuaValue key, - CancellationToken ct + CancellationToken ct, + bool isHostCall = false ) { const int MAX_LOOP = 100; @@ -2145,7 +2151,7 @@ out var metatableValue Function: if (table.TryReadFunction(out var function)) { - return CallGetTableFunc(state, function, targetTable, key, ct); + return CallGetTableFunc(state, function, targetTable, key, ct, isHostCall); } } @@ -2159,12 +2165,16 @@ static async ValueTask CallGetTableFunc( LuaFunction indexTable, LuaValue table, LuaValue key, - CancellationToken ct + CancellationToken ct, + bool isHostCall ) { var stack = state.Stack; var top = stack.Count; var framePushed = false; + var previousHandler = isHostCall ? state.ErrorHandlerIndex : 0; + if (previousHandler != 0) + state.ErrorHandlerIndex = 0; try { stack.Push(table); @@ -2194,8 +2204,15 @@ CancellationToken ct var results = stack.AsSpan()[top..]; return results.Length == 0 ? default : results[0]; } + catch (Exception exception) + when (state.ErrorHandlerIndex != 0 && exception is not OperationCanceledException) + { + throw await state.HandleErrorAsync(exception, ct); + } finally { + if (previousHandler != 0) + state.ErrorHandlerIndex = previousHandler; if (framePushed) state.PopCallStackFrameWithStackPop(top); else @@ -2361,7 +2378,8 @@ internal static ValueTask ExecuteSetTableSlowPath( LuaValue table, LuaValue key, LuaValue value, - CancellationToken ct + CancellationToken ct, + bool isHostCall = false ) { const int MAX_LOOP = 100; @@ -2407,7 +2425,7 @@ out var metatableValue Function: if (table.TryReadFunction(out var function)) { - return CallSetTableFunc(state, function, targetTable, key, value, ct); + return CallSetTableFunc(state, function, targetTable, key, value, ct, isHostCall); } } @@ -2422,12 +2440,16 @@ static async ValueTask CallSetTableFunc( LuaValue table, LuaValue key, LuaValue value, - CancellationToken ct + CancellationToken ct, + bool isHostCall ) { var stack = state.Stack; var top = stack.Count; var framePushed = false; + var previousHandler = isHostCall ? state.ErrorHandlerIndex : 0; + if (previousHandler != 0) + state.ErrorHandlerIndex = 0; try { stack.Push(table); @@ -2460,8 +2482,15 @@ CancellationToken ct await newIndexFunction.Func(functionContext, ct); } } + catch (Exception exception) + when (state.ErrorHandlerIndex != 0 && exception is not OperationCanceledException) + { + throw await state.HandleErrorAsync(exception, ct); + } finally { + if (previousHandler != 0) + state.ErrorHandlerIndex = previousHandler; if (framePushed) state.PopCallStackFrameWithStackPop(top); else @@ -2570,7 +2599,7 @@ CancellationToken ct || vc.TryGetMetamethod(state.GlobalState, name, out metamethod) ) { - return InvokeOperationMetamethod(state, metamethod, vb, vc, ct); + return InvokeOperationMetamethod(state, metamethod, vb, vc, ct, isHostCall: true); } LuaRuntimeException.AttemptInvalidOperation(state, description, vb, vc); @@ -2686,7 +2715,14 @@ CancellationToken cancellationToken if (vb.TryGetMetamethod(state.GlobalState, name, out var metamethod)) { - return InvokeOperationMetamethod(state, metamethod, vb, vb, cancellationToken); + return InvokeOperationMetamethod( + state, + metamethod, + vb, + vb, + cancellationToken, + isHostCall: true + ); } if (opCode == OpCode.Len && vb.TryReadTable(out var table)) @@ -2737,12 +2773,13 @@ static ValueTask InvokeOperationMetamethod( LuaValue metamethod, LuaValue left, LuaValue right, - CancellationToken cancellationToken + CancellationToken cancellationToken, + bool isHostCall ) { var top = state.Stack.Count; var function = PrepareOperationMetamethod(state, metamethod, left, right); - return InvokeOperationMetamethod(state, function, top, cancellationToken); + return InvokeOperationMetamethod(state, function, top, cancellationToken, isHostCall); } [AsyncMethodBuilder(typeof(LightAsyncValueTaskMethodBuilder<>))] @@ -2750,11 +2787,15 @@ static async ValueTask InvokeOperationMetamethod( LuaState state, LuaFunction function, int top, - CancellationToken cancellationToken + CancellationToken cancellationToken, + bool isHostCall ) { var stack = state.Stack; var framePushed = false; + var previousHandler = isHostCall ? state.ErrorHandlerIndex : 0; + if (previousHandler != 0) + state.ErrorHandlerIndex = 0; try { state.ThrowIfCancellationRequested(cancellationToken); @@ -2785,8 +2826,15 @@ CancellationToken cancellationToken state.ThrowIfCancellationRequested(cancellationToken); return stack.Count > top ? stack.Get(top) : default; } + catch (Exception exception) + when (state.ErrorHandlerIndex != 0 && exception is not OperationCanceledException) + { + throw await state.HandleErrorAsync(exception, cancellationToken); + } finally { + if (previousHandler != 0) + state.ErrorHandlerIndex = previousHandler; if (framePushed) state.PopCallStackFrameWithStackPop(top); else @@ -2934,7 +2982,8 @@ internal static async ValueTask ExecuteCompareOperationMetaMethod( LuaValue vb, LuaValue vc, OpCode opCode, - CancellationToken cancellationToken + CancellationToken cancellationToken, + bool isHostCall ) { state.ThrowIfCancellationRequested(cancellationToken); @@ -2949,7 +2998,8 @@ CancellationToken cancellationToken metamethod, vb, vc, - cancellationToken + cancellationToken, + isHostCall ); return reverseLe ? !result.ToBoolean() : result.ToBoolean(); } diff --git a/src/Lua/Standard/ModuleLibrary.cs b/src/Lua/Standard/ModuleLibrary.cs index f156fde5..f3d8f093 100644 --- a/src/Lua/Standard/ModuleLibrary.cs +++ b/src/Lua/Standard/ModuleLibrary.cs @@ -82,7 +82,7 @@ CancellationToken cancellationToken cancellationToken ); if (count > 0 && state.Stack.Get(context.ReturnFrameBase).Type != LuaValueType.Nil) - await state.SetTableAsync( + await state.SetTableCoreAsync( loaded, arg0, state.Stack.Get(context.ReturnFrameBase), @@ -98,7 +98,7 @@ await state.SetTableAsync( if (loadedTable.Type == LuaValueType.Nil) { loadedTable = true; - await state.SetTableAsync(loaded, arg0, loadedTable, cancellationToken); + await state.SetTableCoreAsync(loaded, arg0, loadedTable, cancellationToken); } } return context.Return(loadedTable); @@ -124,7 +124,13 @@ static LuaValue GetPackage(LuaFunctionExecutionContext context) => CancellationToken cancellationToken ) { - var p = await state.GetTableAsync(package, pName, cancellationToken); + if (!package.TryReadTable(out var table) || !table.TryGetValue(pName, out var p)) + p = await LuaVirtualMachine.ExecuteGetTableSlowPath( + state, + package, + pName, + cancellationToken + ); if (p.Type is not (LuaValueType.String or LuaValueType.Number)) { throw new LuaRuntimeException(state, $"package.{pName} must be a string"); diff --git a/src/Lua/Standard/TableLibrary.cs b/src/Lua/Standard/TableLibrary.cs index b9fbe482..123d89ae 100644 --- a/src/Lua/Standard/TableLibrary.cs +++ b/src/Lua/Standard/TableLibrary.cs @@ -422,7 +422,7 @@ CancellationToken cancellationToken { if (comparer == null) { - return state.LessThanAsync(left, right, cancellationToken); + return state.LessThanCoreAsync(left, right, cancellationToken); } var stack = state.Stack; diff --git a/tests/Lua.Tests/MetamethodProtectedCallTests.cs b/tests/Lua.Tests/MetamethodProtectedCallTests.cs new file mode 100644 index 00000000..6b6764b4 --- /dev/null +++ b/tests/Lua.Tests/MetamethodProtectedCallTests.cs @@ -0,0 +1,236 @@ +using Lua.Platforms; +using Lua.Standard; + +namespace Lua.Tests; + +[TestFixture(LuaStringMode.Utf16)] +[TestFixture(LuaStringMode.Bytes)] +public class MetamethodProtectedCallTests(LuaStringMode mode) +{ + LuaState CreateState() + { + var state = LuaState.Create( + mode == LuaStringMode.Bytes ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default + ); + state.OpenStandardLibraries(); + return state; + } + + static readonly string[] HostOperations = + [ + "GetTable", + "SetTable", + "Add", + "Sub", + "Mul", + "Div", + "Mod", + "Pow", + "Unm", + "Len", + "LessThan", + "LessThanOrEquals", + "Equals", + "Concat", + "ConcatStack", + ]; + + [Test, Combinatorial] + public async Task HostCallsDoNotHandleCaughtMetamethodErrors( + [ValueSource(nameof(HostOperations))] string operation, + [Values(false, true)] bool asynchronous + ) + { + using var state = CreateState(); + var suspension = new ManualSuspension(); + state.Environment["pause"] = new LuaFunction( + async (context, _) => + { + if (asynchronous) + await suspension; + return context.Return(); + } + ); + var fail = state.Load("pause();error('caught metamethod failure',0)", "@metamethod"); + var meta = new LuaTable(); + foreach ( + var name in new[] + { + "__index", + "__newindex", + "__add", + "__sub", + "__mul", + "__div", + "__mod", + "__pow", + "__unm", + "__len", + "__lt", + "__le", + "__eq", + "__concat", + } + ) + meta[name] = fail; + var left = new LuaTable { Metatable = meta }; + var right = new LuaTable { Metatable = meta }; + state.Environment["recover"] = new LuaFunction( + async (context, token) => + { + try + { + await InvokeAsync(context.State, operation, left, right, token); + } + catch (LuaRuntimeException) when (context.ArgumentCount == 0) { } + return context.Return(); + } + ); + var task = state + .DoStringAsync( + """ + local seen={} + local handler=function(e) seen[#seen+1]=e;return e end + local ok,value=xpcall(function() recover();return 42 end,handler) + assert(ok and value==42 and #seen==0) + ok,value=xpcall(function() recover();error('real failure',0) end,handler) + local propagated,message=xpcall(function() recover(true) end,handler) + assert(not propagated and message=='caught metamethod failure') + return ok,value,#seen,seen[1],seen[2] + """ + ) + .AsTask(); + if (asynchronous) + { + suspension.Resume(); + suspension.Resume(); + suspension.Resume(); + } + Assert.That( + await task, + Is.EqualTo( + new LuaValue[] + { + false, + "real failure", + 2, + "real failure", + "caught metamethod failure", + } + ) + ); + Assert.That(state.ErrorHandlerIndex, Is.Zero); + Assert.That(state.Stack.Count, Is.Zero); + Assert.That(state.CallStackFrameCount, Is.Zero); + } + + static async ValueTask InvokeAsync( + LuaState state, + string operation, + LuaValue left, + LuaValue right, + CancellationToken token + ) + { + switch (operation) + { + case "GetTable": + await state.GetTableAsync(left, "missing", token); + break; + case "SetTable": + await state.SetTableAsync(left, "missing", 42, token); + break; + case "Add": + await state.AddAsync(left, right, token); + break; + case "Sub": + await state.SubAsync(left, right, token); + break; + case "Mul": + await state.MulAsync(left, right, token); + break; + case "Div": + await state.DivAsync(left, right, token); + break; + case "Mod": + await state.ModAsync(left, right, token); + break; + case "Pow": + await state.PowAsync(left, right, token); + break; + case "Unm": + await state.UnmAsync(left, token); + break; + case "Len": + await state.LenAsync(left, token); + break; + case "LessThan": + await state.LessThanAsync(left, right, token); + break; + case "LessThanOrEquals": + await state.LessThanOrEqualsAsync(left, right, token); + break; + case "Equals": + await state.EqualsAsync(left, right, token); + break; + case "Concat": + await state.ConcatAsync([left, right], token); + break; + case "ConcatStack": + state.Stack.PushRange(left, right); + await state.ConcatAsync(2, token); + break; + } + } + + [Test, Combinatorial] + public async Task LuaHandlersCanInspectFailingManagedMetamethodFrames( + [Values("index", "newindex", "print", "sort", "gsub", "concat")] string site, + [Values(false, true)] bool asynchronous + ) + { + using var state = CreateState(); + var suspension = new ManualSuspension(); + state.Environment["fail"] = asynchronous + ? new LuaFunction( + async (context, _) => + { + await suspension; + throw new LuaRuntimeException(context.State, "managed failure", 0); + } + ) + : state.Environment["error"]; + var source = site switch + { + "index" => "setmetatable(package.loaded,{__index=fail});require('test')", + "newindex" => + "package.preload.test=function()return 42 end;setmetatable(package.loaded,{__newindex=fail});require('test')", + "print" => "tostring=nil;setmetatable(_G,{__index=fail});print(42)", + "sort" => + "local meta={__lt=fail};table.sort({setmetatable({},meta),setmetatable({},meta)})", + "gsub" => "string.gsub('a','.',setmetatable({},{__index=fail}))", + _ => + "local meta={__concat=fail};local a,b=setmetatable({},meta),setmetatable({},meta);return a..b", + }; + state.Environment["operation"] = state.Load(source, "@operation"); + var task = state + .DoStringAsync( + """ + return xpcall(operation,function(e) + for level=1,20 do + local info=debug.getinfo(level,'f') + if info and info.func==fail then return true end + end + return false + end) + """ + ) + .AsTask(); + if (asynchronous) + suspension.Resume(); + Assert.That(await task, Is.EqualTo(new LuaValue[] { false, true })); + Assert.That(state.ErrorHandlerIndex, Is.Zero); + Assert.That(state.Stack.Count, Is.Zero); + Assert.That(state.CallStackFrameCount, Is.Zero); + } +} From 14ea2c2c718680d9ce2d2a18d8ceced44bc72c8d Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Fri, 30 Oct 2026 20:09:42 +0900 Subject: [PATCH 08/16] fix: preserve missing debug metadata in stripped Lua chunks Decode an omitted bytecode source as Lua 5.2's unknown source marker while preserving an actual empty source name. Treat absent line information as unknown in error messages and tracebacks instead of inserting an internal Trace back error diagnostic. Remove console output for missing line data. Add a genuine Lua 5.2.4 luac -s fixture covering pcall error values, xpcall tracebacks, source metadata, redumping, and the distinction between omitted and empty source names in both string modes. Validation: Release build across all targets, focused and full .NET 8/.NET 10 tests, native Lua 5.2 comparisons using the same compiler-produced fixture, native bytecode interoperability checks, and formatting. --- src/Lua/CodeAnalysis/Compilation/Dump.cs | 6 +- src/Lua/Runtime/Tracebacks.cs | 48 +++++++--------- tests/Lua.Tests/StrippedBytecodeTests.cs | 70 ++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 31 deletions(-) create mode 100644 tests/Lua.Tests/StrippedBytecodeTests.cs diff --git a/src/Lua/CodeAnalysis/Compilation/Dump.cs b/src/Lua/CodeAnalysis/Compilation/Dump.cs index 4a22e24e..f8736ae8 100644 --- a/src/Lua/CodeAnalysis/Compilation/Dump.cs +++ b/src/Lua/CodeAnalysis/Compilation/Dump.cs @@ -434,7 +434,7 @@ Prototype UndumpFunction(int depth) var upValues = ReadUpValues(); // Debug - var source = ReadString(); + var source = ReadString("=?"); var lineInfoLength = ReadCount(sizeof(int)); if (lineInfoLength != 0 && lineInfoLength != codeLength) Throw("invalid line info in"); @@ -480,12 +480,12 @@ void ReadInToIntSpan(Span toWrite) } } - string ReadString() + string ReadString(string nullValue = "") { var serializedLength = pointerSize == 4 ? ReadInt() : ReadLong(); if (serializedLength == 0) { - return ""; + return nullValue; } if (serializedLength < 0 || serializedLength > Unread.Length) diff --git a/src/Lua/Runtime/Tracebacks.cs b/src/Lua/Runtime/Tracebacks.cs index c3934d0e..f36ce96c 100644 --- a/src/Lua/Runtime/Tracebacks.cs +++ b/src/Lua/Runtime/Tracebacks.cs @@ -33,23 +33,18 @@ internal static void WriteLastLuaTrace( if (!frame.IsTailCall && lastFunc is LuaClosure closure) { var p = closure.Proto; + if ((uint)frame.CallerInstructionIndex >= (uint)p.LineInfo.Length) + return; var len = LuaDebug.WriteShortSource(p.ChunkName, shortSourceBuffer); list.AddRange(shortSourceBuffer[..len]); list.AddRange(":"); - if (p.LineInfo.Length <= frame.CallerInstructionIndex) - { - list.AddRange("Trace back error"); - } - else - { - p.LineInfo[frame.CallerInstructionIndex] - .TryFormat( - intFormatBuffer, - out var charsWritten, - provider: CultureInfo.InvariantCulture - ); - list.AddRange(intFormatBuffer[..charsWritten]); - } + p.LineInfo[frame.CallerInstructionIndex] + .TryFormat( + intFormatBuffer, + out var charsWritten, + provider: CultureInfo.InvariantCulture + ); + list.AddRange(intFormatBuffer[..charsWritten]); list.AddRange(": "); return; @@ -80,7 +75,6 @@ public int LastLine || p.LineInfo.Length <= frame.CallerInstructionIndex ) { - Console.WriteLine($"Trace back error"); return default; } @@ -110,7 +104,6 @@ public int FirstLine || p.LineInfo.Length <= frame.CallerInstructionIndex ) { - Console.WriteLine($"Trace back error"); return default; } @@ -201,19 +194,18 @@ internal static string CreateTracebackMessage( list.AddRange("\t"); var len = LuaDebug.WriteShortSource(p.ChunkName, shortSourceBuffer); list.AddRange(shortSourceBuffer[..len]); - list.AddRange(":"); - if (p.LineInfo.Length <= frame.CallerInstructionIndex) - { - list.AddRange("Trace back error"); - } - else + var line = + (uint)frame.CallerInstructionIndex < (uint)p.LineInfo.Length + ? p.LineInfo[frame.CallerInstructionIndex] + : 0; + if (line > 0) { - p.LineInfo[frame.CallerInstructionIndex] - .TryFormat( - intFormatBuffer, - out var charsWritten, - provider: CultureInfo.InvariantCulture - ); + list.AddRange(":"); + line.TryFormat( + intFormatBuffer, + out var charsWritten, + provider: CultureInfo.InvariantCulture + ); list.AddRange(intFormatBuffer[..charsWritten]); } diff --git a/tests/Lua.Tests/StrippedBytecodeTests.cs b/tests/Lua.Tests/StrippedBytecodeTests.cs new file mode 100644 index 00000000..7f89ae29 --- /dev/null +++ b/tests/Lua.Tests/StrippedBytecodeTests.cs @@ -0,0 +1,70 @@ +using Lua.Platforms; +using Lua.Standard; + +namespace Lua.Tests; + +[TestFixture(LuaStringMode.Utf16)] +[TestFixture(LuaStringMode.Bytes)] +public class StrippedBytecodeTests(LuaStringMode mode) +{ + // Lua 5.2.4 luac -s, 64-bit size_t: + // local value=42;return function()if value==42 then error('stripped failure') end;return value end + static readonly byte[] Bytecode = Convert.FromBase64String( + "G0x1YVIAAQQIBAgAGZMNChoKAAAAAAAAAAAAAQIEAAAAAQAAAGUAAABfAAABHwCAAAEAAAADAAAAAAAARUABAAAAAQAAAAEAAAAAAAIJAAAABQAAABgAQAAXgACABkDAAEGAAAAdQAABBQAAAB8AAAEfAIAAAwAAAAMAAAAAAABFQAQGAAAAAAAAAGVycm9yAAQRAAAAAAAAAHN0cmlwcGVkIGZhaWx1cmUAAAAAAAIAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + ); + + LuaState CreateState() + { + var state = LuaState.Create( + mode == LuaStringMode.Bytes ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default + ); + state.OpenStandardLibraries(); + state.Environment["binary"] = LuaByteString.FromBytes(Bytecode); + return state; + } + + [TestCase(false)] + [TestCase(true)] + public async Task MissingDebugMetadataPreservesErrorsAndUnknownSource(bool redump) + { + using var state = CreateState(); + state.Environment["redump"] = redump; + var result = await state.DoStringAsync( + """ + local f=assert(load(binary,'@supplied','b'))() + if redump then + f=assert(load(string.dump(f))) + debug.setupvalue(f,1,42) + debug.setupvalue(f,2,_G) + end + local info=debug.getinfo(f,'S') + assert(info.source=='=?' and info.short_src=='?') + local ok,message=pcall(f) + assert(not ok and message=='stripped failure') + return xpcall(f,function(e) + local trace=debug.traceback(e) + assert(not trace:find('Trace back error',1,true)) + assert(trace:find('?:',1,true)) + return e + end) + """ + ); + Assert.That(result, Is.EqualTo(new LuaValue[] { false, "stripped failure" })); + Assert.That(state.Stack.Count, Is.Zero); + Assert.That(state.CallStackFrameCount, Is.Zero); + } + + [Test] + public async Task AnEmptySourceNameRemainsDifferentFromMissingSource() + { + using var state = CreateState(); + await state.DoStringAsync( + """ + local f=assert(load('return 42','')) + f=assert(load(string.dump(f))) + local info=debug.getinfo(f,'S') + assert(info.source=='' and info.short_src=='[string ""]' and f()==42) + """ + ); + } +} From 868a7fb46d2fdbd76799e66eb540d6b43c68d388 Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Fri, 30 Oct 2026 20:09:42 +0900 Subject: [PATCH 09/16] fix: expose managed closure upvalues consistently to debug APIs Report CSharpClosure upvalue counts through debug.getinfo and provide stable slot identities through debug.upvalueid. Key identities by the backing upvalue array in a ConditionalWeakTable so equal values in different slots remain distinct and retained IDs do not keep captured values alive. Add regressions for metadata, repeated identity queries, distinct slots and closures, setupvalue preserving identity, standard package closures, and collection of discarded managed captures while an ID remains retained. Validation: 3,023 runtime tests on each of .NET 8 and .NET 10, 73 generator tests, Release build across all targets, native Lua 5.2 behavior comparisons, native interoperability checks, and formatting. --- src/Lua/Internal/LuaDebug.cs | 5 +- src/Lua/Standard/DebugLibrary.cs | 15 ++++++ tests/Lua.Tests/LuaIdentityLifetimeTests.cs | 58 +++++++++++++++++++++ 3 files changed, 77 insertions(+), 1 deletion(-) diff --git a/src/Lua/Internal/LuaDebug.cs b/src/Lua/Internal/LuaDebug.cs index 19dadd5b..976e6284 100644 --- a/src/Lua/Internal/LuaDebug.cs +++ b/src/Lua/Internal/LuaDebug.cs @@ -244,7 +244,10 @@ ReadOnlySpan what } case 'u': { - UpValueCount = closure is null ? 0 : closure.UpValues.Length; + UpValueCount = + closure is not null ? closure.UpValues.Length + : function is CSharpClosure csClosure ? csClosure.UpValues.Length + : 0; if (closure is null) { IsVarArg = true; diff --git a/src/Lua/Standard/DebugLibrary.cs b/src/Lua/Standard/DebugLibrary.cs index 38939d27..3ad8d652 100644 --- a/src/Lua/Standard/DebugLibrary.cs +++ b/src/Lua/Standard/DebugLibrary.cs @@ -9,6 +9,7 @@ public sealed class DebugLibrary { public static readonly DebugLibrary Instance = new(); static readonly ConditionalWeakTable upValueIds = new(); + static readonly ConditionalWeakTable csUpValueIds = new(); public DebugLibrary() { @@ -429,6 +430,20 @@ CancellationToken cancellationToken if (f1 is not LuaClosure closure) { + if (f1 is CSharpClosure csClosure && n1 > 0 && n1 <= csClosure.UpValues.Length) + { + var identities = csUpValueIds.GetValue( + csClosure.UpValues, + static values => + { + var result = new object[values.Length]; + for (var i = 0; i < result.Length; i++) + result[i] = new object(); + return result; + } + ); + return new(context.Return(LuaValue.FromObject(identities[n1 - 1]))); + } return new(context.Return(LuaValue.Nil)); } diff --git a/tests/Lua.Tests/LuaIdentityLifetimeTests.cs b/tests/Lua.Tests/LuaIdentityLifetimeTests.cs index cba602d3..0017f26c 100644 --- a/tests/Lua.Tests/LuaIdentityLifetimeTests.cs +++ b/tests/Lua.Tests/LuaIdentityLifetimeTests.cs @@ -1,5 +1,6 @@ using System.Runtime.CompilerServices; using Lua.Platforms; +using Lua.Runtime; using Lua.Standard; namespace Lua.Tests; @@ -152,6 +153,63 @@ public async Task UpValueIdsRemainStableForSharedAndJoinedCaptures() Assert.That(result, Is.EqualTo(new LuaValue[] { true, true, true, true, 3, 3, 3 })); } + [Test] + public async Task ManagedClosureUpValuesHaveStableSlotIdentitiesAndMetadata() + { + using var state = CreateState(); + state.Environment["first"] = new CSharpClosure( + "first", + [42, 42], + static (context, _) => new(context.Return()) + ); + state.Environment["second"] = new CSharpClosure( + "second", + [42], + static (context, _) => new(context.Return()) + ); + await state.DoStringAsync( + """ + assert(debug.getinfo(first,'u').nups==2 and debug.getinfo(second,'u').nups==1) + assert(debug.getinfo(require,'u').nups==1) + local id=debug.upvalueid(first,1) + assert(type(id)=='userdata' and id==debug.upvalueid(first,1)) + assert(id~=debug.upvalueid(first,2) and id~=debug.upvalueid(second,1)) + assert(debug.setupvalue(first,1,43)=='') + assert(id==debug.upvalueid(first,1)) + local name,value=debug.getupvalue(first,1) + assert(name=='' and value==43) + assert(debug.upvalueid(require,1)~=debug.upvalueid(package.searchers[2],1)) + """ + ); + } + + [Test] + public void ManagedUpValueIdsDoNotKeepDiscardedCapturesAlive() + { + using var state = CreateState(); + var payload = SaveManagedCaptureId(state); + Collect(); + Assert.That(payload.IsAlive, Is.False); + Assert.That(state.Environment["identity"].Type, Is.EqualTo(LuaValueType.LightUserData)); + GC.KeepAlive(state); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static WeakReference SaveManagedCaptureId(LuaState state) + { + var payload = new object(); + state.Environment["capture"] = new CSharpClosure( + "capture", + [LuaValue.FromObject(payload)], + static (context, _) => new(context.Return()) + ); + state + .DoStringAsync("identity=debug.upvalueid(capture,1);capture=nil") + .GetAwaiter() + .GetResult(); + return new WeakReference(payload); + } + static LuaValue KeyValue(MutableKey key, bool fullUserData) => fullUserData ? LuaValue.FromUserData(key) : LuaValue.FromObject(key); From 0a1ea6aeb971fdb62af21f7a4570a74006603a66 Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Fri, 30 Oct 2026 20:42:41 +0900 Subject: [PATCH 10/16] fix: enforce Lua 5.2 callback and upvalue contracts Keep table-library length metamethods inside the active Lua error boundary so xpcall handlers can inspect and modify failing frames. Require actual functions as xpcall error handlers rather than dispatching __call. Reject Lua yields across the remaining standard-library callback boundaries while preserving C# awaits and restoring state after cancellation. Scope the sort guard around the sort operation rather than each comparison. Share debug upvalue-index validation and reject managed closures in upvaluejoin with Lua 5.2 argument errors. Retain weak upvalue identities. Add regressions for all affected callbacks, table length error frames, handler validation and invalid upvalue requests. Validation: Release build (all targets), 3,123 runtime tests per net8.0 and net10.0, 73 generator tests, formatting, native Lua 5.2.4 repro comparisons, and bytecode interoperability verification. Paired benchmarks show unchanged allocations and timing changes within 2.2% across the measured workloads. --- src/Lua/LuaState.cs | 6 + src/Lua/LuaStateExtensions.cs | 6 +- src/Lua/LuaValue.cs | 5 + src/Lua/Runtime/LuaVirtualMachine.cs | 5 +- src/Lua/Standard/BasicLibrary.cs | 40 ++- src/Lua/Standard/DebugLibrary.cs | 93 +++--- src/Lua/Standard/StringLibrary.cs | 274 +++++++++--------- src/Lua/Standard/TableLibrary.cs | 39 ++- tests/Lua.Tests/LuaIdentityLifetimeTests.cs | 29 ++ tests/Lua.Tests/ProtectedCallTests.cs | 21 +- .../Lua.Tests/StandardLibraryCallbackTests.cs | 94 ++++++ 11 files changed, 408 insertions(+), 204 deletions(-) create mode 100644 tests/Lua.Tests/StandardLibraryCallbackTests.cs diff --git a/src/Lua/LuaState.cs b/src/Lua/LuaState.cs index 9b132047..7a8e2295 100644 --- a/src/Lua/LuaState.cs +++ b/src/Lua/LuaState.cs @@ -689,6 +689,12 @@ CancellationToken cancellationToken else error = Platform.GetErrorMessage(exception); + if (handler.Type != LuaValueType.Function) + { + Stack.Get(handlerIndex) = "error in error handling"; + return exception; + } + var top = Stack.Count; NonYieldableCallCount++; try diff --git a/src/Lua/LuaStateExtensions.cs b/src/Lua/LuaStateExtensions.cs index bbccff74..0f438050 100644 --- a/src/Lua/LuaStateExtensions.cs +++ b/src/Lua/LuaStateExtensions.cs @@ -359,7 +359,8 @@ public static ValueTask UnmAsync( state, value, OpCode.Unm, - cancellationToken + cancellationToken, + isHostCall: true ); } @@ -383,7 +384,8 @@ public static ValueTask LenAsync( state, value, OpCode.Len, - cancellationToken + cancellationToken, + isHostCall: true ); } diff --git a/src/Lua/LuaValue.cs b/src/Lua/LuaValue.cs index 23df1d03..0d5ef397 100644 --- a/src/Lua/LuaValue.cs +++ b/src/Lua/LuaValue.cs @@ -788,6 +788,7 @@ CancellationToken cancellationToken { var stack = state.Stack; var top = stack.Count; + state.NonYieldableCallCount++; try { stack.Push(metamethod); @@ -807,6 +808,10 @@ CancellationToken cancellationToken stack.PopUntil(top); throw; } + finally + { + state.NonYieldableCallCount--; + } } } } diff --git a/src/Lua/Runtime/LuaVirtualMachine.cs b/src/Lua/Runtime/LuaVirtualMachine.cs index 85dee537..0a2b28ef 100644 --- a/src/Lua/Runtime/LuaVirtualMachine.cs +++ b/src/Lua/Runtime/LuaVirtualMachine.cs @@ -2704,7 +2704,8 @@ internal static ValueTask ExecuteUnaryOperationMetaMethod( LuaState state, LuaValue vb, OpCode opCode, - CancellationToken cancellationToken + CancellationToken cancellationToken, + bool isHostCall ) { try @@ -2721,7 +2722,7 @@ CancellationToken cancellationToken vb, vb, cancellationToken, - isHostCall: true + isHostCall ); } diff --git a/src/Lua/Standard/BasicLibrary.cs b/src/Lua/Standard/BasicLibrary.cs index 2da70a0c..abad4e90 100644 --- a/src/Lua/Standard/BasicLibrary.cs +++ b/src/Lua/Standard/BasicLibrary.cs @@ -201,12 +201,20 @@ CancellationToken cancellationToken stack.Push(metamethod); stack.Push(arg0); - await LuaVirtualMachine.Call( - context.State, - top, - context.ReturnFrameBase, - cancellationToken - ); + context.State.NonYieldableCallCount++; + try + { + await LuaVirtualMachine.Call( + context.State, + top, + context.ReturnFrameBase, + cancellationToken + ); + } + finally + { + context.State.NonYieldableCallCount--; + } stack.SetTop(context.ReturnFrameBase + 3); return 3; } @@ -405,12 +413,20 @@ CancellationToken cancellationToken stack.Push(metamethod); stack.Push(arg0); - await LuaVirtualMachine.Call( - context.State, - top, - context.ReturnFrameBase, - cancellationToken - ); + context.State.NonYieldableCallCount++; + try + { + await LuaVirtualMachine.Call( + context.State, + top, + context.ReturnFrameBase, + cancellationToken + ); + } + finally + { + context.State.NonYieldableCallCount--; + } stack.SetTop(context.ReturnFrameBase + 3); return 3; } diff --git a/src/Lua/Standard/DebugLibrary.cs b/src/Lua/Standard/DebugLibrary.cs index 3ad8d652..13764b88 100644 --- a/src/Lua/Standard/DebugLibrary.cs +++ b/src/Lua/Standard/DebugLibrary.cs @@ -420,41 +420,54 @@ CancellationToken cancellationToken return new(context.Return(context.GlobalState.Registry)); } + static LuaFunction CheckUpValue( + in LuaFunctionExecutionContext context, + int functionIndex, + int upValueIndex, + out int index + ) + { + index = context.GetIntegerArgument(upValueIndex); + var function = context.GetArgument(functionIndex); + var count = function switch + { + LuaClosure closure => closure.GetUpValuesSpan().Length, + CSharpClosure closure => closure.UpValues.Length, + _ => 0, + }; + if (index <= 0 || index > count) + context.ThrowBadArgument(upValueIndex + 1, "invalid upvalue index"); + return function; + } + public ValueTask UpValueId( LuaFunctionExecutionContext context, CancellationToken cancellationToken ) { - var n1 = context.GetIntegerArgument(1); - var f1 = context.GetArgument(0); - - if (f1 is not LuaClosure closure) - { - if (f1 is CSharpClosure csClosure && n1 > 0 && n1 <= csClosure.UpValues.Length) - { - var identities = csUpValueIds.GetValue( - csClosure.UpValues, - static values => - { - var result = new object[values.Length]; - for (var i = 0; i < result.Length; i++) - result[i] = new object(); - return result; - } - ); - return new(context.Return(LuaValue.FromObject(identities[n1 - 1]))); - } - return new(context.Return(LuaValue.Nil)); - } + var function = CheckUpValue(context, 0, 1, out var index); - var upValues = closure.GetUpValuesSpan(); - if (n1 <= 0 || n1 > upValues.Length) + if (function is LuaClosure closure) { - return new(context.Return(LuaValue.Nil)); + var identity = upValueIds.GetValue( + closure.GetUpValuesSpan()[index - 1], + static _ => new object() + ); + return new(context.Return(LuaValue.FromObject(identity))); } - var identity = upValueIds.GetValue(upValues[n1 - 1], static _ => new object()); - return new(context.Return(LuaValue.FromObject(identity))); + var csClosure = (CSharpClosure)function; + var identities = csUpValueIds.GetValue( + csClosure.UpValues, + static values => + { + var result = new object[values.Length]; + for (var i = 0; i < result.Length; i++) + result[i] = new object(); + return result; + } + ); + return new(context.Return(LuaValue.FromObject(identities[index - 1]))); } public ValueTask UpValueJoin( @@ -462,28 +475,16 @@ public ValueTask UpValueJoin( CancellationToken cancellationToken ) { - var n2 = context.GetIntegerArgument(3); - var f2 = context.GetArgument(2); - var n1 = context.GetIntegerArgument(1); - var f1 = context.GetArgument(0); - - if (f1 is not LuaClosure closure1 || f2 is not LuaClosure closure2) - { - return new(context.Return(LuaValue.Nil)); - } - - var upValues1 = closure1.GetUpValuesSpan(); - var upValues2 = closure2.GetUpValuesSpan(); - if (n1 <= 0 || n1 > upValues1.Length) - { - context.ThrowBadArgument(2, "invalid upvalue index"); - } + var f1 = CheckUpValue(context, 0, 1, out var n1); + var f2 = CheckUpValue(context, 2, 3, out var n2); - if (n2 <= 0 || n2 > upValues2.Length) - { - context.ThrowBadArgument(4, "invalid upvalue index"); - } + if (f1 is not LuaClosure) + context.ThrowBadArgument(1, "Lua function expected"); + if (f2 is not LuaClosure) + context.ThrowBadArgument(3, "Lua function expected"); + var upValues1 = ((LuaClosure)f1).GetUpValuesSpan(); + var upValues2 = ((LuaClosure)f2).GetUpValuesSpan(); upValues1[n1 - 1] = upValues2[n2 - 1]; return new(context.Return()); } diff --git a/src/Lua/Standard/StringLibrary.cs b/src/Lua/Standard/StringLibrary.cs index 44af181a..e2b9bfea 100644 --- a/src/Lua/Standard/StringLibrary.cs +++ b/src/Lua/Standard/StringLibrary.cs @@ -616,172 +616,180 @@ or LuaValueType.Function if (n == 0) return context.Return(s, 0); - MatchState matchState = new(context.State, s, pattern); - var captures = matchState.Captures; - - StringBuilder builder = new(); - var replacementText = - repl.Type == LuaValueType.String ? repl.UnsafeReadString() - : repl.Type == LuaValueType.Number - ? repl.ToLuaString(context.State.Platform.NumberFormat) - : null; - var replacedBuilder = - replacementText != null && replacementText.IndexOf('%') >= 0 - ? new StringBuilder(replacementText.Length) + context.State.NonYieldableCallCount++; + try + { + MatchState matchState = new(context.State, s, pattern); + var captures = matchState.Captures; + + StringBuilder builder = new(); + var replacementText = + repl.Type == LuaValueType.String ? repl.UnsafeReadString() + : repl.Type == LuaValueType.Number + ? repl.ToLuaString(context.State.Platform.NumberFormat) : null; - var lastIndex = 0; - var replaceCount = 0; + var replacedBuilder = + replacementText != null && replacementText.IndexOf('%') >= 0 + ? new StringBuilder(replacementText.Length) + : null; + var lastIndex = 0; + var replaceCount = 0; + + // Check for anchor at start + var anchor = pattern.Length > 0 && pattern[0] == '^'; + var sIdx = 0; + + // Include the end position so patterns can match an empty string there. + while (sIdx <= s.Length && replaceCount < n) + { + // Reset match state for each attempt + matchState.Level = 0; + Debug.Assert(matchState.MatchDepth == MatchState.MaxCalls); - // Check for anchor at start - var anchor = pattern.Length > 0 && pattern[0] == '^'; - var sIdx = 0; + // Always start pattern from beginning (0 or 1 if anchored) + var pIdx = anchor ? 1 : 0; + var res = matchState.Match(sIdx, pIdx); - // Include the end position so patterns can match an empty string there. - while (sIdx <= s.Length && replaceCount < n) - { - // Reset match state for each attempt - matchState.Level = 0; - Debug.Assert(matchState.MatchDepth == MatchState.MaxCalls); + if (res >= 0) + { + // Found a match + builder.Append(s.AsSpan()[lastIndex..sIdx]); - // Always start pattern from beginning (0 or 1 if anchored) - var pIdx = anchor ? 1 : 0; - var res = matchState.Match(sIdx, pIdx); + // If no captures were made, create one for the whole match + if (matchState.Level == 0) + { + captures[0].Init = sIdx; + captures[0].Len = res - sIdx; + matchState.Level = 1; + } - if (res >= 0) - { - // Found a match - builder.Append(s.AsSpan()[lastIndex..sIdx]); + LuaValue result; + if (replacementText != null) + { + if (replacedBuilder == null) + { + result = replacementText; + } + else + { + // String replacement + replacedBuilder.Clear(); + AppendReplacement( + replacedBuilder, + replacementText, + s, + sIdx, + res, + matchState + ); + result = replacedBuilder.ToString(); + } + } + else if (repl.TryRead(out var table)) + { + var key = matchState.GetCapture(0); - // If no captures were made, create one for the whole match - if (matchState.Level == 0) - { - captures[0].Init = sIdx; - captures[0].Len = res - sIdx; - matchState.Level = 1; - } + if (!table.TryGetValue(key, out result) && table.Metatable != null) + { + result = await LuaVirtualMachine.ExecuteGetTableSlowPath( + context.State, + table, + key, + cancellationToken + ); + } + } + else + { + var func = repl.UnsafeRead(); + // Function call with captures as arguments + var stack = context.State.Stack; - LuaValue result; - if (replacementText != null) - { - if (replacedBuilder == null) + for (var k = 0; k < matchState.Level; k++) + { + stack.Push(matchState.GetCapture(k)); + } + var retCount = await context.State.RunCoreAsync( + func, + matchState.Level, + stack.Count - matchState.Level, + cancellationToken + ); + using var results = context.State.ReadStack(retCount); + result = results.Count > 0 ? results[0] : LuaValue.Nil; + } + // Handle replacement result + if (result.TryRead(out var rs)) + { + builder.Append(rs); + } + else if (result.TryRead(out var rd)) + { + AppendNumber(builder, rd, context.State.Platform.NumberFormat); + } + else if (!result.ToBoolean()) { - result = replacementText; + // False or nil means don't replace + builder.Append(s.AsSpan(sIdx, res - sIdx)); } else { - // String replacement - replacedBuilder.Clear(); - AppendReplacement( - replacedBuilder, - replacementText, - s, - sIdx, - res, - matchState + throw new LuaRuntimeException( + context.State, + $"invalid replacement value (a {result.TypeToString()})" ); - result = replacedBuilder.ToString(); } - } - else if (repl.TryRead(out var table)) - { - var key = matchState.GetCapture(0); - if (!table.TryGetValue(key, out result) && table.Metatable != null) + replaceCount++; + lastIndex = res; + + // If empty match, advance by 1 to avoid infinite loop + if (res == sIdx) { - result = await LuaVirtualMachine.ExecuteGetTableSlowPath( - context.State, - table, - key, - cancellationToken - ); + if (sIdx < s.Length) + { + builder.Append(s[sIdx]); + lastIndex = sIdx + 1; + } + + sIdx++; + } + else + { + sIdx = res; } - } - else - { - var func = repl.UnsafeRead(); - // Function call with captures as arguments - var stack = context.State.Stack; - for (var k = 0; k < matchState.Level; k++) + // Anchored patterns only match once at the start of the string + if (anchor) { - stack.Push(matchState.GetCapture(k)); + break; } - var retCount = await context.State.RunCoreAsync( - func, - matchState.Level, - stack.Count - matchState.Level, - cancellationToken - ); - using var results = context.State.ReadStack(retCount); - result = results.Count > 0 ? results[0] : LuaValue.Nil; - } - // Handle replacement result - if (result.TryRead(out var rs)) - { - builder.Append(rs); - } - else if (result.TryRead(out var rd)) - { - AppendNumber(builder, rd, context.State.Platform.NumberFormat); - } - else if (!result.ToBoolean()) - { - // False or nil means don't replace - builder.Append(s.AsSpan(sIdx, res - sIdx)); } else { - throw new LuaRuntimeException( - context.State, - $"invalid replacement value (a {result.TypeToString()})" - ); - } - - replaceCount++; - lastIndex = res; - - // If empty match, advance by 1 to avoid infinite loop - if (res == sIdx) - { - if (sIdx < s.Length) + // No match at this position + if (anchor) { - builder.Append(s[sIdx]); - lastIndex = sIdx + 1; + // Anchored pattern only tries at start + break; } sIdx++; } - else - { - sIdx = res; - } - - // Anchored patterns only match once at the start of the string - if (anchor) - { - break; - } } - else - { - // No match at this position - if (anchor) - { - // Anchored pattern only tries at start - break; - } - sIdx++; + // Append remaining part of string + if (lastIndex < s.Length) + { + builder.Append(s.AsSpan()[lastIndex..]); } - } - // Append remaining part of string - if (lastIndex < s.Length) + return context.Return(builder.ToString(), replaceCount); + } + finally { - builder.Append(s.AsSpan()[lastIndex..]); + context.State.NonYieldableCallCount--; } - - return context.Return(builder.ToString(), replaceCount); } public ValueTask Len( diff --git a/src/Lua/Standard/TableLibrary.cs b/src/Lua/Standard/TableLibrary.cs index 123d89ae..499958e1 100644 --- a/src/Lua/Standard/TableLibrary.cs +++ b/src/Lua/Standard/TableLibrary.cs @@ -52,12 +52,22 @@ CancellationToken cancellationToken return table.ArrayLength; } - var result = await LuaVirtualMachine.ExecuteUnaryOperationMetaMethod( - state, - value, - OpCode.Len, - cancellationToken - ); + LuaValue result; + state.NonYieldableCallCount++; + try + { + result = await LuaVirtualMachine.ExecuteUnaryOperationMetaMethod( + state, + value, + OpCode.Len, + cancellationToken, + isHostCall: false + ); + } + finally + { + state.NonYieldableCallCount--; + } if (!result.TryReadDouble(out var length)) { throw new LuaRuntimeException(state, "object length is not a number"); @@ -248,7 +258,22 @@ CancellationToken cancellationToken // A __len result can extend beyond the backing array; use raw table keys. if (length > 1) { - await AuxSortAsync(context.State, table, 0, length - 1, comparer, cancellationToken); + context.State.NonYieldableCallCount++; + try + { + await AuxSortAsync( + context.State, + table, + 0, + length - 1, + comparer, + cancellationToken + ); + } + finally + { + context.State.NonYieldableCallCount--; + } } return context.Return(); } diff --git a/tests/Lua.Tests/LuaIdentityLifetimeTests.cs b/tests/Lua.Tests/LuaIdentityLifetimeTests.cs index 0017f26c..e1ede158 100644 --- a/tests/Lua.Tests/LuaIdentityLifetimeTests.cs +++ b/tests/Lua.Tests/LuaIdentityLifetimeTests.cs @@ -183,6 +183,35 @@ await state.DoStringAsync( ); } + [Test] + public async Task UpValueIdentityAndJoinRejectInvalidArguments() + { + using var state = CreateState(); + await state.DoStringAsync( + """ + local value=42 + local function f() return value end + local function checkError(argument,reason,func,...) + local ok,e=pcall(func,...) + assert(not ok and e:find('bad argument #'..argument,1,true) and e:find(reason,1,true),e) + end + for _,func in ipairs({f,require,print}) do + local count=debug.getinfo(func,'u').nups + for _,index in ipairs({-1,0,count+1}) do + checkError(2,"invalid upvalue index",debug.upvalueid,func,index) + end + end + checkError(1,"Lua function expected",debug.upvaluejoin,require,1,f,1) + checkError(3,"Lua function expected",debug.upvaluejoin,f,1,require,1) + checkError(2,"invalid upvalue index",debug.upvaluejoin,require,0,f,1) + checkError(4,"invalid upvalue index",debug.upvaluejoin,require,1,f,0) + checkError(2,"invalid upvalue index",debug.upvaluejoin,f,0,f,1) + checkError(4,"invalid upvalue index",debug.upvaluejoin,f,1,f,0) + assert(f()==42) + """ + ); + } + [Test] public void ManagedUpValueIdsDoNotKeepDiscardedCapturesAlive() { diff --git a/tests/Lua.Tests/ProtectedCallTests.cs b/tests/Lua.Tests/ProtectedCallTests.cs index 7822da46..2e9caa58 100644 --- a/tests/Lua.Tests/ProtectedCallTests.cs +++ b/tests/Lua.Tests/ProtectedCallTests.cs @@ -44,9 +44,12 @@ await state.DoStringAsync( assert(ok and a==12 and b==34) local ok,e=xpcall(function() error('failure') end,nil) assert(not ok and e=='error in error handling') - local handler=setmetatable({}, {__call=function(self,e) return e end}) + local hits=0 + local handler=setmetatable({}, {__call=function(self,e) hits=hits+1;return e end}) + local ok,value=xpcall(function() return 42 end,handler) + assert(ok and value==42) local ok,e=xpcall(function() error(42) end,handler) - assert(not ok and e==42) + assert(not ok and e=='error in error handling' and hits==0) """ ); } @@ -93,6 +96,16 @@ public async Task XpcallSupportsOverlappingAndForwardReturnBuffers(int returnBas [TestCase(true, "sort")] [TestCase(false, "gsub")] [TestCase(true, "gsub")] + [TestCase(false, "table.concat")] + [TestCase(true, "table.concat")] + [TestCase(false, "table.insert")] + [TestCase(true, "table.insert")] + [TestCase(false, "table.remove")] + [TestCase(true, "table.remove")] + [TestCase(false, "table.sort")] + [TestCase(true, "table.sort")] + [TestCase(false, "table.unpack")] + [TestCase(true, "table.unpack")] public async Task HandlerCanInspectFailingFramesAndLocals(bool asynchronous, string call) { using var state = CreateState(); @@ -122,6 +135,10 @@ local function namedFailure() if call=='searcher' or call=='loader' then require('test') elseif call=='sort' then table.sort({2,1},namedFailure) elseif call=='gsub' then string.gsub('a','.',namedFailure) + elseif call:sub(1,6)=='table.' then + local t=setmetatable({},{__len=namedFailure}) + if call=='table.insert' then table.insert(t,42) + else table[call:sub(7)](t) end else namedFailure() end end,function(e) local trace=debug.traceback(e) diff --git a/tests/Lua.Tests/StandardLibraryCallbackTests.cs b/tests/Lua.Tests/StandardLibraryCallbackTests.cs new file mode 100644 index 00000000..11a76fa0 --- /dev/null +++ b/tests/Lua.Tests/StandardLibraryCallbackTests.cs @@ -0,0 +1,94 @@ +using Lua.Platforms; +using Lua.Standard; + +namespace Lua.Tests; + +[TestFixture(LuaStringMode.Utf16)] +[TestFixture(LuaStringMode.Bytes)] +public class StandardLibraryCallbackTests(LuaStringMode mode) +{ + LuaState CreateState() + { + var state = LuaState.Create( + mode == LuaStringMode.Bytes ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default + ); + state.OpenStandardLibraries(); + return state; + } + + static readonly string[] CallbackOperations = + [ + "pairs(setmetatable({},{__pairs=callback}))", + "ipairs(setmetatable({},{__ipairs=callback}))", + "tostring(setmetatable({},{__tostring=callback}))", + "string.format('%s',setmetatable({},{__tostring=callback}))", + "string.gsub('a','.',callback)", + "string.gsub('a','.',setmetatable({},{__index=callback}))", + "table.sort({2,1},callback)", + "local meta={__lt=callback};table.sort({setmetatable({},meta),setmetatable({},meta)})", + "table.concat(setmetatable({},{__len=callback}))", + "table.insert(setmetatable({},{__len=callback}),42)", + "table.remove(setmetatable({},{__len=callback}))", + "table.sort(setmetatable({},{__len=callback}))", + "table.unpack(setmetatable({},{__len=callback}))", + ]; + + [TestCaseSource(nameof(CallbackOperations))] + public async Task CallbacksCannotYieldAndRestoreYieldability(string operation) + { + using var state = CreateState(); + state.Environment["operation"] = state.Load(operation, "@callback"); + await state.DoStringAsync( + """ + callback=function() coroutine.yield('unexpected') end + local co=coroutine.create(function() + local ok,e=pcall(operation) + assert(not ok and e:find('attempt to yield across a C-call boundary',1,true)) + coroutine.yield('after callback') + return 42 + end) + local ok,value=coroutine.resume(co) + assert(ok and value=='after callback') + ok,value=coroutine.resume(co) + assert(ok and value==42 and coroutine.status(co)=='dead') + """ + ); + Assert.That(state.NonYieldableCallCount, Is.Zero); + Assert.That(state.Stack.Count, Is.Zero); + Assert.That(state.CallStackFrameCount, Is.Zero); + } + + [Test, Combinatorial] + public async Task HostCallbacksCanAwaitAndRestoreState( + [ValueSource(nameof(CallbackOperations))] string operation, + [Values(false, true)] bool cancellation + ) + { + using var state = CreateState(); + using var source = new CancellationTokenSource(); + var suspension = new ManualSuspension(); + var calls = 0; + state.Environment["callback"] = new LuaFunction( + async (context, token) => + { + calls++; + await suspension; + token.ThrowIfCancellationRequested(); + return context.Return(0); + } + ); + var task = state.DoStringAsync(operation, cancellationToken: source.Token).AsTask(); + Assert.That(task.IsCompleted, Is.False); + if (cancellation) + source.Cancel(); + suspension.Resume(); + if (cancellation) + Assert.ThrowsAsync(async () => await task); + else + await task; + Assert.That(calls, Is.EqualTo(1)); + Assert.That(state.NonYieldableCallCount, Is.Zero); + Assert.That(state.Stack.Count, Is.Zero); + Assert.That(state.CallStackFrameCount, Is.Zero); + } +} From 7257b7f29988625879d7584ce9adf86fa673e46c Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Fri, 30 Oct 2026 21:10:12 +0900 Subject: [PATCH 11/16] fix: preserve Lua 5.2 module and debug contracts Register _G, package and coroutine in package.loaded when their libraries are opened so require returns the existing library tables. Preserve supplied messages when formatting saved coroutine tracebacks and tracebacks with no active frames. Use the normal gethook metadata result for disabled hooks so callers receive nil, an empty mask and zero count. Reject Lua yields from Lua-installed debug hooks, including callbacks that disable their own hooks. Capture the hook policy before invocation and restore the existing non-yieldable counter on every exit. Keep C# awaits and host-installed hook yields supported without adding instruction-loop checks or callback wrappers. Add regressions for library registration and reopening, coroutine traceback messages, disabled hook metadata, all Lua hook events, and host async/yield compatibility. Validation: Release build across all targets; 3,157 runtime tests on each of net8.0 and net10.0; 73 generator tests; final targeted hook tests on both runtimes; formatting; native Lua 5.2.4 comparisons in both string modes. Paired hook benchmarks show unchanged allocations and timing changes from -1.8% to +3.3% across the measured workloads. --- src/Lua/Runtime/LuaVirtualMachine.Debug.cs | 15 +++ src/Lua/Standard/DebugLibrary.cs | 25 +++-- src/Lua/Standard/OpenLibsExtensions.cs | 3 + tests/Lua.Tests/DebugTracebackTests.cs | 44 ++++++++ tests/Lua.Tests/HookArgumentTests.cs | 121 ++++++++++++++++++++- tests/Lua.Tests/HookTests.cs | 1 + tests/Lua.Tests/ModuleContractTests.cs | 22 ++++ 7 files changed, 217 insertions(+), 14 deletions(-) create mode 100644 tests/Lua.Tests/DebugTracebackTests.cs diff --git a/src/Lua/Runtime/LuaVirtualMachine.Debug.cs b/src/Lua/Runtime/LuaVirtualMachine.Debug.cs index b562532e..0d2b5ae9 100644 --- a/src/Lua/Runtime/LuaVirtualMachine.Debug.cs +++ b/src/Lua/Runtime/LuaVirtualMachine.Debug.cs @@ -39,6 +39,9 @@ static async ValueTask Impl(VirtualMachineExecutionContext context) var top = stack.Count; stack.PushRange("count", LuaValue.Nil); context.State.IsInHook = true; + var isLuaHook = context.State.IsLuaDebugHook; + if (isLuaHook) + context.State.NonYieldableCallCount++; try { var frame = context.State.CreateCallStackFrame(hook, 2, top, pc); @@ -53,6 +56,8 @@ static async ValueTask Impl(VirtualMachineExecutionContext context) } finally { + if (isLuaHook) + context.State.NonYieldableCallCount--; // An aborted hook must not suppress hooks when this state is reused. // Frame creation can also fail before the callback is invoked. context.State.IsInHook = false; @@ -85,6 +90,9 @@ static async ValueTask Impl(VirtualMachineExecutionContext context) var top = stack.Count; stack.PushRange("line", line); context.State.IsInHook = true; + var isLuaHook = context.State.IsLuaDebugHook; + if (isLuaHook) + context.State.NonYieldableCallCount++; try { var frame = context.State.CreateCallStackFrame(hook, 2, top, pc); @@ -99,6 +107,8 @@ static async ValueTask Impl(VirtualMachineExecutionContext context) } finally { + if (isLuaHook) + context.State.NonYieldableCallCount--; context.State.IsInHook = false; } @@ -226,6 +236,9 @@ CancellationToken cancellationToken var top = stack.Count; stack.PushRange(eventName, LuaValue.Nil); state.IsInHook = true; + var isLuaHook = state.IsLuaDebugHook; + if (isLuaHook) + state.NonYieldableCallCount++; var framePushed = false; try { @@ -242,6 +255,8 @@ CancellationToken cancellationToken } finally { + if (isLuaHook) + state.NonYieldableCallCount--; state.IsInHook = false; if (framePushed) state.PopCallStackFrameWithStackPop(); diff --git a/src/Lua/Standard/DebugLibrary.cs b/src/Lua/Standard/DebugLibrary.cs index 13764b88..8b165c0a 100644 --- a/src/Lua/Standard/DebugLibrary.cs +++ b/src/Lua/Standard/DebugLibrary.cs @@ -389,16 +389,20 @@ CancellationToken cancellationToken if (state is { IsCoroutine: true, LuaTraceback: { } traceback }) { - return new(context.Return(traceback.ToString(level))); + return new( + context.Return( + Runtime.Traceback.CreateTracebackMessage( + context.GlobalState, + traceback.StackFrames, + message, + level + ) + ) + ); } var callStack = state.GetCallStackFrames(); - if (callStack.Length == 0) - { - return new(context.Return("stack traceback:")); - } - - var skipCount = Math.Min(Math.Max(level - 1, 0), callStack.Length - 1); + var skipCount = Math.Min(Math.Max(level - 1, 0), callStack.Length); var frames = callStack[..^skipCount]; return new( context.Return( @@ -516,6 +520,7 @@ CancellationToken cancellationToken stack.Push("return"); stack.Push(LuaValue.Nil); context.State.IsInHook = true; + context.State.NonYieldableCallCount++; var framePushed = false; try { @@ -532,6 +537,7 @@ CancellationToken cancellationToken } finally { + context.State.NonYieldableCallCount--; context.State.IsInHook = false; if (framePushed) { @@ -549,11 +555,6 @@ CancellationToken cancellationToken ) { var state = GetLuaThread(context, out _); - if (state.Hook is null && !state.IsLuaDebugHook) - { - return new(context.Return(LuaValue.Nil, LuaValue.Nil, LuaValue.Nil)); - } - return new( context.Return( state.Hook is null ? LuaValue.Nil : state.Hook, diff --git a/src/Lua/Standard/OpenLibsExtensions.cs b/src/Lua/Standard/OpenLibsExtensions.cs index de07aef7..0d3937f6 100644 --- a/src/Lua/Standard/OpenLibsExtensions.cs +++ b/src/Lua/Standard/OpenLibsExtensions.cs @@ -8,6 +8,7 @@ public static void OpenBasicLibrary(this LuaState state) { var globalState = state.GlobalState; globalState.Environment["_G"] = globalState.Environment; + globalState.LoadedModules["_G"] = globalState.Environment; globalState.Environment["_VERSION"] = "Lua 5.2"; foreach (var func in BasicLibrary.Instance.Functions) { @@ -39,6 +40,7 @@ public static void OpenCoroutineLibrary(this LuaState state) } globalState.Environment["coroutine"] = coroutine; + globalState.LoadedModules["coroutine"] = coroutine; } public static void OpenIOLibrary(this LuaState state) @@ -101,6 +103,7 @@ public static void OpenModuleLibrary(this LuaState state) package["searchpath"] = moduleLibrary.SearchPathFunction; package["config"] = $"{Path.DirectorySeparatorChar}\n;\n?\n!\n-"; globalState.Environment["package"] = package; + globalState.LoadedModules["package"] = package; globalState.Environment["require"] = new CSharpClosure( "require", [package], diff --git a/tests/Lua.Tests/DebugTracebackTests.cs b/tests/Lua.Tests/DebugTracebackTests.cs new file mode 100644 index 00000000..551cb87f --- /dev/null +++ b/tests/Lua.Tests/DebugTracebackTests.cs @@ -0,0 +1,44 @@ +using Lua.Platforms; +using Lua.Standard; + +namespace Lua.Tests; + +[TestFixture(LuaStringMode.Utf16)] +[TestFixture(LuaStringMode.Bytes)] +public class DebugTracebackTests(LuaStringMode mode) +{ + [TestCase("failed")] + [TestCase("completed")] + [TestCase("unstarted")] + public async Task CoroutineTracebacksPreserveTheSuppliedMessage(string status) + { + using var state = LuaState.Create( + mode == LuaStringMode.Bytes ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default + ); + state.OpenStandardLibraries(); + state.Environment["status"] = status; + await state.DoStringAsync( + """ + local co=coroutine.create(function() + if status=='failed' then error('failure',0) end + return 42 + end) + if status~='unstarted' then + local ok,e=coroutine.resume(co) + assert(ok==(status=='completed')) + assert(debug.traceback(co,e):sub(1,#tostring(e))==tostring(e)) + end + local message='message ế\0done' + local prefix=message..'\nstack traceback:' + assert(debug.traceback(co,message):sub(1,#prefix)==prefix) + assert(debug.traceback(co,message,0):sub(1,#prefix)==prefix) + assert(debug.traceback(co,message,1000)==prefix) + assert(debug.traceback(co,42,0):sub(1,3)=='42\n') + assert(debug.traceback(co,nil,0):sub(1,16)=='stack traceback:') + local object={} + assert(debug.traceback(co,object)==object) + """, + "@traceback-workload.lua" + ); + } +} diff --git a/tests/Lua.Tests/HookArgumentTests.cs b/tests/Lua.Tests/HookArgumentTests.cs index a478bc7e..dee3e92d 100644 --- a/tests/Lua.Tests/HookArgumentTests.cs +++ b/tests/Lua.Tests/HookArgumentTests.cs @@ -28,16 +28,22 @@ await state.DoStringAsync( """ local co=coroutine.create(function() end) local function hook() end + local function checkDisabled(thread) + local callback,mask,count=debug.gethook(thread) + assert(callback==nil and mask=='' and count==0) + end + checkDisabled() + checkDisabled(co) if thread then debug.sethook(co,hook,'c') assert(debug.gethook(co)==hook) debug.sethook(co,nil,{}, {}) - assert(debug.gethook(co)==nil) + checkDisabled(co) else debug.sethook(hook,'c') assert(debug.gethook()==hook) debug.sethook(nil,{}, {}) - assert(debug.gethook()==nil) + checkDisabled() end """ ); @@ -45,6 +51,117 @@ local function hook() end Assert.That(state.CallStackFrameCount, Is.Zero); } + const string HookWorkload = """ + debug.sethook(function(event) + if event==hookEvent then + debug.sethook() + work() + end + end,hookMask,hookCount) + local function leaf() return 42 end + local value=leaf() + return leaf() + """; + + static LuaClosure LoadHookWorkload(LuaState state, string hookEvent) + { + state.Environment["hookEvent"] = hookEvent; + state.Environment["hookMask"] = hookEvent switch + { + "call" or "tail call" => "c", + "return" => "r", + "line" => "l", + _ => "", + }; + state.Environment["hookCount"] = hookEvent == "count" ? 1 : 0; + return state.Load(HookWorkload, "@hook-workload.lua"); + } + + [TestCase("call")] + [TestCase("tail call")] + [TestCase("return")] + [TestCase("line")] + [TestCase("count")] + public async Task LuaHooksCannotYieldAfterDisablingTheHook(string hookEvent) + { + using var state = CreateState(); + state.Environment["work"] = state.Environment["coroutine"].Read()["yield"]; + state.Environment["operation"] = LoadHookWorkload(state, hookEvent); + await state.DoStringAsync( + """ + thread=coroutine.create(operation) + local ok,e=coroutine.resume(thread) + assert(not ok and e:find('attempt to yield across a C-call boundary',1,true)) + assert(coroutine.status(thread)=='dead') + """ + ); + var thread = state.Environment["thread"].Read(); + Assert.That(thread.NonYieldableCallCount, Is.Zero); + Assert.That(thread.CallStackFrameCount, Is.Zero); + Assert.That(state.Stack.Count, Is.EqualTo(4)); + Assert.That(state.CallStackFrameCount, Is.Zero); + } + + [TestCase("call")] + [TestCase("tail call")] + [TestCase("return")] + [TestCase("line")] + [TestCase("count")] + public async Task LuaHooksCanAwaitHostCallbacks(string hookEvent) + { + using var state = CreateState(); + var suspension = new ManualSuspension(); + var calls = 0; + state.Environment["work"] = new LuaFunction( + async (context, _) => + { + calls++; + await suspension; + return context.Return(); + } + ); + var task = state.ExecuteAsync(LoadHookWorkload(state, hookEvent)).AsTask(); + Assert.That(task.IsCompleted, Is.False); + suspension.Resume(); + Assert.That(await task, Is.EqualTo(new LuaValue[] { 42 })); + Assert.That(calls, Is.EqualTo(1)); + Assert.That(state.NonYieldableCallCount, Is.Zero); + Assert.That(state.CallStackFrameCount, Is.Zero); + Assert.That(state.Stack.Count, Is.EqualTo(4)); + } + + [TestCase("line")] + [TestCase("count")] + public async Task HostHooksCanStillYield(string hookEvent) + { + using var state = CreateState(); + using var thread = state.CreateCoroutine( + state.Load("local value=42;return value", "@host-hook.lua") + ); + thread.SetHook( + new LuaFunction( + async (context, token) => + { + context.State.SetHook(null, ""); + await context.State.YieldAsync(context, token); + return context.Return(); + } + ), + hookEvent == "line" ? "l" : "", + hookEvent == "count" ? 1 : 0 + ); + var output = new LuaStack(); + await thread.ResumeAsync(output); + Assert.That(output.Get(0), Is.EqualTo(new LuaValue(true))); + Assert.That(output.Get(1), Is.EqualTo(new LuaValue(hookEvent))); + Assert.That(thread.GetStatus(), Is.EqualTo(LuaThreadStatus.Suspended)); + output.Clear(); + await thread.ResumeAsync(output); + Assert.That(output.AsSpan().ToArray(), Is.EqualTo(new LuaValue[] { true, 42 })); + Assert.That(thread.NonYieldableCallCount, Is.Zero); + Assert.That(thread.CallStackFrameCount, Is.Zero); + } + [TestCase("")] [TestCase(",nil")] public async Task InstallingHooksRequiresAMask(string mask) diff --git a/tests/Lua.Tests/HookTests.cs b/tests/Lua.Tests/HookTests.cs index 300f8879..dc9fb162 100644 --- a/tests/Lua.Tests/HookTests.cs +++ b/tests/Lua.Tests/HookTests.cs @@ -61,6 +61,7 @@ public async Task Hook_AfterException_StateCanRunHooksAgain( } Assert.That(state.CallStackFrameCount, Is.Zero); + Assert.That(state.NonYieldableCallCount, Is.Zero); var stackTop = state.Stack.Count; state.Environment["echo"] = new LuaFunction((c, ct) => new(c.Return(c.GetArgument(0)))); diff --git a/tests/Lua.Tests/ModuleContractTests.cs b/tests/Lua.Tests/ModuleContractTests.cs index 064085ba..3ccc16fa 100644 --- a/tests/Lua.Tests/ModuleContractTests.cs +++ b/tests/Lua.Tests/ModuleContractTests.cs @@ -9,6 +9,28 @@ namespace Lua.Tests; [TestFixture(LuaStringMode.Bytes)] public class ModuleContractTests(LuaStringMode stringMode) { + [TestCase(false)] + [TestCase(true)] + public async Task StandardLibrariesAreRegisteredForRequire(bool reopen) + { + using var state = CreateState(); + if (reopen) + { + state.OpenBasicLibrary(); + state.OpenCoroutineLibrary(); + state.OpenModuleLibrary(); + } + await state.DoStringAsync( + """ + for _,name in ipairs({'_G','package','coroutine','table','io','os','string','bit32','math','debug'}) do + local library=_G[name] + assert(package.loaded[name]==library,name) + assert(require(name)==library,name) + end + """ + ); + } + [TestCase("return", "true", 1)] [TestCase("return nil", "true", 1)] [TestCase("return false", "false", 2)] From 7b5115235695c2cc622f6fe7c6d651e1ac1fc91a Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:57:39 +0900 Subject: [PATCH 12/16] fix: validate default I/O stream arguments before side effects Reject non-string/non-number filenames and closed handles in io.input and io.output before opening files, flushing output or changing the default stream. Preserve numeric filename coercion and open-handle selection. Add regression coverage in both string modes using a recording filesystem and output stream, avoiding real filesystem side effects for invalid input. Validation: regression tests and full runtime suite on net8.0 and net10.0; Release build across all targets; formatting; native Lua 5.2.4 comparisons. --- src/Lua/Standard/IOLibrary.cs | 14 +-- tests/Lua.Tests/DefaultIOStreamTests.cs | 119 ++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 6 deletions(-) create mode 100644 tests/Lua.Tests/DefaultIOStreamTests.cs diff --git a/src/Lua/Standard/IOLibrary.cs b/src/Lua/Standard/IOLibrary.cs index 01cf8828..30d701f4 100644 --- a/src/Lua/Standard/IOLibrary.cs +++ b/src/Lua/Standard/IOLibrary.cs @@ -89,15 +89,16 @@ CancellationToken cancellationToken var arg = context.Arguments[0]; if (arg.TryRead(out var file)) { + if (!file.IsOpen) + throw new LuaRuntimeException(context.State, "attempt to use a closed file"); registry["_IO_input"] = new(file); return context.Return(new LuaValue(file)); } else { + var fileName = context.GetStringArgument(0); var stream = await context.GlobalState.Platform.FileSystem.Open( - context.State.Platform.DecodeText( - arg.ToLuaString(context.State.Platform.NumberFormat) - ), + context.State.Platform.DecodeText(fileName), LuaFileOpenMode.Read, cancellationToken ); @@ -249,6 +250,8 @@ CancellationToken cancellationToken var arg = context.Arguments[0]; if (arg.TryRead(out var file)) { + if (!file.IsOpen) + throw new LuaRuntimeException(context.State, "attempt to use a closed file"); if (!ReferenceEquals(previousOutput, file) && previousOutput is { IsOpen: true }) { await previousOutput.FlushAsync(cancellationToken); @@ -259,15 +262,14 @@ CancellationToken cancellationToken } else { + var fileName = context.GetStringArgument(0); if (previousOutput is { IsOpen: true }) { await previousOutput.FlushAsync(cancellationToken); } var stream = await context.GlobalState.Platform.FileSystem.Open( - context.State.Platform.DecodeText( - arg.ToLuaString(context.State.Platform.NumberFormat) - ), + context.State.Platform.DecodeText(fileName), LuaFileOpenMode.Write, cancellationToken ); diff --git a/tests/Lua.Tests/DefaultIOStreamTests.cs b/tests/Lua.Tests/DefaultIOStreamTests.cs new file mode 100644 index 00000000..50be29de --- /dev/null +++ b/tests/Lua.Tests/DefaultIOStreamTests.cs @@ -0,0 +1,119 @@ +using Lua.IO; +using Lua.Platforms; +using Lua.Standard; +using Lua.Tests.Helpers; + +namespace Lua.Tests; + +[TestFixture(LuaStringMode.Utf16)] +[TestFixture(LuaStringMode.Bytes)] +public class DefaultIOStreamTests(LuaStringMode mode) +{ + LuaState Create(RecordingFileSystem files, FlushStream output) + { + var platform = + mode == LuaStringMode.Bytes ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default; + var state = LuaState.Create( + platform with + { + FileSystem = files, + StandardIO = new TestStandardIO(mode) { Output = output }, + } + ); + state.OpenStandardLibraries(); + return state; + } + + [Test, Combinatorial] + public async Task InvalidArgumentsDoNotOpenFilesOrChangeTheDefault( + [Values("input", "output")] string operation, + [Values("false", "{}", "function() end", "(coroutine.running())")] string argument + ) + { + var files = new RecordingFileSystem(); + using var output = new FlushStream(); + using var state = Create(files, output); + await state.DoStringAsync( + $$""" + local select=io.{{operation}} + local original=select() + local ok,e=pcall(select,{{argument}}) + assert(not ok and type(e)=='string') + assert(select()==original) + """ + ); + Assert.That(files.Paths, Is.Empty); + Assert.That(output.FlushCount, Is.Zero); + } + + [TestCase("input")] + [TestCase("output")] + public async Task ClosedHandlesDoNotChangeTheDefault(string operation) + { + var files = new RecordingFileSystem(); + using var output = new FlushStream(); + using var state = Create(files, output); + var file = new FileHandle(ILuaStream.CreateFromString("")); + await file.Close(default); + state.Environment["closed"] = new LuaValue(file); + await state.DoStringAsync( + $$""" + local select=io.{{operation}} + local original=select() + local ok,e=pcall(select,closed) + assert(not ok and e:find('closed file',1,true)) + assert(select()==original) + """ + ); + Assert.That(files.Paths, Is.Empty); + Assert.That(output.FlushCount, Is.Zero); + } + + [TestCase("input")] + [TestCase("output")] + public async Task NumericNamesAndOpenHandlesRemainSupported(string operation) + { + var files = new RecordingFileSystem(); + using var output = new FlushStream(); + using var state = Create(files, output); + await state.DoStringAsync( + $$""" + local select=io.{{operation}} + local file=select(123) + assert(io.type(file)=='file' and select()==file) + assert(select(file)==file) + """ + ); + Assert.That(files.Paths, Is.EqualTo(new[] { "123" })); + } + + sealed class RecordingFileSystem : NotImplementedExceptionFileSystemBase + { + public List Paths { get; } = []; + + public override ValueTask Open( + string path, + LuaFileOpenMode mode, + CancellationToken token + ) + { + Paths.Add(path); + return new(new LuaStream(mode, new MemoryStream())); + } + } + + sealed class FlushStream : ILuaStream + { + public bool IsOpen => true; + public LuaFileOpenMode Mode => LuaFileOpenMode.Write; + public int FlushCount { get; private set; } + + public ValueTask FlushAsync(CancellationToken cancellationToken) + { + FlushCount++; + return default; + } + + public void Dispose() { } + } +} From 5d471474f560f30695bbf1caa96937138ba6cb7b Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:57:40 +0900 Subject: [PATCH 13/16] fix: delete temporary files when their handles close Create temporary files with unique names, CreateNew and DeleteOnClose so io.tmpfile does not leave files behind after close or process exit. Avoid creating a separate persistent placeholder file. Add a deterministic deletion-on-close regression and remove ineffective cleanup code from the existing temporary-file round-trip test. Validation: I/O tests and full runtime suite on net8.0 and net10.0; Release build across all targets; formatting; isolated cleanup repro. --- src/Lua/IO/FileSystem.cs | 18 +++++++++++------- tests/Lua.Tests/IOTests.cs | 37 +++++++++++++++++-------------------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/src/Lua/IO/FileSystem.cs b/src/Lua/IO/FileSystem.cs index 8052e6b2..f9d57a4e 100644 --- a/src/Lua/IO/FileSystem.cs +++ b/src/Lua/IO/FileSystem.cs @@ -139,14 +139,18 @@ public string GetTempFileName() return Path.GetTempFileName(); } + internal static FileStream CreateTemporaryFile() => + new( + Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()), + FileMode.CreateNew, + FileAccess.ReadWrite, + FileShare.None, + 4096, + FileOptions.DeleteOnClose + ); + public ValueTask OpenTempFileStream(CancellationToken cancellationToken) { - return new( - new LuaStream( - LuaFileOpenMode.WriteUpdate, - File.Open(Path.GetTempFileName(), FileMode.Open, FileAccess.ReadWrite), - stringMode - ) - ); + return new(new LuaStream(LuaFileOpenMode.WriteUpdate, CreateTemporaryFile(), stringMode)); } } diff --git a/tests/Lua.Tests/IOTests.cs b/tests/Lua.Tests/IOTests.cs index 037499db..436e0298 100644 --- a/tests/Lua.Tests/IOTests.cs +++ b/tests/Lua.Tests/IOTests.cs @@ -428,27 +428,24 @@ public void FileSystem_IsReadable_Works() [Test] public async Task FileSystem_TempFile_Works() { - string? tempPath = null; - - try - { - using (var tempStream = await fileSystem.OpenTempFileStream(CancellationToken.None)) - { - await tempStream.WriteAsync("temp content".AsMemory(), CancellationToken.None); + using var tempStream = await fileSystem.OpenTempFileStream(CancellationToken.None); + await tempStream.WriteAsync("temp content".AsMemory(), CancellationToken.None); + tempStream.Seek(SeekOrigin.Begin, 0); + var content = await tempStream.ReadAllAsync(CancellationToken.None); + Assert.That(content, Is.EqualTo("temp content")); + } - // Seek and read - tempStream.Seek(SeekOrigin.Begin, 0); - var content = await tempStream.ReadAllAsync(CancellationToken.None); - Assert.That(content, Is.EqualTo("temp content")); - } - } - finally - { - if (tempPath != null && File.Exists(tempPath)) - { - File.Delete(tempPath); - } - } + [Test] + public async Task TemporaryFilesAreDeletedOnClose() + { + using var file = FileSystem.CreateTemporaryFile(); + var path = file.Name; + using ILuaStream stream = new LuaStream(LuaFileOpenMode.WriteUpdate, file); + await stream.WriteAsync("temporary", default); + stream.Seek(SeekOrigin.Begin, 0); + Assert.That(await stream.ReadAllAsync(default), Is.EqualTo("temporary")); + await stream.CloseAsync(default); + Assert.That(File.Exists(path), Is.False); } [Test] From 473d9e9171f594f2a2c686528d28a76ace2e3323 Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:00:05 +0900 Subject: [PATCH 14/16] fix: preserve UTF-8 stream data across reads and seeks Detect a UTF-8 BOM only at the start of text input, including short reads and buffered EOF probes. Keep embedded BOM characters as ordinary data. Decode counted UTF-16 reads with enough room for a surrogate pair, retaining one pending character across subsequent reads and format changes. Flush the decoder at EOF and reset it after successful seeks. Position queries still synchronize the underlying stream before writes while preserving pending text. Leave failed seeks untouched. Separate byte reads from UTF-16 decoding and run BOM initialization only when needed. Share buffer refills and keep additional decoder state to a single pending character. Add regressions for buffer boundaries, short reads, supplementary characters, EOF replacement fallback, format changes, seek state and read-seek-write sequences in both string modes. Validation: Release build across all targets; 3,252 runtime tests per net8.0 and net10.0; 73 generator tests; formatting; native Lua 5.2.4 comparisons and StreamReader comparisons. Paired benchmarks show unchanged allocations, byte reads within about 1%, UTF-16 single-character reads 45-47% faster, and UTF-16 bulk reads about 3-4% slower across the measured workloads. --- src/Lua/IO/LuaStream.cs | 3 +- src/Lua/Internal/Utf8Reader.cs | 236 ++++++++++++++++++------- tests/Lua.Tests/Utf8StreamReadTests.cs | 211 ++++++++++++++++++++++ 3 files changed, 390 insertions(+), 60 deletions(-) create mode 100644 tests/Lua.Tests/Utf8StreamReadTests.cs diff --git a/src/Lua/IO/LuaStream.cs b/src/Lua/IO/LuaStream.cs index 71452bd3..e52b9608 100644 --- a/src/Lua/IO/LuaStream.cs +++ b/src/Lua/IO/LuaStream.cs @@ -185,13 +185,14 @@ public void SetVBuf(LuaFileBufferingMode mode, int size) public long Seek(SeekOrigin origin, long offset) { + var keepPendingCharacter = origin == SeekOrigin.Current && offset == 0; if (reader != null && origin == SeekOrigin.Current) { offset -= reader.Remain; } var position = innerStream.Seek(offset, origin); - reader?.Clear(); + reader?.Clear(position == 0, keepPendingCharacter); return position; } diff --git a/src/Lua/Internal/Utf8Reader.cs b/src/Lua/Internal/Utf8Reader.cs index 0659ce63..c7dd0c79 100644 --- a/src/Lua/Internal/Utf8Reader.cs +++ b/src/Lua/Internal/Utf8Reader.cs @@ -13,6 +13,8 @@ sealed class Utf8Reader int bufPos, bufLen; Decoder? decoder; + bool checkBom; + int pendingCharacter = -1; /// /// In byte mode, every byte is decoded to one char (0x00-0xFF) and a UTF-8 BOM is not skipped, @@ -22,23 +24,84 @@ public Utf8Reader(LuaStringMode stringMode = LuaStringMode.Utf16) { byteStrings = stringMode == LuaStringMode.Bytes; encoding = byteStrings ? LuaByteEncoding.Instance : Encoding.UTF8; + checkBom = !byteStrings; } public long Remain => bufLen - bufPos; - bool ReadTextBuffer(Stream stream) + bool ReadBuffer(Stream stream) { bufLen = stream.Read(buffer, 0, buffer.Length); - bufPos = !byteStrings && buffer.AsSpan(0, bufLen).StartsWith(BomUtility.BomUtf8) ? 3 : 0; - return bufPos < bufLen; + bufPos = 0; + return bufLen != 0; + } + + void PrepareText(Stream stream) + { + checkBom = false; + if (stream.CanSeek && stream.Position - Remain != 0) + return; + if (PeekByte(stream) != 0xEF) + return; + + var count = bufLen - bufPos; + if (count < BomUtility.BomUtf8.Length) + { + buffer.AsSpan(bufPos, count).CopyTo(buffer); + bufPos = 0; + bufLen = count; + while (bufLen < BomUtility.BomUtf8.Length) + { + var read = stream.Read(buffer, bufLen, BomUtility.BomUtf8.Length - bufLen); + if (read == 0) + break; + bufLen += read; + } + } + if (buffer.AsSpan(bufPos, bufLen - bufPos).StartsWith(BomUtility.BomUtf8)) + bufPos += BomUtility.BomUtf8.Length; + } + + string DecodeText(ReadOnlySpan bytes) + { + var text = encoding.GetString(bytes); + return pendingCharacter < 0 ? text : PrependPendingCharacter(text); + } + + string PrependPendingCharacter(string text) + { + var first = (char)pendingCharacter; + pendingCharacter = -1; + return string.Create( + text.Length + 1, + (first, text), + static (span, state) => + { + span[0] = state.first; + state.text.AsSpan().CopyTo(span[1..]); + } + ); } public string? ReadLine(Stream stream, bool keepEol = false) { + if (checkBom) + PrepareText(stream); + if (pendingCharacter is '\r' or '\n') + { + var first = (char)pendingCharacter; + pendingCharacter = -1; + var crlf = first == '\r' && PeekByte(stream) == '\n'; + if (crlf) + bufPos++; + return !keepEol ? "" + : crlf ? "\r\n" + : first.ToString(); + } using PooledList result = new(); while (true) { - if (bufPos >= bufLen && !ReadTextBuffer(stream)) + if (bufPos >= bufLen && !ReadBuffer(stream)) break; var span = buffer.AsSpan(bufPos, bufLen - bufPos); @@ -67,7 +130,7 @@ bool ReadTextBuffer(Stream stream) if (keepEol) result.Add((byte)'\n'); } - return encoding.GetString(result.AsSpan()); + return DecodeText(result.AsSpan()); } if (buffer[bufPos] == '\n') { @@ -77,31 +140,36 @@ bool ReadTextBuffer(Stream stream) } } if (result.Count == 0) - return encoding.GetString(span[..count]); + return DecodeText(span[..count]); result.AddRange(span[..count]); - return encoding.GetString(result.AsSpan()); + return DecodeText(result.AsSpan()); } - return result.Count == 0 ? null : encoding.GetString(result.AsSpan()); + return result.Count == 0 && pendingCharacter < 0 ? null : DecodeText(result.AsSpan()); } public string ReadToEnd(Stream stream) { + if (checkBom) + PrepareText(stream); using PooledList result = new(); while (true) { - if (bufPos >= bufLen && !ReadTextBuffer(stream)) + if (bufPos >= bufLen && !ReadBuffer(stream)) break; result.AddRange(buffer.AsSpan(bufPos, bufLen - bufPos)); bufPos = bufLen; } - return encoding.GetString(result.AsSpan()); + return DecodeText(result.AsSpan()); } public int ReadByte(Stream stream) { var value = PeekByte(stream); if (value >= 0) + { + checkBom = false; bufPos++; + } return value; } @@ -110,6 +178,7 @@ public void CopyBufferedBytes(IBufferWriter writer) var count = bufLen - bufPos; if (count == 0) return; + checkBom = false; buffer.AsSpan(bufPos, count).CopyTo(writer.GetSpan(count)); writer.Advance(count); bufPos = bufLen; @@ -128,70 +197,112 @@ public void CopyBufferedBytes(IBufferWriter writer) } var len = 0; - var dataRead = false; var resultBuffer = ArrayPool.Shared.Rent(charCount); try { - while (len < charCount) + if (byteStrings) { - if (bufPos >= bufLen && !ReadTextBuffer(stream)) - break; - - ReadOnlySpan byteSpan = new(buffer, bufPos, bufLen - bufPos); - Span charSpan = new(resultBuffer, len, charCount - len); - if (byteStrings) + while (len < charCount) { - var count = Math.Min(byteSpan.Length, charSpan.Length); - LuaByteEncoding.Instance.GetChars(byteSpan[..count], charSpan); + if (bufPos >= bufLen && !ReadBuffer(stream)) + break; + var count = Math.Min(bufLen - bufPos, charCount - len); + LuaByteEncoding.Instance.GetChars( + buffer.AsSpan(bufPos, count), + resultBuffer.AsSpan(len, count) + ); len += count; bufPos += count; - dataRead = true; - continue; } + } + else + len = ReadText(stream, resultBuffer.AsSpan(0, charCount)); - decoder ??= encoding.GetDecoder(); - decoder.Convert( - byteSpan, - charSpan, - false, - out var bytesUsed, - out var charsUsed, - out _ - ); - - if (charsUsed > 0) - { - len += charsUsed; - dataRead = true; - } + return len == 0 ? null : resultBuffer.AsSpan(0, len).ToString(); + } + finally + { + ArrayPool.Shared.Return(resultBuffer); + } + } + + int ReadText(Stream stream, Span destination) + { + if (checkBom) + PrepareText(stream); + var len = 0; + if (pendingCharacter >= 0) + { + destination[len++] = (char)pendingCharacter; + pendingCharacter = -1; + } + Span pair = stackalloc char[2]; + while (len < destination.Length) + { + var eof = bufPos >= bufLen && !ReadBuffer(stream); + if (eof && decoder is null) + break; - bufPos += bytesUsed; - if (bytesUsed == 0) + ReadOnlySpan byteSpan = new(buffer, bufPos, bufLen - bufPos); + scoped Span charSpan = destination[len..]; + // A UTF-8 scalar can require two UTF-16 code units. Limit final-slot + // input to one sequence, retaining any second code unit. + var singleSlot = charSpan.Length == 1; + if (singleSlot) + { + if (!byteSpan.IsEmpty) { - break; + var sequenceLength = byteSpan[0] switch + { + >= 0xF0 and <= 0xF4 => 4, + >= 0xE0 and <= 0xEF => 3, + >= 0xC2 and <= 0xDF => 2, + _ => 1, + }; + byteSpan = byteSpan[..Math.Min(byteSpan.Length, sequenceLength)]; } + charSpan = pair; } + decoder ??= encoding.GetDecoder(); + decoder.Convert( + byteSpan, + charSpan, + eof, + out var bytesUsed, + out var charsUsed, + out var completed + ); - if (!dataRead) + if (charsUsed > 0) { - return null; + if (singleSlot) + { + destination[len++] = pair[0]; + if (charsUsed == 2) + pendingCharacter = pair[1]; + } + else + len += charsUsed; } - return resultBuffer.AsSpan(0, len).ToString(); - } - finally - { - ArrayPool.Shared.Return(resultBuffer); + bufPos += bytesUsed; + if (eof && completed) + break; } + return len; } static bool IsAsciiSpace(char c) => c == ' ' || (c >= '\t' && c <= '\r'); - public void Clear() + public void Clear(bool atStart = false, bool keepPendingCharacter = false) { bufPos = 0; bufLen = 0; + decoder?.Reset(); + if (!keepPendingCharacter) + pendingCharacter = -1; + checkBom = atStart && !byteStrings; } public double? ReadNumber(Stream stream) @@ -202,8 +313,20 @@ public void Clear() try { + checkBom = false; + if (pendingCharacter >= 0) + { + var c = (char)pendingCharacter; + if (!char.IsWhiteSpace(c)) + { + if (!scanner.TryConsume(c)) + return null; + resultBuffer[len++] = c; + } + pendingCharacter = -1; + } // Skip leading whitespace - while (true) + while (len == 0) { var b = PeekByte(stream); if (b == -1) @@ -217,7 +340,7 @@ public void Clear() break; } - ReadByte(stream); // Consume whitespace + bufPos++; } while (true) @@ -233,7 +356,7 @@ public void Clear() resultBuffer = expanded; } resultBuffer[len++] = (char)b; - ReadByte(stream); + bufPos++; } return len == 0 ? null : NumberReaderHelper.ParseNumber(resultBuffer.AsSpan(0, len)); } @@ -245,18 +368,13 @@ public void Clear() public bool IsEndOfStream(Stream stream) { - return PeekByte(stream) < 0; + return pendingCharacter < 0 && PeekByte(stream) < 0; } int PeekByte(Stream stream) { - if (bufPos >= bufLen) - { - bufLen = stream.Read(buffer, 0, buffer.Length); - bufPos = 0; - if (bufLen == 0) - return -1; - } + if (bufPos >= bufLen && !ReadBuffer(stream)) + return -1; return buffer[bufPos]; } diff --git a/tests/Lua.Tests/Utf8StreamReadTests.cs b/tests/Lua.Tests/Utf8StreamReadTests.cs new file mode 100644 index 00000000..922c96c1 --- /dev/null +++ b/tests/Lua.Tests/Utf8StreamReadTests.cs @@ -0,0 +1,211 @@ +using System.Text; +using Lua.IO; +using Lua.Platforms; +using Lua.Standard; + +namespace Lua.Tests; + +[TestFixture(LuaStringMode.Utf16)] +[TestFixture(LuaStringMode.Bytes)] +public class Utf8StreamReadTests(LuaStringMode mode) +{ + string Expected(byte[] bytes) => + mode == LuaStringMode.Bytes + ? LuaByteString.FromBytes(bytes) + : Encoding.UTF8.GetString(bytes); + + LuaStream Create(byte[] bytes, int chunkSize = int.MaxValue) => + new(LuaFileOpenMode.Read, new ShortReadStream(bytes, chunkSize), mode); + + static ValueTask Read(LuaStream stream, string format) => + format switch + { + "line" => stream.ReadLineAsync(false, default), + "count" => stream.ReadAsync(10000, default), + _ => ReadAll(stream), + }; + + static async ValueTask ReadAll(LuaStream stream) => await stream.ReadAllAsync(default); + + [Test, Combinatorial] + public async Task OnlyTheInitialBomIsSkipped( + [Values("line", "count", "all")] string format, + [Values(false, true)] bool leadingBom + ) + { + var content = new string('a', leadingBom ? 1021 : 1024) + "\uFEFFz"; + var bytes = Encoding.UTF8.GetBytes((leadingBom ? "\uFEFF" : "") + content); + using var stream = Create(bytes); + var expected = mode == LuaStringMode.Bytes ? Expected(bytes) : content; + Assert.That(await Read(stream, format), Is.EqualTo(expected)); + stream.Seek(SeekOrigin.Begin, 0); + Assert.That(await Read(stream, format), Is.EqualTo(expected)); + } + + [Test, Combinatorial] + public async Task InitialBomSurvivesShortReadsAndEofProbes( + [Values("line", "count", "all")] string format, + [Values(1, 2, 3)] int chunkSize + ) + { + var bytes = Encoding.UTF8.GetBytes("\uFEFF42"); + using var stream = Create(bytes, chunkSize); + Assert.That(await stream.ReadAsync(0, default), Is.EqualTo("")); + Assert.That( + await Read(stream, format), + Is.EqualTo(mode == LuaStringMode.Bytes ? Expected(bytes) : "42") + ); + } + + [Test] + public async Task SeekingIntoTheFileDoesNotStripItsBomCharacter() + { + var bytes = Encoding.UTF8.GetBytes("a\uFEFFb"); + using var stream = Create(bytes); + stream.Seek(SeekOrigin.Begin, 1); + Assert.That(await stream.ReadAllAsync(default), Is.EqualTo(Expected(bytes[1..]))); + } + + [Test, Combinatorial] + public async Task CountedReadsPreserveUnicodeAcrossSmallBuffers( + [Values(1, 2, 3)] int count, + [Values(1, 1024)] int chunkSize + ) + { + var bytes = Encoding.UTF8.GetBytes("a😀bế𐀀z"); + using var stream = Create(bytes, chunkSize); + var result = new StringBuilder(); + while (await stream.ReadAsync(count, default) is { } text) + { + Assert.That(text.Length, Is.InRange(1, count)); + result.Append(text); + } + Assert.That(result.ToString(), Is.EqualTo(Expected(bytes))); + } + + [Test] + public async Task LuaCountedReadsCanIterateEmoji() + { + using var state = LuaState.Create( + mode == LuaStringMode.Bytes ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default + ); + state.OpenStandardLibraries(); + var bytes = Encoding.UTF8.GetBytes("😀x"); + using var stream = Create(bytes); + state.Environment["file"] = new LuaValue(new FileHandle(stream)); + var result = await state.DoStringAsync( + "local t={};for c in file:lines(1) do t[#t+1]=c end;return table.concat(t)" + ); + Assert.That(result, Is.EqualTo(new LuaValue[] { Expected(bytes) })); + } + + [TestCase("line")] + [TestCase("all")] + [TestCase("number")] + public async Task OtherFormatsPreserveThePendingCharacter(string format) + { + var bytes = Encoding.UTF8.GetBytes("😀12\n34"); + using var stream = Create(bytes); + var first = await stream.ReadAsync(1, default); + Assert.That(await stream.ReadAsync(0, default), Is.EqualTo("")); + if (format == "number") + Assert.That(await stream.ReadNumberAsync(default), Is.Null); + if (format == "line") + { + Assert.That( + first + await stream.ReadLineAsync(false, default), + Is.EqualTo(Expected(bytes[..^3])) + ); + Assert.That(await stream.ReadNumberAsync(default), Is.EqualTo(34)); + } + else + Assert.That(first + await stream.ReadAllAsync(default), Is.EqualTo(Expected(bytes))); + } + + [TestCase(1)] + [TestCase(2)] + [TestCase(20)] + public async Task EofFlushesTheDecoderAndSeekingResetsIt(int count) + { + byte[] bytes = [(byte)'a', 0xe2, 0x82]; + using var stream = Create(bytes, 1); + for (var pass = 0; pass < 2; pass++) + { + var result = new StringBuilder(); + while (await stream.ReadAsync(count, default) is { } text) + result.Append(text); + Assert.That(result.ToString(), Is.EqualTo(Expected(bytes))); + stream.Seek(SeekOrigin.Begin, 0); + } + } + + [Test] + public async Task PendingCharactersRemainReadableAtEofAndAfterPositionQueries() + { + var bytes = Encoding.UTF8.GetBytes("😀"); + using var stream = Create(bytes); + var first = await stream.ReadAsync(1, default); + Assert.That( + stream.Seek(SeekOrigin.Current, 0), + Is.EqualTo(mode == LuaStringMode.Bytes ? 1 : 4) + ); + Assert.That(await stream.ReadAsync(0, default), Is.EqualTo("")); + Assert.That(first + await stream.ReadAllAsync(default), Is.EqualTo(Expected(bytes))); + Assert.That(await stream.ReadAsync(0, default), Is.Null); + } + + [TestCase("line")] + [TestCase("all")] + [TestCase("number")] + public async Task FormatChangesPreserveFallbackAndFollowingCharacters(string format) + { + var tail = format == "line" ? "\nrest" : "7 rest"; + var bytes = new byte[] { 0xe2, 0x82 } + .Concat(Encoding.UTF8.GetBytes(tail)) + .ToArray(); + using var stream = Create(bytes); + var result = await stream.ReadAsync(1, default); + if (format == "line") + result += await stream.ReadLineAsync(false, default) + "\n"; + if (format == "number") + { + var number = await stream.ReadNumberAsync(default); + Assert.That(number, Is.EqualTo(mode == LuaStringMode.Bytes ? (double?)null : 7d)); + if (number.HasValue) + result += "7"; + } + result += await stream.ReadAllAsync(default); + Assert.That(result, Is.EqualTo(Expected(bytes))); + } + + [Test] + public async Task PositionQueriesSynchronizeTheCursorBeforeWriting() + { + using var stream = new LuaStream( + LuaFileOpenMode.ReadUpdate, + new MemoryStream("abc"u8.ToArray()), + mode + ); + Assert.That(await stream.ReadAsync(1, default), Is.EqualTo("a")); + Assert.That(stream.Seek(SeekOrigin.Current, 0), Is.EqualTo(1)); + await stream.WriteAsync("x".AsMemory(), default); + stream.Seek(SeekOrigin.Begin, 0); + Assert.That(await stream.ReadAllAsync(default), Is.EqualTo("axc")); + } + + [Test] + public async Task FailedSeekPreservesThePendingCharacter() + { + var bytes = Encoding.UTF8.GetBytes("😀x"); + using var stream = Create(bytes); + var first = await stream.ReadAsync(1, default); + Assert.Throws(() => stream.Seek(SeekOrigin.Begin, -1)); + Assert.That(first + await stream.ReadAllAsync(default), Is.EqualTo(Expected(bytes))); + } + + sealed class ShortReadStream(byte[] bytes, int chunkSize) : MemoryStream(bytes) + { + public override int Read(byte[] buffer, int offset, int count) => + base.Read(buffer, offset, Math.Min(count, chunkSize)); + } +} From 73c3b77ecbc6cce60a4d5e6bacb1ea1c18b82179 Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:53:31 +0900 Subject: [PATCH 15/16] fix: preserve Lua 5.2 I/O argument and error contracts Distinguish omitted io.close arguments from explicit nil, reject closed file line iterators when they are created, and return access-denied failures from io.open. Reuse the existing open helper's error path. Add regression coverage in UTF-16 and byte-string modes. --- src/Lua/Standard/FileHandle.cs | 2 + src/Lua/Standard/IOLibrary.cs | 31 +++------ src/Lua/Standard/Internal/IOHelper.cs | 2 +- tests/Lua.Tests/DefaultIOStreamTests.cs | 85 ++++++++++++++++++++++++- 4 files changed, 95 insertions(+), 25 deletions(-) diff --git a/src/Lua/Standard/FileHandle.cs b/src/Lua/Standard/FileHandle.cs index 115d22dc..8cd9de93 100644 --- a/src/Lua/Standard/FileHandle.cs +++ b/src/Lua/Standard/FileHandle.cs @@ -190,6 +190,8 @@ public async ValueTask Close(CancellationToken cancellationToken) (context, cancellationToken) => { var file = context.GetArgument(0); + if (!file.IsOpen) + throw new LuaRuntimeException(context.State, "attempt to use a closed file"); var upValues = new LuaValue[context.ArgumentCount]; upValues[0] = new(file); context.Arguments[1..].CopyTo(upValues.AsSpan(1)); diff --git a/src/Lua/Standard/IOLibrary.cs b/src/Lua/Standard/IOLibrary.cs index 30d701f4..ddf7f05d 100644 --- a/src/Lua/Standard/IOLibrary.cs +++ b/src/Lua/Standard/IOLibrary.cs @@ -33,9 +33,10 @@ public async ValueTask Close( CancellationToken cancellationToken ) { - var file = context.HasArgument(0) - ? context.GetArgument(0) - : context.GlobalState.Registry["_IO_output"].Read(); + var file = + context.ArgumentCount > 0 + ? context.GetArgument(0) + : context.GlobalState.Registry["_IO_output"].Read(); try { @@ -116,6 +117,8 @@ CancellationToken cancellationToken if (context.ArgumentCount == 0 || context.Arguments[0].Type is LuaValueType.Nil) { var file = context.GlobalState.Registry["_IO_input"].Read(); + if (!file.IsOpen) + throw new LuaRuntimeException(context.State, "attempt to use a closed file"); LuaValue[] upValues = new LuaValue[ context.ArgumentCount == 0 ? 1 : context.ArgumentCount ]; @@ -203,7 +206,7 @@ static async (context, cancellationToken) => } } - public async ValueTask Open( + public ValueTask Open( LuaFunctionExecutionContext context, CancellationToken cancellationToken ) @@ -211,25 +214,7 @@ CancellationToken cancellationToken var fileName = context.GetStringArgument(0); var mode = context.HasArgument(1) ? context.GetStringArgument(1) : "r"; context.Return(); - try - { - var resultCount = await IOHelper.Open( - context.State, - fileName, - mode, - true, - cancellationToken - ); - return resultCount; - } - catch (IOException ex) - { - return context.Return( - LuaValue.Nil, - context.GlobalState.Platform.EncodeText(ex.Message), - ex.HResult - ); - } + return IOHelper.Open(context.State, fileName, mode, false, cancellationToken); } public async ValueTask Output( diff --git a/src/Lua/Standard/Internal/IOHelper.cs b/src/Lua/Standard/Internal/IOHelper.cs index ae07e615..0f64d3e6 100644 --- a/src/Lua/Standard/Internal/IOHelper.cs +++ b/src/Lua/Standard/Internal/IOHelper.cs @@ -30,7 +30,7 @@ CancellationToken cancellationToken state.Stack.Push(new(new FileHandle(stream, state.GlobalState.FileHandleMetatable))); return 1; } - catch (IOException ex) + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { if (throwError) { diff --git a/tests/Lua.Tests/DefaultIOStreamTests.cs b/tests/Lua.Tests/DefaultIOStreamTests.cs index 50be29de..5727812e 100644 --- a/tests/Lua.Tests/DefaultIOStreamTests.cs +++ b/tests/Lua.Tests/DefaultIOStreamTests.cs @@ -9,7 +9,7 @@ namespace Lua.Tests; [TestFixture(LuaStringMode.Bytes)] public class DefaultIOStreamTests(LuaStringMode mode) { - LuaState Create(RecordingFileSystem files, FlushStream output) + LuaState Create(ILuaFileSystem files, FlushStream output) { var platform = mode == LuaStringMode.Bytes ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default; @@ -87,6 +87,89 @@ await state.DoStringAsync( Assert.That(files.Paths, Is.EqualTo(new[] { "123" })); } + [Test] + public async Task CloseDistinguishesAnOmittedArgumentFromNil() + { + using var output = new FlushStream(); + using var state = Create(new RecordingFileSystem(), output); + using var stream = new LuaStream(LuaFileOpenMode.Write, new MemoryStream(), mode); + state.Environment["file"] = new LuaValue(new FileHandle(stream)); + await state.DoStringAsync( + """ + local original=io.output() + io.output(file) + local ok,e=pcall(io.close,nil) + assert(not ok and type(e)=='string') + assert(io.type(file)=='file' and io.output()==file) + assert(io.close()) + assert(io.type(file)=='closed file') + io.output(original) + """ + ); + } + + [TestCase("file:lines()")] + [TestCase("io.lines()")] + [TestCase("io.lines(nil,'*l')")] + public async Task LineIteratorFactoriesRejectClosedFiles(string expression) + { + using var output = new FlushStream(); + using var state = Create(new RecordingFileSystem(), output); + using var stream = ILuaStream.CreateFromString("line\n"); + state.Environment["file"] = new LuaValue(new FileHandle(stream)); + await state.DoStringAsync( + $$""" + local original=io.input() + io.input(file) + local iterator=file:lines() + assert(iterator()=='line') + file:close() + local ok,e=pcall(function() return {{expression}} end) + assert(not ok and e:find('closed file',1,true)) + io.input(original) + """ + ); + } + + [Test] + public async Task AccessDeniedIsReturnedByOpenButRaisedByLines() + { + var files = new DeniedFileSystem(); + using var output = new FlushStream(); + using var state = Create(files, output); + var result = await state.DoStringAsync( + """ + local file,message,code=io.open('denied','r') + local ok,e=pcall(io.lines,'denied') + return file,message,code,ok,type(e) + """ + ); + Assert.That( + result, + Is.EqualTo( + new LuaValue[] + { + LuaValue.Nil, + "access denied", + files.Error.HResult, + false, + "string", + } + ) + ); + } + + sealed class DeniedFileSystem : NotImplementedExceptionFileSystemBase + { + public UnauthorizedAccessException Error { get; } = new("access denied"); + + public override ValueTask Open( + string path, + LuaFileOpenMode mode, + CancellationToken token + ) => throw Error; + } + sealed class RecordingFileSystem : NotImplementedExceptionFileSystemBase { public List Paths { get; } = []; From 0f650c3e73422f192437a5d637540c7ada81d647 Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:53:43 +0900 Subject: [PATCH 16/16] fix: retain unread UTF-8 prefixes and cancel buffered reads Keep incomplete UTF-8 input in the byte buffer so format changes, raw reads, EOF probes, and seeks share the same unread-byte accounting. Flush truncated input at EOF and keep the single-character ASCII path allocation-free apart from the returned string. Observe read cancellation at entry and before buffer fills, without adding checks to character or numeric-scanning loops. Add deterministic regressions for format changes, seek positions, EOF, and cancellation. Validation: Release build for all targets; 3,302 runtime tests on each of .NET 8 and .NET 10; 73 source-generator tests; formatting and native Lua 5.2 comparisons. Stream benchmark allocations are unchanged. --- src/Lua/IO/LuaStream.cs | 17 ++- src/Lua/Internal/Utf8Reader.cs | 133 +++++++++++++-------- tests/Lua.Tests/StreamCancellationTests.cs | 78 ++++++++++++ tests/Lua.Tests/Utf8StreamReadTests.cs | 55 +++++++++ 4 files changed, 225 insertions(+), 58 deletions(-) create mode 100644 tests/Lua.Tests/StreamCancellationTests.cs diff --git a/src/Lua/IO/LuaStream.cs b/src/Lua/IO/LuaStream.cs index e52b9608..4ac84741 100644 --- a/src/Lua/IO/LuaStream.cs +++ b/src/Lua/IO/LuaStream.cs @@ -25,15 +25,17 @@ public LuaStream(LuaFileOpenMode mode, Stream innerStream) public ValueTask ReadLineAsync(bool keepEol, CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); + cancellationToken.ThrowIfCancellationRequested(); reader ??= new(stringMode); - return new(reader.ReadLine(innerStream, keepEol)); + return new(reader.ReadLine(innerStream, keepEol, cancellationToken)); } public ValueTask ReadAllAsync(CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); + cancellationToken.ThrowIfCancellationRequested(); reader ??= new(stringMode); - var text = reader.ReadToEnd(innerStream); + var text = reader.ReadToEnd(innerStream, cancellationToken); return new(text); } @@ -63,29 +65,32 @@ public ValueTask ReadBytesAsync(IBufferWriter writer, CancellationToken ca public ValueTask ReadByteAsync(CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); + cancellationToken.ThrowIfCancellationRequested(); if (reader == null) return new(innerStream.ReadByte()); - return new(reader.ReadByte(innerStream)); + return new(reader.ReadByte(innerStream, cancellationToken)); } public ValueTask ReadAsync(int count, CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); + cancellationToken.ThrowIfCancellationRequested(); reader ??= new(stringMode); if (count == 0) { - return new(reader.IsEndOfStream(innerStream) ? null : string.Empty); + return new(reader.IsEndOfStream(innerStream, cancellationToken) ? null : string.Empty); } - return new(reader.Read(innerStream, count)); + return new(reader.Read(innerStream, count, cancellationToken)); } public ValueTask ReadNumberAsync(CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); + cancellationToken.ThrowIfCancellationRequested(); reader ??= new(stringMode); - return new(reader.ReadNumber(innerStream)); + return new(reader.ReadNumber(innerStream, cancellationToken)); } public ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken) diff --git a/src/Lua/Internal/Utf8Reader.cs b/src/Lua/Internal/Utf8Reader.cs index c7dd0c79..f8eec332 100644 --- a/src/Lua/Internal/Utf8Reader.cs +++ b/src/Lua/Internal/Utf8Reader.cs @@ -1,4 +1,5 @@ using System.Buffers; +using System.Runtime.CompilerServices; using System.Text; using Lua.CodeAnalysis.Compilation; using Lua.IO; @@ -29,19 +30,23 @@ public Utf8Reader(LuaStringMode stringMode = LuaStringMode.Utf16) public long Remain => bufLen - bufPos; - bool ReadBuffer(Stream stream) + bool ReadBuffer(Stream stream, CancellationToken cancellationToken, int keep = 0) { - bufLen = stream.Read(buffer, 0, buffer.Length); + cancellationToken.ThrowIfCancellationRequested(); + if (keep > 0) + buffer.AsSpan(bufPos, keep).CopyTo(buffer); + var read = stream.Read(buffer, keep, buffer.Length - keep); + bufLen = keep + read; bufPos = 0; - return bufLen != 0; + return read != 0; } - void PrepareText(Stream stream) + void PrepareText(Stream stream, CancellationToken cancellationToken) { checkBom = false; if (stream.CanSeek && stream.Position - Remain != 0) return; - if (PeekByte(stream) != 0xEF) + if (PeekByte(stream, cancellationToken) != 0xEF) return; var count = bufLen - bufPos; @@ -52,6 +57,7 @@ void PrepareText(Stream stream) bufLen = count; while (bufLen < BomUtility.BomUtf8.Length) { + cancellationToken.ThrowIfCancellationRequested(); var read = stream.Read(buffer, bufLen, BomUtility.BomUtf8.Length - bufLen); if (read == 0) break; @@ -83,15 +89,15 @@ string PrependPendingCharacter(string text) ); } - public string? ReadLine(Stream stream, bool keepEol = false) + public string? ReadLine(Stream stream, bool keepEol, CancellationToken cancellationToken) { if (checkBom) - PrepareText(stream); + PrepareText(stream, cancellationToken); if (pendingCharacter is '\r' or '\n') { var first = (char)pendingCharacter; pendingCharacter = -1; - var crlf = first == '\r' && PeekByte(stream) == '\n'; + var crlf = first == '\r' && PeekByte(stream, cancellationToken) == '\n'; if (crlf) bufPos++; return !keepEol ? "" @@ -101,7 +107,7 @@ string PrependPendingCharacter(string text) using PooledList result = new(); while (true) { - if (bufPos >= bufLen && !ReadBuffer(stream)) + if (bufPos >= bufLen && !ReadBuffer(stream, cancellationToken)) break; var span = buffer.AsSpan(bufPos, bufLen - bufPos); @@ -124,7 +130,7 @@ string PrependPendingCharacter(string text) { // Preserve the segment before looking for LF in the next buffer. result.AddRange(span[..count]); - if (PeekByte(stream) == '\n') + if (PeekByte(stream, cancellationToken) == '\n') { bufPos++; if (keepEol) @@ -147,14 +153,14 @@ string PrependPendingCharacter(string text) return result.Count == 0 && pendingCharacter < 0 ? null : DecodeText(result.AsSpan()); } - public string ReadToEnd(Stream stream) + public string ReadToEnd(Stream stream, CancellationToken cancellationToken) { if (checkBom) - PrepareText(stream); + PrepareText(stream, cancellationToken); using PooledList result = new(); while (true) { - if (bufPos >= bufLen && !ReadBuffer(stream)) + if (bufPos >= bufLen && !ReadBuffer(stream, cancellationToken)) break; result.AddRange(buffer.AsSpan(bufPos, bufLen - bufPos)); bufPos = bufLen; @@ -162,9 +168,9 @@ public string ReadToEnd(Stream stream) return DecodeText(result.AsSpan()); } - public int ReadByte(Stream stream) + public int ReadByte(Stream stream, CancellationToken cancellationToken) { - var value = PeekByte(stream); + var value = PeekByte(stream, cancellationToken); if (value >= 0) { checkBom = false; @@ -184,7 +190,7 @@ public void CopyBufferedBytes(IBufferWriter writer) bufPos = bufLen; } - public string? Read(Stream stream, int charCount) + public string? Read(Stream stream, int charCount, CancellationToken cancellationToken) { if (charCount < 0) { @@ -205,7 +211,7 @@ public void CopyBufferedBytes(IBufferWriter writer) { while (len < charCount) { - if (bufPos >= bufLen && !ReadBuffer(stream)) + if (bufPos >= bufLen && !ReadBuffer(stream, cancellationToken)) break; var count = Math.Min(bufLen - bufPos, charCount - len); LuaByteEncoding.Instance.GetChars( @@ -217,7 +223,7 @@ public void CopyBufferedBytes(IBufferWriter writer) } } else - len = ReadText(stream, resultBuffer.AsSpan(0, charCount)); + len = ReadText(stream, resultBuffer.AsSpan(0, charCount), cancellationToken); return len == 0 ? null : resultBuffer.AsSpan(0, len).ToString(); } @@ -227,10 +233,10 @@ public void CopyBufferedBytes(IBufferWriter writer) } } - int ReadText(Stream stream, Span destination) + int ReadText(Stream stream, Span destination, CancellationToken cancellationToken) { if (checkBom) - PrepareText(stream); + PrepareText(stream, cancellationToken); var len = 0; if (pendingCharacter >= 0) { @@ -240,8 +246,7 @@ int ReadText(Stream stream, Span destination) Span pair = stackalloc char[2]; while (len < destination.Length) { - var eof = bufPos >= bufLen && !ReadBuffer(stream); - if (eof && decoder is null) + if (bufPos >= bufLen && !ReadBuffer(stream, cancellationToken)) break; ReadOnlySpan byteSpan = new(buffer, bufPos, bufLen - bufPos); @@ -251,61 +256,85 @@ int ReadText(Stream stream, Span destination) var singleSlot = charSpan.Length == 1; if (singleSlot) { - if (!byteSpan.IsEmpty) + var first = byteSpan[0]; + if (first < 0x80) { - var sequenceLength = byteSpan[0] switch - { - >= 0xF0 and <= 0xF4 => 4, - >= 0xE0 and <= 0xEF => 3, - >= 0xC2 and <= 0xDF => 2, - _ => 1, - }; - byteSpan = byteSpan[..Math.Min(byteSpan.Length, sequenceLength)]; + destination[len++] = (char)first; + bufPos++; + continue; } + var sequenceLength = Utf8SequenceLength(first); + byteSpan = byteSpan[..Math.Min(byteSpan.Length, sequenceLength)]; charSpan = pair; } + // Keep incomplete sequences in the byte buffer, where other formats + // and seeks can account for them without depending on decoder state. + var completeLength = CompleteUtf8Length(byteSpan); + if (completeLength == 0) + { + if (ReadBuffer(stream, cancellationToken, bufLen - bufPos)) + continue; + byteSpan = buffer.AsSpan(bufPos, bufLen - bufPos); + } + else + byteSpan = byteSpan[..completeLength]; decoder ??= encoding.GetDecoder(); decoder.Convert( byteSpan, charSpan, - eof, + flush: true, out var bytesUsed, out var charsUsed, - out var completed + out _ ); - if (charsUsed > 0) + if (singleSlot) { - if (singleSlot) - { - destination[len++] = pair[0]; - if (charsUsed == 2) - pendingCharacter = pair[1]; - } - else - len += charsUsed; + destination[len++] = pair[0]; + if (charsUsed == 2) + pendingCharacter = pair[1]; } + else + len += charsUsed; bufPos += bytesUsed; - if (eof && completed) - break; } return len; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static int Utf8SequenceLength(byte first) => + first switch + { + >= 0xF0 and <= 0xF4 => 4, + >= 0xE0 and <= 0xEF => 3, + >= 0xC2 and <= 0xDF => 2, + _ => 1, + }; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static int CompleteUtf8Length(ReadOnlySpan bytes) + { + var start = bytes.Length - 1; + if (bytes[start] < 0x80) + return bytes.Length; + while (start > 0 && bytes.Length - start < 4 && (bytes[start] & 0xC0) == 0x80) + start--; + return Utf8SequenceLength(bytes[start]) > bytes.Length - start ? start : bytes.Length; + } + static bool IsAsciiSpace(char c) => c == ' ' || (c >= '\t' && c <= '\r'); public void Clear(bool atStart = false, bool keepPendingCharacter = false) { bufPos = 0; bufLen = 0; - decoder?.Reset(); if (!keepPendingCharacter) pendingCharacter = -1; checkBom = atStart && !byteStrings; } - public double? ReadNumber(Stream stream) + public double? ReadNumber(Stream stream, CancellationToken cancellationToken) { var resultBuffer = ArrayPool.Shared.Rent(64); // Numbers shouldn't be too long var len = 0; @@ -328,7 +357,7 @@ public void Clear(bool atStart = false, bool keepPendingCharacter = false) // Skip leading whitespace while (len == 0) { - var b = PeekByte(stream); + var b = PeekByte(stream, cancellationToken); if (b == -1) { return null; // EOF @@ -345,7 +374,7 @@ public void Clear(bool atStart = false, bool keepPendingCharacter = false) while (true) { - var b = PeekByte(stream); + var b = PeekByte(stream, cancellationToken); if (b < 0 || !scanner.TryConsume((char)b)) break; if (len == resultBuffer.Length) @@ -366,14 +395,14 @@ public void Clear(bool atStart = false, bool keepPendingCharacter = false) } } - public bool IsEndOfStream(Stream stream) + public bool IsEndOfStream(Stream stream, CancellationToken cancellationToken) { - return pendingCharacter < 0 && PeekByte(stream) < 0; + return pendingCharacter < 0 && PeekByte(stream, cancellationToken) < 0; } - int PeekByte(Stream stream) + int PeekByte(Stream stream, CancellationToken cancellationToken) { - if (bufPos >= bufLen && !ReadBuffer(stream)) + if (bufPos >= bufLen && !ReadBuffer(stream, cancellationToken)) return -1; return buffer[bufPos]; } diff --git a/tests/Lua.Tests/StreamCancellationTests.cs b/tests/Lua.Tests/StreamCancellationTests.cs new file mode 100644 index 00000000..c70954a8 --- /dev/null +++ b/tests/Lua.Tests/StreamCancellationTests.cs @@ -0,0 +1,78 @@ +using System.Text; +using Lua.IO; + +namespace Lua.Tests; + +[TestFixture(LuaStringMode.Utf16)] +[TestFixture(LuaStringMode.Bytes)] +public class StreamCancellationTests(LuaStringMode mode) +{ + static async Task Read(LuaStream stream, string format, CancellationToken token) + { + switch (format) + { + case "line": + await stream.ReadLineAsync(false, token); + break; + case "all": + await stream.ReadAllAsync(token); + break; + case "number": + await stream.ReadNumberAsync(token); + break; + case "byte": + await stream.ReadByteAsync(token); + break; + default: + await stream.ReadAsync(int.Parse(format), token); + break; + } + } + + [TestCase("line")] + [TestCase("all")] + [TestCase("number")] + [TestCase("byte")] + [TestCase("0")] + [TestCase("10000")] + public async Task AlreadyCancelledReadsPreserveBufferedData(string format) + { + using var stream = new LuaStream( + LuaFileOpenMode.Read, + new MemoryStream("first\n12 rest"u8.ToArray()), + mode + ); + Assert.That(await stream.ReadLineAsync(false, default), Is.EqualTo("first")); + using var source = new CancellationTokenSource(); + source.Cancel(); + Assert.ThrowsAsync(() => Read(stream, format, source.Token)); + Assert.That(await stream.ReadAllAsync(default), Is.EqualTo("12 rest")); + } + + [TestCase("line")] + [TestCase("all")] + [TestCase("number")] + [TestCase("10000")] + public void CancellationStopsReadsAtTheNextBufferBoundary(string format) + { + using var source = new CancellationTokenSource(); + var text = format == "number" ? new string(' ', 10000) + "12" : new string('x', 10000); + using var inner = new CancellingStream(Encoding.UTF8.GetBytes(text), source); + using var stream = new LuaStream(LuaFileOpenMode.Read, inner, mode); + Assert.ThrowsAsync(() => Read(stream, format, source.Token)); + Assert.That(inner.ReadCount, Is.EqualTo(1)); + } + + sealed class CancellingStream(byte[] bytes, CancellationTokenSource source) + : MemoryStream(bytes) + { + public int ReadCount { get; private set; } + + public override int Read(byte[] buffer, int offset, int count) + { + ReadCount++; + source.Cancel(); + return base.Read(buffer, offset, Math.Min(count, 100)); + } + } +} diff --git a/tests/Lua.Tests/Utf8StreamReadTests.cs b/tests/Lua.Tests/Utf8StreamReadTests.cs index 922c96c1..6657a4e6 100644 --- a/tests/Lua.Tests/Utf8StreamReadTests.cs +++ b/tests/Lua.Tests/Utf8StreamReadTests.cs @@ -1,3 +1,4 @@ +using System.Buffers; using System.Text; using Lua.IO; using Lua.Platforms; @@ -203,6 +204,60 @@ public async Task FailedSeekPreservesThePendingCharacter() Assert.That(first + await stream.ReadAllAsync(default), Is.EqualTo(Expected(bytes))); } + [Test, Combinatorial] + public async Task CountedReadsKeepUnreadUtf8PrefixesInTheByteBuffer( + [Values(3, 1024)] int chunkSize, + [Values("all", "line", "number", "bytes")] string format + ) + { + var prefix = new string('a', chunkSize - 1); + var tail = Encoding.UTF8.GetBytes("ế7\nrest"); + using var stream = Create(Encoding.UTF8.GetBytes(prefix).Concat(tail).ToArray(), chunkSize); + Assert.That(await stream.ReadAsync(prefix.Length, default), Is.EqualTo(prefix)); + if (format == "bytes") + { + var writer = new ArrayBufferWriter(); + await stream.ReadBytesAsync(writer, default); + Assert.That(writer.WrittenSpan.ToArray(), Is.EqualTo(tail)); + } + else if (format == "line") + { + Assert.That( + await stream.ReadLineAsync(false, default), + Is.EqualTo(Expected(tail[..^5])) + ); + Assert.That(await stream.ReadAllAsync(default), Is.EqualTo("rest")); + } + else + { + if (format == "number") + Assert.That(await stream.ReadNumberAsync(default), Is.Null); + Assert.That(await stream.ReadAllAsync(default), Is.EqualTo(Expected(tail))); + } + } + + [Test] + public async Task PositionQueriesDoNotSkipAnUnreadUtf8Prefix() + { + var prefix = new string('a', 1023); + var tail = Encoding.UTF8.GetBytes("ế7"); + using var stream = Create(Encoding.UTF8.GetBytes(prefix).Concat(tail).ToArray()); + Assert.That(await stream.ReadAsync(prefix.Length, default), Is.EqualTo(prefix)); + Assert.That(stream.Seek(SeekOrigin.Current, 0), Is.EqualTo(prefix.Length)); + Assert.That(await stream.ReadAllAsync(default), Is.EqualTo(Expected(tail))); + } + + [Test] + public async Task AnUnreadTruncatedUtf8PrefixIsNotEof() + { + byte[] bytes = [(byte)'a', (byte)'b', 0xe2]; + using var stream = Create(bytes); + Assert.That(await stream.ReadAsync(2, default), Is.EqualTo("ab")); + Assert.That(await stream.ReadAsync(0, default), Is.EqualTo("")); + Assert.That(await stream.ReadAllAsync(default), Is.EqualTo(Expected(bytes[2..]))); + Assert.That(await stream.ReadAsync(0, default), Is.Null); + } + sealed class ShortReadStream(byte[] bytes, int chunkSize) : MemoryStream(bytes) { public override int Read(byte[] buffer, int offset, int count) =>