Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -773,6 +773,10 @@ Paths and names passed to `ILuaFileSystem`, `ILuaFileLoader` and `ILuaModuleLoad

Files, temporary files and redirected standard I/O preserve bytes, including a BOM read as data. Raw line reads stop at LF and retain CR; `*L` also retains the LF. Invalid UTF-8 is replaced when decoding with `DecodeUtf8`, so keep binary data in byte strings or byte arrays. Passing unconverted .NET characters above U+00FF to raw output writes `?` for those characters.

Lua's `load` accepts source strings and reader functions, including readers that return a binary chunk in pieces. `string.dump` serializes a Lua function's prototype; it does not preserve captured upvalue values. The resulting string holds raw bytes in both string modes. Use byte-mode streams when writing it from Lua, or `LuaByteString.ToBytes` when writing it from C#. UTF-16 streams encode text as UTF-8 and cannot preserve arbitrary binary strings.

`Prototype.StringMode` records its string representation, so `ToBytecode` and `WriteBytecode` preserve byte-mode constants. `Prototype.FromBytecode(bytes, name, stringMode)` selects the representation when loading without a state; `LuaState.Load` uses the state's mode. Bytecode uses the Lua 5.2 function-header layout. Regenerate older Lua-CSharp chunks that used a different field order.

### Garbage Collection

Since Lua-CSharp is implemented in C#, it relies on .NET's garbage collector. As a result, memory management behavior differs from regular Lua.
Expand Down
4 changes: 4 additions & 0 deletions README_JA.md
Original file line number Diff line number Diff line change
Expand Up @@ -667,6 +667,10 @@ print(l)

Stringライブラリの関数はすべて文字列をUTF-16として扱う実装に変更されていることに注意してください。

`load`は文字列とreader関数に対応し、バイナリチャンクを分割して返すreaderも利用できます。`string.dump`はLua関数のプロトタイプを保存しますが、キャプチャしたupvalueの値は保存しません。戻り値はどちらの文字列モードでも生のバイト列です。Luaからファイルに書く場合はバイトモードのストリームを、C#から書く場合は`LuaByteString.ToBytes`を使用してください。UTF-16モードのストリームはUTF-8でテキストを書き込むため、任意のバイナリ文字列を保持できません。

`Prototype.StringMode`は文字列表現を保持し、`ToBytecode`と`WriteBytecode`でもバイトモードの定数を保持します。stateを使わずに読み込む場合は`Prototype.FromBytecode(bytes, name, stringMode)`で表現を指定できます。`LuaState.Load`はstateのモードを使用します。バイトコードはLua 5.2の関数ヘッダー順に従います。異なる順序で生成された古いLua-CSharpのチャンクは再生成してください。

### ガベージコレクション

Lua-CSharpはC#で実装されているため.NETのGCに依存しています。そのため、メモリ管理に関する動作が通常のLuaとは異なります。
Expand Down
120 changes: 120 additions & 0 deletions sandbox/Benchmark/BytecodeInteropVerification.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
using Lua;
using Lua.Platforms;
using Lua.Standard;
using NuLua;
using NuLua.Lua52;
using LuaFunction = Lua.LuaFunction;
using LuaTable = Lua.LuaTable;

static class BytecodeInteropVerification
{
static readonly ComparisonCase Closures = new(
"dump-closures",
"Bytecode",
"""
local value=10
local function make(step)
return function(...)
value=value+step
local first=...
return value+select('#',...)+first
end
end
local a,b=make(3),make(5)
assert(a(4)==18 and b(7,8)==27)
return a(20)
""",
42
);

static readonly ComparisonCase BinaryStrings = new(
"dump-binary-strings",
"Bytecode",
"""
local bytes='\0\128\255'
local utf8='\225\186\191'
local function check()
assert(#bytes==3 and bytes:byte(1)==0 and bytes:byte(2)==128 and bytes:byte(3)==255)
assert(#utf8==3 and utf8:byte(1)==225 and utf8:byte(2)==186 and utf8:byte(3)==191)
assert(''==string.sub(bytes,1,0))
assert(true and not false and nil==nil)
return 42
end
return check()
""",
42
);

public static async Task VerifyAsync()
{
var buffer = new byte[1024 * 1024];
foreach (
var workload in ComparisonWorkloads
.Cases.Concat(ComparisonWorkloads.CompilationCases)
.Append(Closures)
.Append(BinaryStrings)
.DistinctBy(c => c.Name)
)
{
using var native = Lua52State.Create();
native.OpenLibraries();
native.RegisterFunction(
"host_add",
(vm, args) =>
{
vm.PushNumber(args[0].Read<double>() + args[1].Read<double>());
return 1;
}
);
native.LoadString(workload.Source, "@bytecode");
if (!native.TryDump(1, false, buffer, out var written))
throw new InvalidOperationException(
$"Native dump exceeds verification buffer: {workload.Name}"
);
native.SetTop(0);

foreach (var mode in new[] { LuaStringMode.Utf16, LuaStringMode.Bytes })
{
// Arbitrary bytes have the same semantics as C Lua in byte mode.
if (workload == BinaryStrings && mode != LuaStringMode.Bytes)
continue;
using var state = LuaState.Create(
mode == LuaStringMode.Bytes
? LuaPlatform.DefaultByteStrings
: LuaPlatform.Default
);
state.OpenStandardLibraries();
state.Environment["host_add"] = new LuaFunction(
(context, _) =>
new(
context.Return(
context.GetArgument<double>(0) + context.GetArgument<double>(1)
)
)
);

var loaded = state.Load(buffer.AsSpan(0, written), "native", "b");
var results = await state.ExecuteAsync(loaded);
ComparisonWorkloads.Check(
workload,
$"NuLua dump -> Lua-CSharp {mode}",
results.Single().Read<double>()
);

var original = state.Load(workload.Source, "@bytecode");
var dump = state.Environment["string"].Read<LuaTable>()["dump"].Read<LuaFunction>();
var dumped = (await state.CallAsync(dump, [original])).Single().Read<string>();
native.LoadString("return assert(load(...,'managed','b'))()", "@bytecode");
native.PushString(LuaByteString.ToBytes(dumped));
native.Call(1, 1);
ComparisonWorkloads.Check(
workload,
$"Lua-CSharp {mode} dump -> NuLua",
native.ToNumber(-1)
);
native.SetTop(0);
}
Console.WriteLine($"PASS Bytecode {workload.Name, -20} NuLua <-> Lua-CSharp");
}
}
}
1 change: 1 addition & 0 deletions sandbox/Benchmark/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
$"NuLua Lua52 assembly: {typeof(NuLua.Lua52.Lua52State).Assembly.GetName().Version}"
);
InterpreterComparison.Verify();
await BytecodeInteropVerification.VerifyAsync();
return;
}

Expand Down
67 changes: 41 additions & 26 deletions src/Lua/CodeAnalysis/Compilation/Dump.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Runtime.InteropServices;
using System.Text;
using Lua.Internal;
using Lua.IO;
using Lua.Runtime;

namespace Lua.CodeAnalysis.Compilation;
Expand Down Expand Up @@ -96,8 +97,14 @@ public void Validate(ReadOnlySpan<char> name)
}
}

unsafe ref struct DumpState(IBufferWriter<byte> writer, bool reversedEndian)
unsafe ref struct DumpState(
IBufferWriter<byte> writer,
bool reversedEndian,
LuaStringMode stringMode
)
{
readonly Encoding encoding =
stringMode == LuaStringMode.Bytes ? LuaByteEncoding.Instance : Encoding.UTF8;
public readonly IBufferWriter<byte> Writer = writer;
Span<byte> unWritten;

Expand Down Expand Up @@ -142,8 +149,8 @@ void DumpFunction(Prototype prototype)
WriteInt(prototype.LineDefined); // 4
WriteInt(prototype.LastLineDefined); // 4
WriteByte((byte)prototype.ParameterCount); // 1
WriteByte((byte)prototype.MaxStackSize); // 1
WriteByte((byte)(prototype.HasVariableArguments ? 1 : 0)); // 1
WriteByte((byte)prototype.MaxStackSize); // 1
WriteIntSpanWithLength(MemoryMarshal.Cast<Instruction, int>(prototype.Code)); // 4
WriteConstants(prototype.Constants); // 4
WritePrototypes(prototype.ChildPrototypes); // 4
Expand Down Expand Up @@ -215,12 +222,8 @@ void WriteBool(bool v)

void WriteString(string v)
{
var bytes = Encoding.UTF8.GetBytes(v);
var len = bytes.Length;
if (bytes.Length != 0)
{
len++;
}
var bytes = encoding.GetBytes(v);
var len = bytes.Length + 1;

if (sizeof(IntPtr) == 8)
{
Expand All @@ -231,32 +234,38 @@ void WriteString(string v)
WriteInt(len);
}

if (len != 0)
{
Write(bytes);
WriteByte(0);
}
Write(bytes);
WriteByte(0);
}

void WriteConstants(ReadOnlySpan<LuaValue> constants)
{
WriteInt(constants.Length);
foreach (var c in constants)
{
WriteByte((byte)c.Type);
// Lua bytecode tags differ from LuaValueType for strings.
switch (c.Type)
{
case LuaValueType.Nil:
WriteByte(0);
break;
case LuaValueType.Boolean:
WriteByte(1);
WriteBool(c.UnsafeReadDouble() != 0);
break;
case LuaValueType.Number:
WriteByte(3);
WriteDouble(c.UnsafeReadDouble());
break;
case LuaValueType.String:
WriteByte(4);
WriteString(c.UnsafeRead<string>());
break;
default:
throw new ArgumentException(
"Unsupported bytecode constant type",
nameof(constants)
);
}
}
}
Expand Down Expand Up @@ -295,9 +304,12 @@ void WriteUpValues(ReadOnlySpan<UpValueDesc> upValues)
unsafe ref struct UndumpState(
ReadOnlySpan<byte> span,
ReadOnlySpan<char> name,
StringInternPool internPool
StringInternPool internPool,
LuaStringMode stringMode
)
{
readonly Encoding encoding =
stringMode == LuaStringMode.Bytes ? LuaByteEncoding.Instance : Encoding.UTF8;
public ReadOnlySpan<byte> Unread = span;
bool otherEndian;
int pointerSize;
Expand Down Expand Up @@ -412,8 +424,8 @@ Prototype UndumpFunction(int depth)
var lineDefined = ReadInt(); // 4
var lastLineDefined = ReadInt(); // 4
var parameterCount = ReadByte(); // 1
var maxStackSize = ReadByte(); // 1
var isVarArg = ReadBool(); // 1
var maxStackSize = ReadByte(); // 1
var codeLength = ReadCount(sizeof(int));
var code = new Instruction[codeLength];
ReadInToIntSpan(MemoryMarshal.Cast<Instruction, int>(code));
Expand All @@ -422,7 +434,7 @@ Prototype UndumpFunction(int depth)
var upValues = ReadUpValues();

// Debug
var source = ReadString();
var source = ReadString("=?");
var lineInfoLength = ReadCount(sizeof(int));
if (lineInfoLength != 0 && lineInfoLength != codeLength)
Throw("invalid line info in");
Expand Down Expand Up @@ -450,7 +462,10 @@ Prototype UndumpFunction(int depth)
lineInfo,
localVariables,
upValues
);
)
{
StringMode = stringMode,
};
new BytecodeVerifier(prototype, name).Verify();
return prototype;
}
Expand All @@ -465,12 +480,12 @@ void ReadInToIntSpan(Span<int> toWrite)
}
}

string ReadString()
string ReadString(string nullValue = "")
{
var serializedLength = pointerSize == 4 ? ReadInt() : ReadLong();
if (serializedLength == 0)
{
return "";
return nullValue;
}

if (serializedLength < 0 || serializedLength > Unread.Length)
Expand All @@ -489,7 +504,7 @@ string ReadString()
len <= 128
? stackalloc char[len]
: (charArrayPooled = ArrayPool<char>.Shared.Rent(len));
var count = Encoding.UTF8.GetChars(bytes, chars);
var count = encoding.GetChars(bytes, chars);
return internPool.Intern(chars[..count]);
}
finally
Expand All @@ -507,18 +522,18 @@ LuaValue[] ReadConstants()
var constants = new LuaValue[count];
for (var i = 0; i < count; i++)
{
var type = (LuaValueType)ReadByte();
var type = ReadByte();
switch (type)
{
case LuaValueType.Nil:
case 0:
break;
case LuaValueType.Boolean:
case 1:
constants[i] = ReadBool();
break;
case LuaValueType.Number:
case 3:
constants[i] = ReadDouble();
break;
case LuaValueType.String:
case 4:
constants[i] = ReadString();
break;
default:
Expand Down
16 changes: 12 additions & 4 deletions src/Lua/CodeAnalysis/Compilation/Parser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -979,7 +979,7 @@ public static Prototype Parse(LuaState l, TextReader r, string name)
f.Proto.IsVarArg = true;
f.Proto.LineDefined = 0;
p.MainFunction();
var prototype = f.Proto.CreatePrototypeAndRelease();
var prototype = f.Proto.CreatePrototypeAndRelease(l.Platform.StringMode);
f.Release();
return prototype;
}
Expand All @@ -990,7 +990,11 @@ public static void Dump(
bool useLittleEndian = true
)
{
DumpState state = new(writer, useLittleEndian ^ BitConverter.IsLittleEndian);
DumpState state = new(
writer,
useLittleEndian ^ BitConverter.IsLittleEndian,
prototype.StringMode
);
state.Dump(prototype);
}

Expand All @@ -1001,7 +1005,11 @@ public static byte[] Dump(Prototype prototype, bool useLittleEndian = true)
return writer.WrittenSpan.ToArray();
}

public static Prototype Undump(ReadOnlySpan<byte> span, ReadOnlySpan<char> name)
public static Prototype Undump(
ReadOnlySpan<byte> span,
ReadOnlySpan<char> name,
LuaStringMode stringMode = LuaStringMode.Utf16
)
{
if (name.Length > 0)
{
Expand All @@ -1014,7 +1022,7 @@ public static Prototype Undump(ReadOnlySpan<byte> span, ReadOnlySpan<char> name)
}

using var internPool = new StringInternPool(4);
UndumpState state = new(span, name, internPool);
UndumpState state = new(span, name, internPool, stringMode);
return state.Undump();
}
}
Loading
Loading