From 09eed8209af84c68578ed29f59f9cf2fd0deec7c Mon Sep 17 00:00:00 2001 From: BennyFranciscus <268274351+BennyFranciscus@users.noreply.github.com> Date: Mon, 16 Mar 2026 20:18:06 +0000 Subject: [PATCH 01/15] Add zerg: C# io_uring TCP server with manual HTTP parsing zerg is a low-level TCP framework built directly on Linux io_uring with zero-copy buffer rings, multishot accept/recv, and DEFER_TASKRUN optimizations. This entry builds a full HTTP/1.1 server on top using the PipeReader adapter for pipelining support. Interesting comparison with aspnet-minimal: same .NET runtime but radically different I/O strategy (io_uring vs Kestrel). - Language: C# - Engine: io_uring via liburing shim - Tests: baseline, noisy, pipelined, limited-conn, json, upload, compression, mixed --- frameworks/zerg/Dockerfile | 12 + frameworks/zerg/Program.cs | 673 ++++++++++++++++++++++++++ frameworks/zerg/README.md | 24 + frameworks/zerg/meta.json | 19 + frameworks/zerg/zerg-httparena.csproj | 16 + 5 files changed, 744 insertions(+) create mode 100644 frameworks/zerg/Dockerfile create mode 100644 frameworks/zerg/Program.cs create mode 100644 frameworks/zerg/README.md create mode 100644 frameworks/zerg/meta.json create mode 100644 frameworks/zerg/zerg-httparena.csproj diff --git a/frameworks/zerg/Dockerfile b/frameworks/zerg/Dockerfile new file mode 100644 index 0000000..98ab14e --- /dev/null +++ b/frameworks/zerg/Dockerfile @@ -0,0 +1,12 @@ +FROM mcr.microsoft.com/dotnet/sdk:10.0-preview AS build +WORKDIR /app +COPY zerg-httparena.csproj . +RUN dotnet restore +COPY . . +RUN dotnet publish -c Release -o out + +FROM mcr.microsoft.com/dotnet/runtime:10.0-preview +WORKDIR /app +COPY --from=build /app/out . +EXPOSE 8080 +ENTRYPOINT ["dotnet", "zerg-httparena.dll"] diff --git a/frameworks/zerg/Program.cs b/frameworks/zerg/Program.cs new file mode 100644 index 0000000..34d741c --- /dev/null +++ b/frameworks/zerg/Program.cs @@ -0,0 +1,673 @@ +using System.Buffers; +using System.Buffers.Text; +using System.IO.Compression; +using System.IO.Pipelines; +using System.Runtime.CompilerServices; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Data.Sqlite; +using zerg; +using zerg.Engine; +using zerg.Engine.Configs; +using Zerg.Core; + +// ── Data models ── + +public class DatasetItem +{ + public int Id { get; set; } + public string Name { get; set; } = ""; + public string Category { get; set; } = ""; + public double Price { get; set; } + public int Quantity { get; set; } + public bool Active { get; set; } + public List Tags { get; set; } = new(); + public RatingInfo Rating { get; set; } = new(); +} + +public class RatingInfo +{ + public double Score { get; set; } + public int Count { get; set; } +} + +public class ProcessedItem +{ + public int Id { get; set; } + public string Name { get; set; } = ""; + public string Category { get; set; } = ""; + public double Price { get; set; } + public int Quantity { get; set; } + public bool Active { get; set; } + public List Tags { get; set; } = new(); + public RatingInfo Rating { get; set; } = new(); + public double Total { get; set; } +} + +[JsonSourceGenerationOptions( + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + GenerationMode = JsonSourceGenerationMode.Default)] +[JsonSerializable(typeof(JsonResponse))] +[JsonSerializable(typeof(DbResponse))] +[JsonSerializable(typeof(List))] +partial class AppJsonContext : JsonSerializerContext { } + +public class JsonResponse +{ + public List Items { get; set; } = new(); + public int Count { get; set; } +} + +public class DbResponse +{ + public List Items { get; set; } = new(); + public int Count { get; set; } +} + +public class DbItem +{ + public int Id { get; set; } + public string Name { get; set; } = ""; + public string Category { get; set; } = ""; + public double Price { get; set; } + public int Quantity { get; set; } + public bool Active { get; set; } + public List Tags { get; set; } = new(); + public DbRating Rating { get; set; } = new(); +} + +public class DbRating +{ + public double Score { get; set; } + public int Count { get; set; } +} + +// ── Shared app data ── + +static class AppData +{ + public static List Dataset = new(); + public static byte[] JsonCache = Array.Empty(); + public static byte[] LargeJsonCache = Array.Empty(); + public static Dictionary StaticFiles = new(); + public static SqliteConnection? Db; + + public static readonly JsonSerializerOptions JsonOpts = new() + { + PropertyNameCaseInsensitive = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }; + + public static void Load() + { + // Dataset + var path = Environment.GetEnvironmentVariable("DATASET_PATH") ?? "/data/dataset.json"; + if (File.Exists(path)) + { + Dataset = JsonSerializer.Deserialize>(File.ReadAllText(path), JsonOpts) ?? new(); + JsonCache = BuildJsonCache(Dataset); + } + + // Large dataset for compression + var largePath = "/data/dataset-large.json"; + if (File.Exists(largePath)) + { + var largeItems = JsonSerializer.Deserialize>(File.ReadAllText(largePath), JsonOpts) ?? new(); + LargeJsonCache = BuildJsonCache(largeItems); + } + + // Static files + if (Directory.Exists("/data/static")) + { + var mimeTypes = new Dictionary + { + {".css","text/css"},{".js","application/javascript"},{".html","text/html"}, + {".woff2","font/woff2"},{".svg","image/svg+xml"},{".webp","image/webp"},{".json","application/json"} + }; + foreach (var file in Directory.GetFiles("/data/static")) + { + var name = Path.GetFileName(file); + var ext = Path.GetExtension(file); + var ct = mimeTypes.GetValueOrDefault(ext, "application/octet-stream"); + StaticFiles[name] = (File.ReadAllBytes(file), ct); + } + } + + // Database + var dbPath = "/data/benchmark.db"; + if (File.Exists(dbPath)) + { + Db = new SqliteConnection($"Data Source={dbPath};Mode=ReadOnly"); + Db.Open(); + using var pragma = Db.CreateCommand(); + pragma.CommandText = "PRAGMA mmap_size=268435456"; + pragma.ExecuteNonQuery(); + } + } + + static byte[] BuildJsonCache(List items) + { + var processed = items.Select(d => new ProcessedItem + { + Id = d.Id, Name = d.Name, Category = d.Category, + Price = d.Price, Quantity = d.Quantity, Active = d.Active, + Tags = d.Tags, Rating = d.Rating, + Total = Math.Round(d.Price * d.Quantity, 2) + }).ToList(); + var resp = new JsonResponse { Items = processed, Count = processed.Count }; + return JsonSerializer.SerializeToUtf8Bytes(resp, AppJsonContext.Default.JsonResponse); + } +} + +// ── Date header (updated every second) ── + +static class DateHelper +{ + private const int PrefixLen = 6; + private const int DateLen = 29; + private static byte[] _master = new byte[PrefixLen + DateLen + 4]; + private static byte[] _scratch = new byte[PrefixLen + DateLen + 4]; + private static readonly Timer _timer; + + static DateHelper() + { + "Date: "u8.CopyTo(_master); + "Date: "u8.CopyTo(_scratch); + _master[PrefixLen + DateLen] = (byte)'\r'; + _master[PrefixLen + DateLen + 1] = (byte)'\n'; + _master[PrefixLen + DateLen + 2] = (byte)'\r'; + _master[PrefixLen + DateLen + 3] = (byte)'\n'; + _scratch[PrefixLen + DateLen] = (byte)'\r'; + _scratch[PrefixLen + DateLen + 1] = (byte)'\n'; + _scratch[PrefixLen + DateLen + 2] = (byte)'\r'; + _scratch[PrefixLen + DateLen + 3] = (byte)'\n'; + SetDate(DateTimeOffset.UtcNow); + _timer = new Timer(_ => SetDate(DateTimeOffset.UtcNow), null, 1000, 1000); + } + + private static void SetDate(DateTimeOffset value) + { + lock (_scratch) + { + Utf8Formatter.TryFormat(value, _scratch.AsSpan(PrefixLen), out _, 'R'); + (_scratch, _master) = (_master, _scratch); + } + } + + public static ReadOnlySpan HeaderBytes => _master; +} + +// ── HTTP request parsing ── + +readonly struct HttpRequest +{ + public readonly ReadOnlyMemory Method; + public readonly ReadOnlyMemory Path; + public readonly ReadOnlyMemory Query; + public readonly ReadOnlyMemory Body; + public readonly int TotalLength; + public readonly int ContentLength; + public readonly bool AcceptsGzip; + + public HttpRequest(ReadOnlyMemory method, ReadOnlyMemory path, + ReadOnlyMemory query, ReadOnlyMemory body, + int totalLength, int contentLength, bool acceptsGzip) + { + Method = method; Path = path; Query = query; Body = body; + TotalLength = totalLength; ContentLength = contentLength; + AcceptsGzip = acceptsGzip; + } +} + +static class HttpParser +{ + // Returns null if buffer doesn't contain a complete request yet + public static HttpRequest? TryParse(ReadOnlySequence buffer) + { + // Find end of headers + var span = buffer.IsSingleSegment ? buffer.FirstSpan : buffer.ToArray().AsSpan(); + return TryParse(span, buffer); + } + + public static HttpRequest? TryParse(ReadOnlySpan span, ReadOnlySequence buffer) + { + int headerEnd = span.IndexOf("\r\n\r\n"u8); + if (headerEnd < 0) return null; + + int headersLen = headerEnd + 4; + + // Parse request line: METHOD SP PATH[?QUERY] SP HTTP/x.x\r\n + int firstSpace = span.IndexOf((byte)' '); + if (firstSpace < 0) return null; + var method = span[..firstSpace]; + + int pathStart = firstSpace + 1; + int secondSpace = span[pathStart..].IndexOf((byte)' '); + if (secondSpace < 0) return null; + var uri = span[pathStart..(pathStart + secondSpace)]; + + ReadOnlySpan path; + ReadOnlySpan query = default; + int qmark = uri.IndexOf((byte)'?'); + if (qmark >= 0) + { + path = uri[..qmark]; + query = uri[(qmark + 1)..]; + } + else + { + path = uri; + } + + // Parse Content-Length if present + int contentLength = 0; + bool acceptsGzip = false; + var headers = span[..headerEnd]; + int pos = 0; + while (pos < headers.Length) + { + int lineEnd = headers[pos..].IndexOf("\r\n"u8); + if (lineEnd < 0) break; + var line = headers[pos..(pos + lineEnd)]; + pos += lineEnd + 2; + + if (line.Length > 16 && + (line[0] == (byte)'C' || line[0] == (byte)'c') && + (line[8] == (byte)'L' || line[8] == (byte)'l')) + { + // Content-Length: + int colon = line.IndexOf((byte)':'); + if (colon >= 0) + { + var val = line[(colon + 1)..]; + // Trim leading space + while (val.Length > 0 && val[0] == (byte)' ') val = val[1..]; + if (Utf8Parser.TryParse(val, out int cl, out _)) + contentLength = cl; + } + } + else if (line.Length > 15 && + (line[0] == (byte)'A' || line[0] == (byte)'a') && + (line[7] == (byte)'E' || line[7] == (byte)'e')) + { + // Accept-Encoding: + if (line.IndexOf("gzip"u8) >= 0) + acceptsGzip = true; + } + } + + int totalLen = headersLen + contentLength; + if (span.Length < totalLen) return null; // body not yet complete + + ReadOnlyMemory body = default; + if (contentLength > 0) + { + body = buffer.Slice(headersLen, contentLength).ToArray(); + } + + return new HttpRequest( + method.ToArray(), + path.ToArray(), + query.Length > 0 ? query.ToArray() : ReadOnlyMemory.Empty, + body, + totalLen, + contentLength, + acceptsGzip); + } +} + +// ── Response writing ── + +static class HttpResponse +{ + static ReadOnlySpan ServerHeader => "Server: zerg\r\n"u8; + + public static void WriteText(Connection conn, ReadOnlySpan body, int statusCode = 200) + { + Span lenBuf = stackalloc byte[16]; + Utf8Formatter.TryFormat(body.Length, lenBuf, out int lenWritten); + + conn.Write(statusCode == 200 + ? "HTTP/1.1 200 OK\r\n"u8 + : "HTTP/1.1 404 Not Found\r\n"u8); + conn.Write(ServerHeader); + conn.Write("Content-Type: text/plain\r\nContent-Length: "u8); + conn.Write(lenBuf[..lenWritten]); + conn.Write("\r\n"u8); + conn.Write(DateHelper.HeaderBytes); + conn.Write(body); + } + + public static void WriteJson(Connection conn, byte[] body, bool compress, bool acceptsGzip) + { + if (compress && acceptsGzip && body.Length > 256) + { + using var ms = new MemoryStream(); + using (var gz = new GZipStream(ms, CompressionLevel.Fastest, true)) + gz.Write(body); + var compressed = ms.ToArray(); + + Span lenBuf = stackalloc byte[16]; + Utf8Formatter.TryFormat(compressed.Length, lenBuf, out int lenWritten); + + conn.Write("HTTP/1.1 200 OK\r\n"u8); + conn.Write(ServerHeader); + conn.Write("Content-Type: application/json\r\nContent-Encoding: gzip\r\nContent-Length: "u8); + conn.Write(lenBuf[..lenWritten]); + conn.Write("\r\n"u8); + conn.Write(DateHelper.HeaderBytes); + conn.Write(compressed); + } + else + { + Span lenBuf = stackalloc byte[16]; + Utf8Formatter.TryFormat(body.Length, lenBuf, out int lenWritten); + + conn.Write("HTTP/1.1 200 OK\r\n"u8); + conn.Write(ServerHeader); + conn.Write("Content-Type: application/json\r\nContent-Length: "u8); + conn.Write(lenBuf[..lenWritten]); + conn.Write("\r\n"u8); + conn.Write(DateHelper.HeaderBytes); + conn.Write(body); + } + } + + public static void WriteBytes(Connection conn, byte[] body, string contentType) + { + Span lenBuf = stackalloc byte[16]; + Utf8Formatter.TryFormat(body.Length, lenBuf, out int lenWritten); + + conn.Write("HTTP/1.1 200 OK\r\n"u8); + conn.Write(ServerHeader); + conn.Write("Content-Type: "u8); + conn.Write(Encoding.UTF8.GetBytes(contentType)); + conn.Write("\r\nContent-Length: "u8); + conn.Write(lenBuf[..lenWritten]); + conn.Write("\r\n"u8); + conn.Write(DateHelper.HeaderBytes); + conn.Write(body); + } + + public static void Write404(Connection conn) + { + conn.Write("HTTP/1.1 404 Not Found\r\n"u8); + conn.Write(ServerHeader); + conn.Write("Content-Length: 9\r\n"u8); + conn.Write(DateHelper.HeaderBytes); + conn.Write("Not Found"u8); + } + + public static void Write500(Connection conn, string msg) + { + var body = Encoding.UTF8.GetBytes(msg); + Span lenBuf = stackalloc byte[16]; + Utf8Formatter.TryFormat(body.Length, lenBuf, out int lenWritten); + + conn.Write("HTTP/1.1 500 Internal Server Error\r\n"u8); + conn.Write(ServerHeader); + conn.Write("Content-Type: text/plain\r\nContent-Length: "u8); + conn.Write(lenBuf[..lenWritten]); + conn.Write("\r\n"u8); + conn.Write(DateHelper.HeaderBytes); + conn.Write(body); + } +} + +// ── Route handling ── + +static class Router +{ + public static void Handle(Connection conn, in HttpRequest req) + { + var path = req.Path.Span; + + if (path.SequenceEqual("/pipeline"u8)) + HandlePipeline(conn); + else if (path.SequenceEqual("/baseline11"u8)) + HandleBaseline11(conn, req); + else if (path.SequenceEqual("/baseline2"u8)) + HandleBaseline2(conn, req); + else if (path.SequenceEqual("/json"u8)) + HandleJson(conn); + else if (path.SequenceEqual("/compression"u8)) + HandleCompression(conn, req); + else if (path.SequenceEqual("/db"u8)) + HandleDb(conn, req); + else if (path.SequenceEqual("/upload"u8)) + HandleUpload(conn, req); + else if (path.Length > 8 && path[..8].SequenceEqual("/static/"u8)) + HandleStatic(conn, path); + else + HttpResponse.Write404(conn); + } + + static void HandlePipeline(Connection conn) + { + HttpResponse.WriteText(conn, "ok"u8); + } + + static void HandleBaseline11(Connection conn, in HttpRequest req) + { + long sum = SumQuery(req.Query.Span); + + // POST: add body value + if (req.Method.Span.SequenceEqual("POST"u8) && req.Body.Length > 0) + { + var bodyStr = Encoding.UTF8.GetString(req.Body.Span).Trim(); + if (long.TryParse(bodyStr, out long bval)) + sum += bval; + } + + HttpResponse.WriteText(conn, Encoding.UTF8.GetBytes(sum.ToString())); + } + + static void HandleBaseline2(Connection conn, in HttpRequest req) + { + long sum = SumQuery(req.Query.Span); + HttpResponse.WriteText(conn, Encoding.UTF8.GetBytes(sum.ToString())); + } + + static void HandleJson(Connection conn) + { + if (AppData.JsonCache.Length == 0) + { + HttpResponse.Write500(conn, "Dataset not loaded"); + return; + } + HttpResponse.WriteJson(conn, AppData.JsonCache, false, false); + } + + static void HandleCompression(Connection conn, in HttpRequest req) + { + if (AppData.LargeJsonCache.Length == 0) + { + HttpResponse.Write500(conn, "Dataset not loaded"); + return; + } + HttpResponse.WriteJson(conn, AppData.LargeJsonCache, true, req.AcceptsGzip); + } + + static void HandleDb(Connection conn, in HttpRequest req) + { + if (AppData.Db == null) + { + HttpResponse.Write500(conn, "Database not available"); + return; + } + + double min = 10, max = 50; + var query = req.Query.Span; + if (query.Length > 0) + { + var qs = Encoding.UTF8.GetString(query); + foreach (var pair in qs.Split('&')) + { + if (pair.StartsWith("min=") && double.TryParse(pair[4..], out double pmin)) + min = pmin; + else if (pair.StartsWith("max=") && double.TryParse(pair[4..], out double pmax)) + max = pmax; + } + } + + lock (AppData.Db) + { + using var cmd = AppData.Db.CreateCommand(); + cmd.CommandText = "SELECT id, name, category, price, quantity, active, tags, rating_score, rating_count FROM items WHERE price BETWEEN @min AND @max LIMIT 50"; + cmd.Parameters.AddWithValue("@min", min); + cmd.Parameters.AddWithValue("@max", max); + using var reader = cmd.ExecuteReader(); + + var items = new List(); + while (reader.Read()) + { + items.Add(new DbItem + { + Id = reader.GetInt32(0), + Name = reader.GetString(1), + Category = reader.GetString(2), + Price = reader.GetDouble(3), + Quantity = reader.GetInt32(4), + Active = reader.GetInt32(5) == 1, + Tags = JsonSerializer.Deserialize>(reader.GetString(6)) ?? new(), + Rating = new DbRating { Score = reader.GetDouble(7), Count = reader.GetInt32(8) } + }); + } + + var resp = new DbResponse { Items = items, Count = items.Count }; + var body = JsonSerializer.SerializeToUtf8Bytes(resp, AppJsonContext.Default.DbResponse); + HttpResponse.WriteJson(conn, body, false, false); + } + } + + static void HandleUpload(Connection conn, in HttpRequest req) + { + HttpResponse.WriteText(conn, Encoding.UTF8.GetBytes(req.ContentLength.ToString())); + } + + static void HandleStatic(Connection conn, ReadOnlySpan path) + { + var filename = Encoding.UTF8.GetString(path[8..]); + if (AppData.StaticFiles.TryGetValue(filename, out var sf)) + HttpResponse.WriteBytes(conn, sf.Data, sf.ContentType); + else + HttpResponse.Write404(conn); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static long SumQuery(ReadOnlySpan query) + { + if (query.IsEmpty) return 0; + long sum = 0; + var qs = Encoding.UTF8.GetString(query); + foreach (var pair in qs.Split('&')) + { + var eq = pair.IndexOf('='); + if (eq >= 0 && long.TryParse(pair[(eq + 1)..], out long n)) + sum += n; + } + return sum; + } +} + +// ── Connection handler ── + +static class ConnectionHandler +{ + internal static async Task HandleAsync(Connection connection) + { + var reader = new ConnectionPipeReader(connection); + + try + { + while (true) + { + var result = await reader.ReadAsync(); + if (result.IsCompleted || result.IsCanceled) + break; + + var buffer = result.Buffer; + bool wrote = false; + + while (buffer.Length > 0) + { + var req = HttpParser.TryParse(buffer); + if (req == null) break; + + Router.Handle(connection, req.Value); + wrote = true; + buffer = buffer.Slice(req.Value.TotalLength); + } + + reader.AdvanceTo(buffer.Start, buffer.End); + + if (wrote) + await connection.FlushAsync(); + } + } + catch (Exception ex) + { + Console.Error.WriteLine($"Connection error: {ex.Message}"); + } + + reader.Complete(); + } +} + +// ── Entry point ── + +AppData.Load(); + +int reactorCount = Environment.ProcessorCount; +if (args.Length > 0 && int.TryParse(args[0], out int rc)) + reactorCount = rc; + +Console.WriteLine($"zerg HttpArena server starting on :8080 with {reactorCount} reactors"); + +var engine = new Engine(new EngineOptions +{ + Ip = "0.0.0.0", + Port = 8080, + Backlog = 65535, + ReactorCount = reactorCount, + AcceptorConfig = new AcceptorConfig( + RingFlags: 0, + SqCpuThread: -1, + SqThreadIdleMs: 100, + RingEntries: 8 * 1024, + BatchSqes: 4096, + CqTimeout: 100_000_000, + IPVersion: IPVersion.IPv6DualStack + ), + ReactorConfigs = Enumerable.Range(0, reactorCount).Select(_ => new ReactorConfig( + RingFlags: (1u << 12) | (1u << 13), // SINGLE_ISSUER | DEFER_TASKRUN + SqCpuThread: -1, + SqThreadIdleMs: 100, + RingEntries: 8 * 1024, + RecvBufferSize: 16 * 1024, + BufferRingEntries: 16 * 1024, + BatchCqes: 4096, + MaxConnectionsPerReactor: 8 * 1024, + CqTimeout: 1_000_000, + ConnectionBufferRingEntries: 32, + IncrementalBufferConsumption: false + )).ToArray() +}); + +engine.Listen(); + +var cts = new CancellationTokenSource(); + +try +{ + while (engine.ServerRunning) + { + var connection = await engine.AcceptAsync(cts.Token); + if (connection is null) continue; + _ = ConnectionHandler.HandleAsync(connection); + } +} +catch (OperationCanceledException) { } + +Console.WriteLine("Server stopped."); diff --git a/frameworks/zerg/README.md b/frameworks/zerg/README.md new file mode 100644 index 0000000..135a92b --- /dev/null +++ b/frameworks/zerg/README.md @@ -0,0 +1,24 @@ +# zerg — C# io_uring TCP server + +[zerg](https://github.com/MDA2AV/zerg) is a low-level TCP server framework for C# built directly on Linux `io_uring`. It provides zero-copy buffer rings, multishot accept/recv, and `DEFER_TASKRUN`/`SINGLE_ISSUER` optimizations — no HTTP abstractions, just raw TCP with async/await. + +This entry builds a full HTTP/1.1 server on top of zerg using its `ConnectionPipeReader` adapter for robust buffer management, with manual HTTP parsing and routing. + +## What makes it interesting + +- **io_uring native:** Direct ring submission via liburing shim — no epoll, no kqueue +- **Zero-copy reads:** Provided buffer rings let the kernel write directly into pre-allocated memory +- **C# without Kestrel:** Shows what .NET can do when you bypass the ASP.NET stack entirely +- **Same language, different I/O:** Direct comparison with `aspnet-minimal` (Kestrel) — same runtime, radically different I/O strategy + +## Configuration + +- Reactor count = CPU count (one io_uring instance per reactor thread) +- 16KB recv buffers, 16K buffer ring entries per reactor +- SINGLE_ISSUER + DEFER_TASKRUN ring flags for minimal kernel transitions +- PipeReader adapter for correct HTTP pipelining support + +## Requirements + +- Linux kernel 6.1+ (io_uring provided buffers) +- .NET 10 preview diff --git a/frameworks/zerg/meta.json b/frameworks/zerg/meta.json new file mode 100644 index 0000000..2c18c5c --- /dev/null +++ b/frameworks/zerg/meta.json @@ -0,0 +1,19 @@ +{ + "display_name": "zerg", + "language": "C#", + "type": "framework", + "engine": "io_uring", + "description": "Raw C# TCP server on Linux io_uring via zerg, with zero-copy buffer rings and manual HTTP parsing.", + "repo": "https://github.com/MDA2AV/zerg", + "enabled": true, + "tests": [ + "baseline", + "noisy", + "pipelined", + "limited-conn", + "json", + "upload", + "compression", + "mixed" + ] +} diff --git a/frameworks/zerg/zerg-httparena.csproj b/frameworks/zerg/zerg-httparena.csproj new file mode 100644 index 0000000..41e6f3b --- /dev/null +++ b/frameworks/zerg/zerg-httparena.csproj @@ -0,0 +1,16 @@ + + + Exe + net10.0 + enable + enable + true + true + Speed + true + + + + + + From 59bafde9e06f7562a4e3fa8f0f3979075f0d56a5 Mon Sep 17 00:00:00 2001 From: BennyFranciscus <268274351+BennyFranciscus@users.noreply.github.com> Date: Mon, 16 Mar 2026 20:24:11 +0000 Subject: [PATCH 02/15] fix: wrap entry point in Program.Main to fix CS8803 build error Top-level statements must precede namespace and type declarations in C#. Converted to explicit async Main method. --- frameworks/zerg/Program.cs | 104 ++++++++++++++++++++----------------- 1 file changed, 55 insertions(+), 49 deletions(-) diff --git a/frameworks/zerg/Program.cs b/frameworks/zerg/Program.cs index 34d741c..2cdf9c9 100644 --- a/frameworks/zerg/Program.cs +++ b/frameworks/zerg/Program.cs @@ -617,57 +617,63 @@ internal static async Task HandleAsync(Connection connection) // ── Entry point ── -AppData.Load(); +static class Program +{ + static async Task Main(string[] args) + { + AppData.Load(); -int reactorCount = Environment.ProcessorCount; -if (args.Length > 0 && int.TryParse(args[0], out int rc)) - reactorCount = rc; + int reactorCount = Environment.ProcessorCount; + if (args.Length > 0 && int.TryParse(args[0], out int rc)) + reactorCount = rc; -Console.WriteLine($"zerg HttpArena server starting on :8080 with {reactorCount} reactors"); + Console.WriteLine($"zerg HttpArena server starting on :8080 with {reactorCount} reactors"); -var engine = new Engine(new EngineOptions -{ - Ip = "0.0.0.0", - Port = 8080, - Backlog = 65535, - ReactorCount = reactorCount, - AcceptorConfig = new AcceptorConfig( - RingFlags: 0, - SqCpuThread: -1, - SqThreadIdleMs: 100, - RingEntries: 8 * 1024, - BatchSqes: 4096, - CqTimeout: 100_000_000, - IPVersion: IPVersion.IPv6DualStack - ), - ReactorConfigs = Enumerable.Range(0, reactorCount).Select(_ => new ReactorConfig( - RingFlags: (1u << 12) | (1u << 13), // SINGLE_ISSUER | DEFER_TASKRUN - SqCpuThread: -1, - SqThreadIdleMs: 100, - RingEntries: 8 * 1024, - RecvBufferSize: 16 * 1024, - BufferRingEntries: 16 * 1024, - BatchCqes: 4096, - MaxConnectionsPerReactor: 8 * 1024, - CqTimeout: 1_000_000, - ConnectionBufferRingEntries: 32, - IncrementalBufferConsumption: false - )).ToArray() -}); - -engine.Listen(); - -var cts = new CancellationTokenSource(); - -try -{ - while (engine.ServerRunning) - { - var connection = await engine.AcceptAsync(cts.Token); - if (connection is null) continue; - _ = ConnectionHandler.HandleAsync(connection); + var engine = new Engine(new EngineOptions + { + Ip = "0.0.0.0", + Port = 8080, + Backlog = 65535, + ReactorCount = reactorCount, + AcceptorConfig = new AcceptorConfig( + RingFlags: 0, + SqCpuThread: -1, + SqThreadIdleMs: 100, + RingEntries: 8 * 1024, + BatchSqes: 4096, + CqTimeout: 100_000_000, + IPVersion: IPVersion.IPv6DualStack + ), + ReactorConfigs = Enumerable.Range(0, reactorCount).Select(_ => new ReactorConfig( + RingFlags: (1u << 12) | (1u << 13), // SINGLE_ISSUER | DEFER_TASKRUN + SqCpuThread: -1, + SqThreadIdleMs: 100, + RingEntries: 8 * 1024, + RecvBufferSize: 16 * 1024, + BufferRingEntries: 16 * 1024, + BatchCqes: 4096, + MaxConnectionsPerReactor: 8 * 1024, + CqTimeout: 1_000_000, + ConnectionBufferRingEntries: 32, + IncrementalBufferConsumption: false + )).ToArray() + }); + + engine.Listen(); + + var cts = new CancellationTokenSource(); + + try + { + while (engine.ServerRunning) + { + var connection = await engine.AcceptAsync(cts.Token); + if (connection is null) continue; + _ = ConnectionHandler.HandleAsync(connection); + } + } + catch (OperationCanceledException) { } + + Console.WriteLine("Server stopped."); } } -catch (OperationCanceledException) { } - -Console.WriteLine("Server stopped."); From e839b4640bd731ad2879e43cf2735905d95f7f17 Mon Sep 17 00:00:00 2001 From: BennyFranciscus <268274351+BennyFranciscus@users.noreply.github.com> Date: Mon, 16 Mar 2026 20:30:07 +0000 Subject: [PATCH 03/15] =?UTF-8?q?fix:=20correct=20namespace=20casing=20Zer?= =?UTF-8?q?g.Core=20=E2=86=92=20zerg.Core?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frameworks/zerg/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frameworks/zerg/Program.cs b/frameworks/zerg/Program.cs index 2cdf9c9..3d5a132 100644 --- a/frameworks/zerg/Program.cs +++ b/frameworks/zerg/Program.cs @@ -10,7 +10,7 @@ using zerg; using zerg.Engine; using zerg.Engine.Configs; -using Zerg.Core; +using zerg.Core; // ── Data models ── From 87191e853013e6885e4d74e30d26a1c5f716e23b Mon Sep 17 00:00:00 2001 From: BennyFranciscus <268274351+BennyFranciscus@users.noreply.github.com> Date: Mon, 16 Mar 2026 21:09:52 +0000 Subject: [PATCH 04/15] =?UTF-8?q?fix(zerg):=20remove=20unused=20zerg.Core?= =?UTF-8?q?=20import=20=E2=80=94=20namespace=20doesn't=20exist=20in=20zerg?= =?UTF-8?q?=200.5.23?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The zerg.Core namespace was never used (no types from it referenced). All required types (Connection, ConnectionPipeReader, etc.) live in zerg.Engine which is already imported. Fixes CI build failure. --- frameworks/zerg/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frameworks/zerg/Program.cs b/frameworks/zerg/Program.cs index 3d5a132..1f512ee 100644 --- a/frameworks/zerg/Program.cs +++ b/frameworks/zerg/Program.cs @@ -10,7 +10,7 @@ using zerg; using zerg.Engine; using zerg.Engine.Configs; -using zerg.Core; + // ── Data models ── From 1f62dff8d905d54954ce0d7d921317ad39e638ce Mon Sep 17 00:00:00 2001 From: BennyFranciscus <268274351+BennyFranciscus@users.noreply.github.com> Date: Mon, 16 Mar 2026 21:24:32 +0000 Subject: [PATCH 05/15] =?UTF-8?q?fix(zerg):=20build=20from=20source=20?= =?UTF-8?q?=E2=80=94=20NuGet=20package=20missing=20core.dll?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The zerg NuGet package (0.5.23) only ships zerg.dll but not core.dll, which contains ConnectionBase. This causes CS0012 at compile time. Fix: clone zerg from source and use ProjectReference instead of PackageReference, so both zerg.dll and core.dll are built together. --- frameworks/zerg/Dockerfile | 7 +++++-- frameworks/zerg/zerg-httparena.csproj | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/frameworks/zerg/Dockerfile b/frameworks/zerg/Dockerfile index 98ab14e..ff446fa 100644 --- a/frameworks/zerg/Dockerfile +++ b/frameworks/zerg/Dockerfile @@ -1,7 +1,10 @@ FROM mcr.microsoft.com/dotnet/sdk:10.0-preview AS build + +# Clone zerg from source (NuGet package missing core.dll — ConnectionBase lives there) +RUN apt-get update && apt-get install -y --no-install-recommends git && \ + git clone --depth 1 https://github.com/MDA2AV/zerg.git /src/zerg-repo + WORKDIR /app -COPY zerg-httparena.csproj . -RUN dotnet restore COPY . . RUN dotnet publish -c Release -o out diff --git a/frameworks/zerg/zerg-httparena.csproj b/frameworks/zerg/zerg-httparena.csproj index 41e6f3b..9e1b8ec 100644 --- a/frameworks/zerg/zerg-httparena.csproj +++ b/frameworks/zerg/zerg-httparena.csproj @@ -10,7 +10,8 @@ true - + + From 7ffb7a62e2d458c0f78f98db132d007802fdcca6 Mon Sep 17 00:00:00 2001 From: BennyFranciscus <268274351+BennyFranciscus@users.noreply.github.com> Date: Mon, 16 Mar 2026 21:55:13 +0000 Subject: [PATCH 06/15] fix: update to zerg 0.5.24 NuGet, fix Write() ambiguity and ConnectionPipeReader namespace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Switch back to PackageReference (zerg 0.5.24 ships core.dll properly) - Remove source build from Dockerfile - Add 'using Zerg.Core' for ConnectionPipeReader (namespace changed in 0.5.24) - Add explicit ReadOnlySpan casts for byte[] → Write() calls (ambiguous overloads) --- frameworks/zerg/Dockerfile | 5 ----- frameworks/zerg/Program.cs | 11 ++++++----- frameworks/zerg/zerg-httparena.csproj | 3 +-- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/frameworks/zerg/Dockerfile b/frameworks/zerg/Dockerfile index ff446fa..3603440 100644 --- a/frameworks/zerg/Dockerfile +++ b/frameworks/zerg/Dockerfile @@ -1,9 +1,4 @@ FROM mcr.microsoft.com/dotnet/sdk:10.0-preview AS build - -# Clone zerg from source (NuGet package missing core.dll — ConnectionBase lives there) -RUN apt-get update && apt-get install -y --no-install-recommends git && \ - git clone --depth 1 https://github.com/MDA2AV/zerg.git /src/zerg-repo - WORKDIR /app COPY . . RUN dotnet publish -c Release -o out diff --git a/frameworks/zerg/Program.cs b/frameworks/zerg/Program.cs index 1f512ee..7078aef 100644 --- a/frameworks/zerg/Program.cs +++ b/frameworks/zerg/Program.cs @@ -7,6 +7,7 @@ using System.Text.Json; using System.Text.Json.Serialization; using Microsoft.Data.Sqlite; +using Zerg.Core; using zerg; using zerg.Engine; using zerg.Engine.Configs; @@ -357,7 +358,7 @@ public static void WriteJson(Connection conn, byte[] body, bool compress, bool a conn.Write(lenBuf[..lenWritten]); conn.Write("\r\n"u8); conn.Write(DateHelper.HeaderBytes); - conn.Write(compressed); + conn.Write((ReadOnlySpan)compressed); } else { @@ -370,7 +371,7 @@ public static void WriteJson(Connection conn, byte[] body, bool compress, bool a conn.Write(lenBuf[..lenWritten]); conn.Write("\r\n"u8); conn.Write(DateHelper.HeaderBytes); - conn.Write(body); + conn.Write((ReadOnlySpan)body); } } @@ -382,12 +383,12 @@ public static void WriteBytes(Connection conn, byte[] body, string contentType) conn.Write("HTTP/1.1 200 OK\r\n"u8); conn.Write(ServerHeader); conn.Write("Content-Type: "u8); - conn.Write(Encoding.UTF8.GetBytes(contentType)); + conn.Write((ReadOnlySpan)Encoding.UTF8.GetBytes(contentType)); conn.Write("\r\nContent-Length: "u8); conn.Write(lenBuf[..lenWritten]); conn.Write("\r\n"u8); conn.Write(DateHelper.HeaderBytes); - conn.Write(body); + conn.Write((ReadOnlySpan)body); } public static void Write404(Connection conn) @@ -411,7 +412,7 @@ public static void Write500(Connection conn, string msg) conn.Write(lenBuf[..lenWritten]); conn.Write("\r\n"u8); conn.Write(DateHelper.HeaderBytes); - conn.Write(body); + conn.Write((ReadOnlySpan)body); } } diff --git a/frameworks/zerg/zerg-httparena.csproj b/frameworks/zerg/zerg-httparena.csproj index 9e1b8ec..5d22618 100644 --- a/frameworks/zerg/zerg-httparena.csproj +++ b/frameworks/zerg/zerg-httparena.csproj @@ -10,8 +10,7 @@ true - - + From 1329758361ddc5defa1d13963020c6aa01fc8590 Mon Sep 17 00:00:00 2001 From: BennyFranciscus <268274351+BennyFranciscus@users.noreply.github.com> Date: Mon, 16 Mar 2026 22:13:40 +0000 Subject: [PATCH 07/15] =?UTF-8?q?fix:=20build=20zerg=20from=20source=20?= =?UTF-8?q?=E2=80=94=20NuGet=20'core'=20dependency=20resolves=20to=20wrong?= =?UTF-8?q?=20package?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The zerg 0.5.24 NuGet depends on a package called 'core' v1.0.0, which resolves to 'Core' by Conesoft (a 2013 jQuery wrapper) instead of MDA2AV's core assembly. This means Zerg.Core namespace is never available. Fix: clone zerg source and use ProjectReference instead of PackageReference. Also install clang + zlib1g-dev for NativeAOT compilation. @MDA2AV the root cause is in zerg.nuspec — the 'core' dependency needs a unique package name (e.g. 'zerg.core') or core.dll should be bundled into the main zerg NuGet package. --- frameworks/zerg/Dockerfile | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/frameworks/zerg/Dockerfile b/frameworks/zerg/Dockerfile index 3603440..f6d90a2 100644 --- a/frameworks/zerg/Dockerfile +++ b/frameworks/zerg/Dockerfile @@ -1,6 +1,19 @@ FROM mcr.microsoft.com/dotnet/sdk:10.0-preview AS build + +# Install clang for NativeAOT linking +RUN apt-get update && apt-get install -y --no-install-recommends clang zlib1g-dev && rm -rf /var/lib/apt/lists/* + +WORKDIR /src + +# Clone zerg source (NuGet package has broken 'core' dependency — resolves to wrong package) +RUN git clone --depth 1 https://github.com/MDA2AV/zerg.git /src/zerg-repo + WORKDIR /app COPY . . + +# Replace NuGet PackageReference with ProjectReference to zerg source +RUN sed -i 's|||' zerg-httparena.csproj + RUN dotnet publish -c Release -o out FROM mcr.microsoft.com/dotnet/runtime:10.0-preview From bae3641f3634aa79e55d3444e207d461eaa5fdf8 Mon Sep 17 00:00:00 2001 From: BennyFranciscus <268274351+BennyFranciscus@users.noreply.github.com> Date: Mon, 16 Mar 2026 22:26:32 +0000 Subject: [PATCH 08/15] fix(zerg): use native entrypoint for NativeAOT binary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NativeAOT publishes a self-contained native binary, not a managed DLL. The entrypoint was 'dotnet zerg-httparena.dll' which fails because there's no DLL — only the native executable. Also switched runtime base from dotnet/runtime to dotnet/runtime-deps since NativeAOT binaries don't need the managed runtime, just native deps (libc, etc). --- frameworks/zerg/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frameworks/zerg/Dockerfile b/frameworks/zerg/Dockerfile index f6d90a2..1c9664d 100644 --- a/frameworks/zerg/Dockerfile +++ b/frameworks/zerg/Dockerfile @@ -16,8 +16,8 @@ RUN sed -i 's|| Date: Mon, 16 Mar 2026 22:41:59 +0000 Subject: [PATCH 09/15] =?UTF-8?q?feat(zerg):=20switch=20to=20zerg=200.5.25?= =?UTF-8?q?=20NuGet=20=E2=80=94=20drop=20source=20build=20workaround?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MDA2AV fixed the core dependency collision in 0.5.25 (renamed to zerg.core). Removed git clone + sed ProjectReference workaround from Dockerfile. --- frameworks/zerg/Dockerfile | 8 -------- frameworks/zerg/zerg-httparena.csproj | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/frameworks/zerg/Dockerfile b/frameworks/zerg/Dockerfile index 1c9664d..61cc612 100644 --- a/frameworks/zerg/Dockerfile +++ b/frameworks/zerg/Dockerfile @@ -3,17 +3,9 @@ FROM mcr.microsoft.com/dotnet/sdk:10.0-preview AS build # Install clang for NativeAOT linking RUN apt-get update && apt-get install -y --no-install-recommends clang zlib1g-dev && rm -rf /var/lib/apt/lists/* -WORKDIR /src - -# Clone zerg source (NuGet package has broken 'core' dependency — resolves to wrong package) -RUN git clone --depth 1 https://github.com/MDA2AV/zerg.git /src/zerg-repo - WORKDIR /app COPY . . -# Replace NuGet PackageReference with ProjectReference to zerg source -RUN sed -i 's|||' zerg-httparena.csproj - RUN dotnet publish -c Release -o out FROM mcr.microsoft.com/dotnet/runtime-deps:10.0-preview diff --git a/frameworks/zerg/zerg-httparena.csproj b/frameworks/zerg/zerg-httparena.csproj index 5d22618..e4fa4ae 100644 --- a/frameworks/zerg/zerg-httparena.csproj +++ b/frameworks/zerg/zerg-httparena.csproj @@ -10,7 +10,7 @@ true - + From c4711342a9ed266e1d55fa91dddf9deecba4daa1 Mon Sep 17 00:00:00 2001 From: BennyFranciscus <268274351+BennyFranciscus@users.noreply.github.com> Date: Mon, 16 Mar 2026 23:02:38 +0000 Subject: [PATCH 10/15] fix: use JSON source generators for NativeAOT compatibility .NET 10 NativeAOT disables reflection-based JSON serialization by default. The AppData.Load() and DB query methods were using JsonSerializer.Deserialize() with JsonSerializerOptions instead of the source-generated AppJsonContext. Changes: - Use AppJsonContext.Default.ListDatasetItem for dataset deserialization - Use AppJsonContext.Default.ListString for DB tag parsing - Add List to AppJsonContext serializable types --- frameworks/zerg/Program.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frameworks/zerg/Program.cs b/frameworks/zerg/Program.cs index 7078aef..0bacf0e 100644 --- a/frameworks/zerg/Program.cs +++ b/frameworks/zerg/Program.cs @@ -52,6 +52,7 @@ public class ProcessedItem [JsonSerializable(typeof(JsonResponse))] [JsonSerializable(typeof(DbResponse))] [JsonSerializable(typeof(List))] +[JsonSerializable(typeof(List))] partial class AppJsonContext : JsonSerializerContext { } public class JsonResponse @@ -106,7 +107,7 @@ public static void Load() var path = Environment.GetEnvironmentVariable("DATASET_PATH") ?? "/data/dataset.json"; if (File.Exists(path)) { - Dataset = JsonSerializer.Deserialize>(File.ReadAllText(path), JsonOpts) ?? new(); + Dataset = JsonSerializer.Deserialize(File.ReadAllText(path), AppJsonContext.Default.ListDatasetItem) ?? new(); JsonCache = BuildJsonCache(Dataset); } @@ -114,7 +115,7 @@ public static void Load() var largePath = "/data/dataset-large.json"; if (File.Exists(largePath)) { - var largeItems = JsonSerializer.Deserialize>(File.ReadAllText(largePath), JsonOpts) ?? new(); + var largeItems = JsonSerializer.Deserialize(File.ReadAllText(largePath), AppJsonContext.Default.ListDatasetItem) ?? new(); LargeJsonCache = BuildJsonCache(largeItems); } @@ -531,7 +532,7 @@ static void HandleDb(Connection conn, in HttpRequest req) Price = reader.GetDouble(3), Quantity = reader.GetInt32(4), Active = reader.GetInt32(5) == 1, - Tags = JsonSerializer.Deserialize>(reader.GetString(6)) ?? new(), + Tags = JsonSerializer.Deserialize(reader.GetString(6), AppJsonContext.Default.ListString) ?? new(), Rating = new DbRating { Score = reader.GetDouble(7), Count = reader.GetInt32(8) } }); } From c6eb67738250ab917f03b09f436f1420a38c1740 Mon Sep 17 00:00:00 2001 From: BennyFranciscus <268274351+BennyFranciscus@users.noreply.github.com> Date: Tue, 17 Mar 2026 02:46:11 +0000 Subject: [PATCH 11/15] fix: convert to top-level statements (CS8803 with .NET 10 NativeAOT) .NET 10 preview + NativeAOT source generators emit top-level code that conflicts with an explicit static class Program / Main entry point. Moving the entry point logic to top-level statements (before all type declarations) fixes CS8803. --- frameworks/zerg/Program.cs | 119 ++++++++++++++++++------------------- 1 file changed, 58 insertions(+), 61 deletions(-) diff --git a/frameworks/zerg/Program.cs b/frameworks/zerg/Program.cs index 0bacf0e..5c6c8bb 100644 --- a/frameworks/zerg/Program.cs +++ b/frameworks/zerg/Program.cs @@ -13,6 +13,64 @@ using zerg.Engine.Configs; +// ── Entry point (top-level statements) ── + +AppData.Load(); + +int reactorCount = Environment.ProcessorCount; +if (args.Length > 0 && int.TryParse(args[0], out int rc)) + reactorCount = rc; + +Console.WriteLine($"zerg HttpArena server starting on :8080 with {reactorCount} reactors"); + +var engine = new Engine(new EngineOptions +{ + Ip = "0.0.0.0", + Port = 8080, + Backlog = 65535, + ReactorCount = reactorCount, + AcceptorConfig = new AcceptorConfig( + RingFlags: 0, + SqCpuThread: -1, + SqThreadIdleMs: 100, + RingEntries: 8 * 1024, + BatchSqes: 4096, + CqTimeout: 100_000_000, + IPVersion: IPVersion.IPv6DualStack + ), + ReactorConfigs = Enumerable.Range(0, reactorCount).Select(_ => new ReactorConfig( + RingFlags: (1u << 12) | (1u << 13), // SINGLE_ISSUER | DEFER_TASKRUN + SqCpuThread: -1, + SqThreadIdleMs: 100, + RingEntries: 8 * 1024, + RecvBufferSize: 16 * 1024, + BufferRingEntries: 16 * 1024, + BatchCqes: 4096, + MaxConnectionsPerReactor: 8 * 1024, + CqTimeout: 1_000_000, + ConnectionBufferRingEntries: 32, + IncrementalBufferConsumption: false + )).ToArray() +}); + +engine.Listen(); + +var cts = new CancellationTokenSource(); + +try +{ + while (engine.ServerRunning) + { + var connection = await engine.AcceptAsync(cts.Token); + if (connection is null) continue; + _ = ConnectionHandler.HandleAsync(connection); + } +} +catch (OperationCanceledException) { } + +Console.WriteLine("Server stopped."); + + // ── Data models ── public class DatasetItem @@ -617,65 +675,4 @@ internal static async Task HandleAsync(Connection connection) } } -// ── Entry point ── -static class Program -{ - static async Task Main(string[] args) - { - AppData.Load(); - - int reactorCount = Environment.ProcessorCount; - if (args.Length > 0 && int.TryParse(args[0], out int rc)) - reactorCount = rc; - - Console.WriteLine($"zerg HttpArena server starting on :8080 with {reactorCount} reactors"); - - var engine = new Engine(new EngineOptions - { - Ip = "0.0.0.0", - Port = 8080, - Backlog = 65535, - ReactorCount = reactorCount, - AcceptorConfig = new AcceptorConfig( - RingFlags: 0, - SqCpuThread: -1, - SqThreadIdleMs: 100, - RingEntries: 8 * 1024, - BatchSqes: 4096, - CqTimeout: 100_000_000, - IPVersion: IPVersion.IPv6DualStack - ), - ReactorConfigs = Enumerable.Range(0, reactorCount).Select(_ => new ReactorConfig( - RingFlags: (1u << 12) | (1u << 13), // SINGLE_ISSUER | DEFER_TASKRUN - SqCpuThread: -1, - SqThreadIdleMs: 100, - RingEntries: 8 * 1024, - RecvBufferSize: 16 * 1024, - BufferRingEntries: 16 * 1024, - BatchCqes: 4096, - MaxConnectionsPerReactor: 8 * 1024, - CqTimeout: 1_000_000, - ConnectionBufferRingEntries: 32, - IncrementalBufferConsumption: false - )).ToArray() - }); - - engine.Listen(); - - var cts = new CancellationTokenSource(); - - try - { - while (engine.ServerRunning) - { - var connection = await engine.AcceptAsync(cts.Token); - if (connection is null) continue; - _ = ConnectionHandler.HandleAsync(connection); - } - } - catch (OperationCanceledException) { } - - Console.WriteLine("Server stopped."); - } -} From 3c719023593dc4c87a119ce2686038611a62f350 Mon Sep 17 00:00:00 2001 From: BennyFranciscus <268274351+BennyFranciscus@users.noreply.github.com> Date: Wed, 18 Mar 2026 08:06:34 +0000 Subject: [PATCH 12/15] validate.sh: add --security-opt seccomp=unconfined for io_uring frameworks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docker's default seccomp profile blocks io_uring_setup. benchmark.sh already has this flag — validate.sh needs it too for io_uring-based frameworks (zerg, and any future ones) to start successfully. Reads the 'engine' field from meta.json and adds the flag when it's 'io_uring'. --- scripts/validate.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/validate.sh b/scripts/validate.sh index 074b877..bbd540e 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -63,6 +63,12 @@ if has_test "static-h2" || has_test "static-h3"; then docker_args+=(-v "$DATA_DIR/static:/data/static:ro") fi +# Allow io_uring syscalls for frameworks that need them (blocked by default seccomp) +ENGINE=$(python3 -c "import json; print(json.load(open('$META_FILE')).get('engine',''))" 2>/dev/null || true) +if [ "$ENGINE" = "io_uring" ]; then + docker_args+=(--security-opt seccomp=unconfined) +fi + docker run "${docker_args[@]}" "$IMAGE_NAME" # Wait for server to start From d53514897feee04b57d1c5909c631c77f0648fe5 Mon Sep 17 00:00:00 2001 From: BennyFranciscus <268274351+BennyFranciscus@users.noreply.github.com> Date: Wed, 18 Mar 2026 08:12:26 +0000 Subject: [PATCH 13/15] validate.sh: add --ulimit memlock=-1:-1 for io_uring frameworks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 128 reactors × 16K buffer ring entries = lots of locked memory. Docker's default memlock limit (64KB) causes create_ring to fail with ENOMEM. benchmark.sh already passes both --security-opt seccomp=unconfined and --ulimit memlock=-1:-1 for io_uring frameworks — validate.sh was missing the memlock part. --- scripts/validate.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/validate.sh b/scripts/validate.sh index bbd540e..453a5c6 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -67,6 +67,7 @@ fi ENGINE=$(python3 -c "import json; print(json.load(open('$META_FILE')).get('engine',''))" 2>/dev/null || true) if [ "$ENGINE" = "io_uring" ]; then docker_args+=(--security-opt seccomp=unconfined) + docker_args+=(--ulimit memlock=-1:-1) fi docker run "${docker_args[@]}" "$IMAGE_NAME" From 254f345273a459f8828cce07a5f749e2f1a8b598 Mon Sep 17 00:00:00 2001 From: BennyFranciscus <268274351+BennyFranciscus@users.noreply.github.com> Date: Wed, 18 Mar 2026 09:06:14 +0000 Subject: [PATCH 14/15] validate.sh: cleanup stale container before docker run On self-hosted runners, if a previous CI run is cancelled mid-execution, the EXIT trap may not fire, leaving a stale container. Adding an explicit cleanup before docker run prevents the 'container name already in use' error. --- scripts/validate.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/validate.sh b/scripts/validate.sh index 453a5c6..459e963 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -70,6 +70,9 @@ if [ "$ENGINE" = "io_uring" ]; then docker_args+=(--ulimit memlock=-1:-1) fi +# Remove any stale container from a previous run +cleanup + docker run "${docker_args[@]}" "$IMAGE_NAME" # Wait for server to start From 641060925f77a994ba5451fff00929b5e9312c97 Mon Sep 17 00:00:00 2001 From: BennyFranciscus <268274351+BennyFranciscus@users.noreply.github.com> Date: Wed, 18 Mar 2026 09:47:06 +0000 Subject: [PATCH 15/15] =?UTF-8?q?fix:=20rewrite=20HTTP=20header=20parser?= =?UTF-8?q?=20=E2=80=94=20fix=20Content-Length/Accept-Encoding/chunked=20T?= =?UTF-8?q?E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous header parser had a bug where it iterated over span[..headerEnd] but the loop's IndexOf(CRLF) would miss the last header line before the double-CRLF terminator. If Content-Length was the last header (common with curl), it was never parsed — resulting in: - POST body not being read (returns a+b instead of a+b+body) - Upload returning 0 instead of content length - Accept-Encoding: gzip not detected, causing uncompressed large responses Fixes: 1. Rewrite header parser to correctly handle last header line 2. Use case-insensitive header matching via bitwise OR 0x20 3. Add Transfer-Encoding: chunked support (required by validate.sh) 4. Parse chunked body with proper hex size + trailing CRLF handling --- frameworks/zerg/Program.cs | 176 +++++++++++++++++++++++++++++++------ 1 file changed, 149 insertions(+), 27 deletions(-) diff --git a/frameworks/zerg/Program.cs b/frameworks/zerg/Program.cs index 5c6c8bb..17de295 100644 --- a/frameworks/zerg/Program.cs +++ b/frameworks/zerg/Program.cs @@ -269,14 +269,15 @@ readonly struct HttpRequest public readonly int TotalLength; public readonly int ContentLength; public readonly bool AcceptsGzip; + public readonly bool IsChunked; public HttpRequest(ReadOnlyMemory method, ReadOnlyMemory path, ReadOnlyMemory query, ReadOnlyMemory body, - int totalLength, int contentLength, bool acceptsGzip) + int totalLength, int contentLength, bool acceptsGzip, bool isChunked = false) { Method = method; Path = path; Query = query; Body = body; TotalLength = totalLength; ContentLength = contentLength; - AcceptsGzip = acceptsGzip; + AcceptsGzip = acceptsGzip; IsChunked = isChunked; } } @@ -320,45 +321,166 @@ static class HttpParser path = uri; } - // Parse Content-Length if present + // Parse headers — include the full header block up to the double CRLF int contentLength = 0; bool acceptsGzip = false; - var headers = span[..headerEnd]; - int pos = 0; - while (pos < headers.Length) + bool isChunked = false; + + // Skip the request line first + int reqLineEnd = span.IndexOf("\r\n"u8); + if (reqLineEnd < 0) return null; + + // Parse each header line between request line and header end + int pos = reqLineEnd + 2; + while (pos < headerEnd) { - int lineEnd = headers[pos..].IndexOf("\r\n"u8); - if (lineEnd < 0) break; - var line = headers[pos..(pos + lineEnd)]; - pos += lineEnd + 2; - - if (line.Length > 16 && - (line[0] == (byte)'C' || line[0] == (byte)'c') && - (line[8] == (byte)'L' || line[8] == (byte)'l')) + int lineEnd = span[pos..headerEnd].IndexOf("\r\n"u8); + ReadOnlySpan line; + if (lineEnd < 0) + { + // Last header line before \r\n\r\n + line = span[pos..headerEnd]; + pos = headerEnd; + } + else + { + line = span[pos..(pos + lineEnd)]; + pos += lineEnd + 2; + } + + if (line.Length < 2) continue; + + byte first = (byte)(line[0] | 0x20); // lowercase + + if (first == (byte)'c') { - // Content-Length: + // Content-Length or Content-Type int colon = line.IndexOf((byte)':'); if (colon >= 0) { - var val = line[(colon + 1)..]; - // Trim leading space - while (val.Length > 0 && val[0] == (byte)' ') val = val[1..]; - if (Utf8Parser.TryParse(val, out int cl, out _)) - contentLength = cl; + var headerName = line[..colon]; + if (headerName.Length >= 14) + { + // Check for Content-Length (case insensitive) + bool isContentLength = true; + ReadOnlySpan cl = "content-length"u8; + if (headerName.Length >= cl.Length) + { + for (int i = 0; i < cl.Length; i++) + { + if ((headerName[i] | 0x20) != cl[i]) { isContentLength = false; break; } + } + } + else isContentLength = false; + + if (isContentLength) + { + var val = line[(colon + 1)..]; + while (val.Length > 0 && val[0] == (byte)' ') val = val[1..]; + if (Utf8Parser.TryParse(val, out int clv, out _)) + contentLength = clv; + } + } } } - else if (line.Length > 15 && - (line[0] == (byte)'A' || line[0] == (byte)'a') && - (line[7] == (byte)'E' || line[7] == (byte)'e')) + else if (first == (byte)'a') { - // Accept-Encoding: + // Accept-Encoding if (line.IndexOf("gzip"u8) >= 0) acceptsGzip = true; } + else if (first == (byte)'t') + { + // Transfer-Encoding: chunked + int colon = line.IndexOf((byte)':'); + if (colon >= 0) + { + var headerName = line[..colon]; + ReadOnlySpan te = "transfer-encoding"u8; + bool isTE = headerName.Length >= te.Length; + if (isTE) + { + for (int i = 0; i < te.Length; i++) + { + if ((headerName[i] | 0x20) != te[i]) { isTE = false; break; } + } + } + if (isTE) + { + var val = line[(colon + 1)..]; + if (val.IndexOf("chunked"u8) >= 0) + isChunked = true; + } + } + } + } + + // Handle chunked transfer encoding + if (isChunked) + { + // Parse chunked body: read chunks until 0\r\n + var remaining = span[headersLen..]; + int bodyStart = 0; + using var bodyStream = new MemoryStream(); + + while (bodyStart < remaining.Length) + { + // Find chunk size line + int chunkLineEnd = remaining[bodyStart..].IndexOf("\r\n"u8); + if (chunkLineEnd < 0) return null; // incomplete + + var chunkSizeLine = remaining[bodyStart..(bodyStart + chunkLineEnd)]; + // Parse hex chunk size + int chunkSize = 0; + for (int i = 0; i < chunkSizeLine.Length; i++) + { + byte b = chunkSizeLine[i]; + if (b >= (byte)'0' && b <= (byte)'9') + chunkSize = chunkSize * 16 + (b - '0'); + else if (b >= (byte)'a' && b <= (byte)'f') + chunkSize = chunkSize * 16 + (b - 'a' + 10); + else if (b >= (byte)'A' && b <= (byte)'F') + chunkSize = chunkSize * 16 + (b - 'A' + 10); + else if (b == (byte)';') + break; // chunk extension + else + break; + } + + bodyStart += chunkLineEnd + 2; // skip size line + CRLF + + if (chunkSize == 0) + { + // Terminal chunk — skip trailing CRLF + if (bodyStart + 2 <= remaining.Length) + bodyStart += 2; + break; + } + + // Ensure we have enough data for the chunk + trailing CRLF + if (bodyStart + chunkSize + 2 > remaining.Length) + return null; // incomplete + + bodyStream.Write(remaining.Slice(bodyStart, chunkSize)); + bodyStart += chunkSize + 2; // skip chunk data + CRLF + } + + int totalLen = headersLen + bodyStart; + var bodyBytes = bodyStream.ToArray(); + + return new HttpRequest( + method.ToArray(), + path.ToArray(), + query.Length > 0 ? query.ToArray() : ReadOnlyMemory.Empty, + bodyBytes, + totalLen, + bodyBytes.Length, + acceptsGzip, + isChunked: true); } - int totalLen = headersLen + contentLength; - if (span.Length < totalLen) return null; // body not yet complete + int totalLength = headersLen + contentLength; + if (span.Length < totalLength) return null; // body not yet complete ReadOnlyMemory body = default; if (contentLength > 0) @@ -371,7 +493,7 @@ static class HttpParser path.ToArray(), query.Length > 0 ? query.ToArray() : ReadOnlyMemory.Empty, body, - totalLen, + totalLength, contentLength, acceptsGzip); }