diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..de06626 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,72 @@ +# Builds, tests and packs on every push and pull request. A tag named +# v (e.g. v2.0.1) also publishes that version to nuget.org through +# Trusted Publishing: no API key is stored, the publish job trades GitHub's +# OIDC token for a short-lived nuget.org key. +# +# The nuget.org Trusted Publishing policy matches on repository owner, +# repository name, this file's name (build.yml) and, if it names one, the +# `production` environment: renaming this file breaks publishing. + +name: Build + +on: + push: + branches: [main, gen3] + tags: ['v*'] + pull_request: + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-dotnet@v6 + with: + dotnet-version: 10.0.x + + - name: Test + run: dotnet test Cosmos.Network.Http.slnx -c Release + + # On a release tag the tag names the version; otherwise the .csproj does. + - name: Pack + run: | + args=() + if [[ "$GITHUB_REF" == refs/tags/v* ]]; then + args+=("-p:Version=${GITHUB_REF_NAME#v}") + fi + dotnet pack src/Cosmos.Network.Http/Cosmos.Network.Http.csproj -c Release -o artifacts "${args[@]}" + + - uses: actions/upload-artifact@v7 + with: + name: nupkg + path: artifacts/*.nupkg + + publish: + needs: build + if: startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-latest + environment: production + permissions: + id-token: write # lets NuGet/login request GitHub's OIDC token + steps: + - uses: actions/download-artifact@v8 + with: + name: nupkg + path: artifacts + + - name: NuGet login + id: login + uses: NuGet/login@v1 + with: + # The user who CREATED the policy, not the CosmosOS profile that owns + # it. nuget.org looks the policy up by creator and answers a wrong + # guess with "No matching trust policy owned by user 'CosmosOS' was + # found" (HTTP 401). Public, so not a secret. + user: valentinbreiz + + - name: Push to nuget.org + run: dotnet nuget push "artifacts/*.nupkg" --api-key "${{ steps.login.outputs.NUGET_API_KEY }}" --source https://api.nuget.org/v3/index.json diff --git a/.gitignore b/.gitignore index f677870..4c9a8fe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ bin obj -.vs \ No newline at end of file +.vs +artifacts diff --git a/Cosmos.Network.Http.slnx b/Cosmos.Network.Http.slnx new file mode 100644 index 0000000..6005ea2 --- /dev/null +++ b/Cosmos.Network.Http.slnx @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/CosmosHttp.csproj b/CosmosHttp.csproj deleted file mode 100644 index 11360ea..0000000 --- a/CosmosHttp.csproj +++ /dev/null @@ -1,42 +0,0 @@ - - - - net6.0 - True - Cosmos - Cosmos - HTTP Client for CosmosOS. - 1.0.4 - - - - CosmosHttpClient - icon.png - LICENSE.txt - https://github.com/CosmosOS/CosmosHttp - https://github.com/CosmosOS/CosmosHttp - README.md - - - - - - - - - - - - - - - True - \ - - - True - \ - - - - \ No newline at end of file diff --git a/README.md b/README.md index 0da87f0..690526e 100644 --- a/README.md +++ b/README.md @@ -1,70 +1,90 @@ -

CosmosHTTP Client [WIP]

+

CosmosHTTP Client 🚀

- - Version + + Version - + License: BSD Clause 3 License

-> CosmosHTTP is a HTTP client made in C# for the Cosmos operating system construction kit. GET and PUT are currently supported. - -### Todo -See [this issue](https://github.com/CosmosOS/CosmosHttp/issues/1) for todo list. +> CosmosHTTP is an HTTP/1.1 client made in C# for the Cosmos operating system construction kit. ## Usage -### Installation - -Install the Nuget Package from [Nuget](https://www.nuget.org/packages/CosmosHttp/): +Add the package to your kernel .csproj: -```PM -Install-Package CosmosHttp -Version 1.0.4 +```xml + + + ``` -```PM -dotnet add PROJECT package CosmosHttp --version 1.0.4 -``` +The kernel needs networking (`CosmosEnableNetwork`, on by default), an IP configuration (DHCP or static) and, for host names, a DNS server. `Send()` runs the request on the calling thread and returns the response once it has arrived whole: + +```csharp +using System; +using System.IO; +using Cosmos.Network.Http; -Or add these lines to your Cosmos kernel .csproj: +HttpResponse response = new HttpRequest("http://httpforever.com/").Send(); +Console.WriteLine($"{response.StatusCode} {response.ReasonPhrase}, {response.Content.Length} bytes"); +File.WriteAllBytes("/0/index.html", response.Content); ``` - - - + +`Send()` returns error statuses too; `EnsureSuccessStatusCode()` turns them into an `HttpException`, and `GetString()` decodes the body with the charset of its Content-Type: + +```csharp +string json = new HttpRequest("http://example.com/data.json").Send().EnsureSuccessStatusCode().GetString(); ``` -### Examples +A request can set its method, body, headers, timeout and how many redirects it follows: + +```csharp +HttpResponse response = new HttpRequest("http://example.com/api") +{ + Method = "POST", + Body = Encoding.UTF8.GetBytes("{\"name\":\"cosmos\"}"), + Headers = { ["Content-Type"] = "application/json" }, + Timeout = 10_000, // how long the server may stay silent, in milliseconds (15 s by default) + MaxRedirects = 0, // return redirects instead of following them (5 by default) + // Optional: one line per response and redirect. + Log = message => Cosmos.Kernel.System.Diagnostics.Log.WriteString(message + "\n"), +}.Send(); +``` -```CS -using CosmosHttp.Client; +The Host header comes from the URL. Setting it in `Headers` reaches a virtual host by IP address: -HttpRequest request = new(); -request.IP = "34.223.124.45"; -request.Domain = "neverssl.com"; //very useful for subdomains on same IP -request.Path = "/"; -request.Method = "GET"; -request.Send(); -Console.WriteLine(request.Response.Content); // or to get bytes Encoding.ASCII.getString(request.Response.GetStream()) +```csharp +new HttpRequest("http://34.223.124.45/") { Headers = { ["Host"] = "neverssl.com" } }.Send(); ``` -Here is a basic wget command implementation using CosmosHttp: [github.com/aura-systems/Aura-Operating-System](https://github.com/aura-systems/Aura-Operating-System/blob/master/SRC/Aura_OS/System/Interpreter/Commands/Network/Wget.cs#L63). +### Limits + +- `http://` only: there is no TLS, so `https://` URLs, and redirects to them, throw. +- Each request opens a connection of its own, and the server closes it once it has answered. +- Responses come without content coding (`Accept-Encoding: identity`): a Cosmos kernel has no gzip to undo. +- The whole body is held in memory. + +### Threads + +`Send()` never waits in `Thread.Sleep`: it waits in `Socket.Poll`, which returns at once on a Cosmos kernel. So it runs on the kernel's main loop, which must never block, as well as on a thread of its own. ## Authors 👤 **[@valentinbreiz](https://github.com/valentinbreiz)** -👤 **[@2881099](https://github.com/2881099)** +👤 **[@2881099](https://github.com/2881099)** (the first version was inspired by [TcpClientHttpRequest](https://github.com/2881099/TcpClientHttpRequest)) ## 🤝 Contributing -Contributions, issues and feature requests are welcome! Feel free to check [issues page](https://github.com/CosmosOS/CosmosHttp/issues). +Contributions, issues and feature requests are welcome! -## Show your support - -Give a ⭐️ if this project helped you! +Feel free to check [issues page](https://github.com/CosmosOS/Cosmos.Network.Http/issues). ## 📝 License -Copyright © 2023 [CosmosOS](https://github.com/CosmosOS). This project is [BSD Clause 3](https://github.com/CosmosOS/CosmosHttp/blob/main/LICENSE.txt) licensed. +Copyright © 2023-2026 [CosmosOS](https://github.com/CosmosOS). + +This project is [BSD Clause 3](https://github.com/CosmosOS/Cosmos.Network.Http/blob/main/LICENSE.txt) licensed. diff --git a/resources/icon.png b/resources/icon.png index 391927b..def4113 100644 Binary files a/resources/icon.png and b/resources/icon.png differ diff --git a/source/Client/HttpRequest.cs b/source/Client/HttpRequest.cs deleted file mode 100644 index 56f57f3..0000000 --- a/source/Client/HttpRequest.cs +++ /dev/null @@ -1,287 +0,0 @@ -/* -* PROJECT: CosmosHttp Development -* CONTENT: Http Request class (Heavily inspered by https://github.com/2881099/TcpClientHttpRequest) -* PROGRAMMERS: Valentin Charbonnier -*/ - -using System; -using System.Collections.Generic; -using System.IO; -using System.Net; -using System.Net.Sockets; -using System.Text; - -namespace CosmosHttp.Client -{ - public class HttpRequest : HttpPacket - { - private TcpClient _client; - private string _remote; - private string _path; - private int _timeout = 20000; - private NetworkStream _stream; - private HttpResponse _response; - - public string Path - { - get - { - return _path; - } - set - { - _path = value; - } - } - - public HttpResponse Response - { - get { return _response; } - } - - public HttpRequest() - { - _headers.Add("Connection", "Keep-Alive"); - _headers.Add("Accept", "*/*"); - _headers.Add("User-Agent", "CosmosHttp Client (CosmosOS)"); - _headers.Add("Accept-Language", "en-us"); - _headers.Add("Accept-Encoding", "gzip, deflate"); - } - - public void Close() - { - if (_client != null) - { - if (_stream != null) - { - _stream.Close(); - } - _client.Close(); - _client = null; - } - } - - public void Send() - { - Send(string.Empty); - } - - public virtual void Send(string data) - { - Send(data, 0); - } - - private void Send(string data, int redirections) - { - _data = data; - - _headers.Remove("Content-Length"); - if (!string.IsNullOrEmpty(data) && string.Compare(_method, "post", true) == 0) - { - _headers["Content-Length"] = string.Concat(Encoding.ASCII.GetBytes(data).Length); - if (string.IsNullOrEmpty(_headers["Content-Type"])) - { - _headers["Content-Type"] = "application/x-www-form-urlencoded; charset=" + _charset; - } - else if (_headers["Content-Type"].IndexOf("multipart/form-data") == -1) - { - if (_headers["Content-Type"].IndexOf("application/x-www-form-urlencoded") == -1) - { - _headers["Content-Type"] += "; application/x-www-form-urlencoded"; - } - if (_headers["Content-Type"].IndexOf("charset=") == -1) - { - _headers["Content-Type"] += "; charset=" + _charset; - } - } - data += "\r\n\r\n"; - } - _headers["Host"] = _domain; - - string http = _method + " " + _path + " HTTP/1.1\r\n"; - foreach (string head in _headers.Keys) - { - http += head + ": " + _headers[head] + "\r\n"; - } - - http += "\r\n" + data; - _head = http; - byte[] request = Encoding.ASCII.GetBytes(http); - if (_client == null || _remote == null) - { - _remote = _ip; - this.Close(); - _client = new TcpClient(_ip, 80); - } - try - { - _stream = getStream(); - _stream.Write(request, 0, request.Length); - } - catch - { - this.Close(); - _client = new TcpClient(_ip, 80); - _stream = getStream(); - _stream.Write(request, 0, request.Length); - } - receive(_stream, redirections, _ip); - } - - protected void receive(Stream stream, int redirections, string action) - { - // stream.ReadTimeout = _timeout; TO PLUG - _response = null; - byte[] bytes = new byte[1024]; - int bytesRead = 0; - byte[] headBuffer = null; - byte[] bodyBuffer = null; - Exception exception = null; - - while (true) - { - int idx = -1; - try - { - bytesRead = stream.Read(bytes, 0, bytes.Length); - if (bytesRead == 0) - { - if (headBuffer == null || headBuffer.Length == 0) - { - throw new Exception("headBuffer is empty and no more data to read"); - } - break; - } - } - catch (Exception e) - { - exception = e; - break; - } - - if (_response == null) - { - // Add the newly read bytes to the head buffer - int oldLength = headBuffer != null ? headBuffer.Length : 0; - byte[] newHeadBuffer = new byte[oldLength + bytesRead]; - if (headBuffer != null) - { - Array.Copy(headBuffer, 0, newHeadBuffer, 0, oldLength); - } - Array.Copy(bytes, 0, newHeadBuffer, oldLength, bytesRead); - headBuffer = newHeadBuffer; - - // Check for the header delimiter - idx = Utils.findBytes(headBuffer, new byte[] { 13, 10, 13, 10 }, 0); - if (idx != -1) - { - // Create the response with the header - byte[] header = new byte[idx]; - Array.Copy(headBuffer, 0, header, 0, idx); - _response = new HttpResponse(this, header); - _response.Received += headBuffer.Length - idx - 4; - - // Transfer remaining bytes to the body buffer - int bodyLength = headBuffer.Length - idx - 4; - bodyBuffer = new byte[bodyLength]; - Array.Copy(headBuffer, idx + 4, bodyBuffer, 0, bodyLength); - } - } - else - { - _response.Received += bytesRead; - // Add the newly read bytes to the body buffer - int oldLength = bodyBuffer != null ? bodyBuffer.Length : 0; - byte[] newBodyBuffer = new byte[oldLength + bytesRead]; - if (bodyBuffer != null) - { - Array.Copy(bodyBuffer, 0, newBodyBuffer, 0, oldLength); - } - Array.Copy(bytes, 0, newBodyBuffer, oldLength, bytesRead); - bodyBuffer = newBodyBuffer; - } - - if (_response != null) - { - if (_response.ContentLength >= 0) - { - if (_response.ContentLength <= bodyBuffer.Length) - { - break; - } - } - } - } - - if (_response == null) - { - this.closeTcp(); - - // Construct the request headers string - List sb = new List(); - sb.Add(_method.ToUpper() + " " + _ip + " HTTP/1.1"); - foreach (string header in _headers.Keys) - { - sb.Add(header + ": " + _headers[header]); - } - - // Throw a WebException with the appropriate message - if (exception == null) - { - throw new WebException("WebException " + string.Join("\r\n", sb.ToArray())); - } - else - { - throw new WebException(exception.Message + "\r\n" + string.Join("\r\n", sb.ToArray()), exception); - } - } - - _response.SetStream(bodyBuffer); - - this.closeTcp(); - } - - protected bool closeTcp() - { - this.Close(); - return false; - } - - protected NetworkStream getStream() - { - return _client.GetStream(); - } - } - - public class Utils - { - public static int findBytes(byte[] source, byte[] find, int startIndex) - { - if (find == null) return -1; - if (find.Length == 0) return -1; - if (source == null) return -1; - if (source.Length == 0) return -1; - if (startIndex < 0) startIndex = 0; - int idx = -1, idx2 = startIndex - 1; - do - { - idx2 = idx = Array.FindIndex(source, Math.Min(idx2 + 1, source.Length), delegate (byte b) { - return b == find[0]; - }); - if (idx2 != -1) - { - for (int a = 1; a < find.Length; a++) - { - if (++idx2 >= source.Length || source[idx2] != find[a]) - { - idx = -1; - break; - } - } - if (idx != -1) break; - } - } while (idx2 != -1); - return idx; - } - } -} \ No newline at end of file diff --git a/source/Client/HttpResponse.cs b/source/Client/HttpResponse.cs deleted file mode 100644 index ab4b99e..0000000 --- a/source/Client/HttpResponse.cs +++ /dev/null @@ -1,154 +0,0 @@ -/* -* PROJECT: CosmosHttp Development -* CONTENT: Http Response class (Heavily inspered by https://github.com/2881099/TcpClientHttpRequest) -* PROGRAMMERS: Valentin Charbonnier -*/ - -using System; -using System.Net; -using System.Text; - -namespace CosmosHttp.Client -{ - public class HttpResponse : HttpPacket - { - private int _received = 0; - private HttpStatusCode _statusCode; - private int _contentLength = -1; - private string _contentType; - private string _server; - private string _content; - private string _contentEncoding = string.Empty; - private byte[] _stream = new byte[] { }; - - public int Received - { - get { return _received; } - internal set { _received = value; } - } - - public string TransferEncoding - { - get { return _headers["Transfer-Encoding"]; } - set { _headers["Transfer-Encoding"] = value; } - } - - public int ContentLength - { - get { return _contentLength; } - } - - public string Content - { - get - { - if (_content == null) - { - _content = Encoding.ASCII.GetString(_stream); - } - return _content; - } - } - - public HttpResponse(HttpRequest ie, byte[] headBytes) - { - _ip = ie.IP; - _method = ie.Method; - _charset = ie.Charset; - string head = Encoding.ASCII.GetString(headBytes); - _head = head = head.Trim(); - int idx = head.IndexOf(' '); - if (idx != -1) - { - head = head.Substring(idx + 1); - } - idx = head.IndexOf(' '); - if (idx != -1) - { - _statusCode = (HttpStatusCode)int.Parse(head.Remove(idx)); - head = head.Substring(idx + 1); - } - idx = head.IndexOf("\r\n"); - if (idx != -1) - { - head = head.Substring(idx + 2); - } - - string[] heads = head.Split(new string[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries); - foreach (string h in heads) - { - string[] nv = h.Split(new char[] { ':' }, 2); - if (nv.Length == 2) - { - string n = nv[0].Trim(); - string v = nv[1].Trim(); - - // Handle specific headers and their unique cases - switch (n.ToLower()) - { - case "content-length": - if (!int.TryParse(v, out _contentLength)) _contentLength = -1; - break; - case "content-type": - _contentType = v; - idx = v.IndexOf("charset=", StringComparison.OrdinalIgnoreCase); - if (idx != -1) - { - string charset = v.Substring(idx + 8).Split(';')[0].Trim(); - if (string.Compare(_charset, charset, StringComparison.OrdinalIgnoreCase) != 0) - { - try - { - Encoding testEncode = Encoding.GetEncoding(charset); - _charset = charset; - } - catch (Exception ex) - { - Cosmos.HAL.Global.debugger.Send("Ex: " + ex.ToString()); - } - } - } - break; - case "server": - _server = v; - break; - case "content-encoding": - _contentEncoding = v; - break; - // Add more specific headers as needed - default: - if (_headers.ContainsKey(n)) - { - // Append or replace based on your requirement - _headers[n] = v; - } - else - { - _headers.Add(n, v); - } - break; - } - } - } - } - - public void SetStream(byte[] bodyBytes) - { - _stream = bodyBytes; - _contentLength = bodyBytes.Length; - } - - public byte[] GetStream() - { - switch (_contentEncoding.ToLower()) - { - case "gzip": - return GZip.Decompress(_stream); - case "deflate": - return Deflate.Decompress(_stream); - default: - return _stream; - } - } - } -} \ No newline at end of file diff --git a/source/Compress.cs b/source/Compress.cs deleted file mode 100644 index 6b6158b..0000000 --- a/source/Compress.cs +++ /dev/null @@ -1,105 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace CosmosHttp -{ - public static class GZip - { - public static byte[] Decompress(Stream stream) - { - try - { - stream.Position = 0; - using (MemoryStream ms = new MemoryStream()) - { - using (Ionic.Zlib.GZipStream gzip = new Ionic.Zlib.GZipStream(stream, Ionic.Zlib.CompressionMode.Decompress)) - { - byte[] data = new byte[1024]; - int size = 0; - while ((size = gzip.Read(data, 0, data.Length)) > 0) - { - ms.Write(data, 0, size); - } - } - return ms.ToArray(); - } - } - catch { return (stream as MemoryStream).ToArray(); }; - } - public static byte[] Decompress(byte[] bt) - { - return Decompress(new MemoryStream(bt)); - } - public static byte[] Compress(string text) - { - return Compress(Encoding.UTF8.GetBytes(text)); - } - public static byte[] Compress(byte[] bt) - { - return Compress(bt, 0, bt.Length); - } - public static byte[] Compress(byte[] bt, int startIndex, int length) - { - using (MemoryStream ms = new MemoryStream()) - { - using (Ionic.Zlib.GZipStream gzip = new Ionic.Zlib.GZipStream(ms, Ionic.Zlib.CompressionMode.Compress)) - { - gzip.Write(bt, startIndex, length); - } - return ms.ToArray(); - } - } - } - - public static class Deflate - { - public static byte[] Decompress(Stream stream) - { - try - { - stream.Position = 0; - using (MemoryStream ms = new MemoryStream()) - { - using (Ionic.Zlib.DeflateStream def = new Ionic.Zlib.DeflateStream(stream, Ionic.Zlib.CompressionMode.Decompress)) - { - byte[] data = new byte[1024]; - int size = 0; - while ((size = def.Read(data, 0, data.Length)) > 0) - { - ms.Write(data, 0, size); - } - } - return ms.ToArray(); - } - } - catch { return (stream as MemoryStream).ToArray(); }; - } - public static byte[] Decompress(byte[] bt) - { - return Decompress(new MemoryStream(bt)); - } - public static byte[] Compress(string text) - { - return Compress(Encoding.UTF8.GetBytes(text)); - } - public static byte[] Compress(byte[] bt) - { - return Compress(bt, 0, bt.Length); - } - public static byte[] Compress(byte[] bt, int startIndex, int length) - { - using (MemoryStream ms = new MemoryStream()) - { - using (Ionic.Zlib.DeflateStream def = new Ionic.Zlib.DeflateStream(ms, Ionic.Zlib.CompressionMode.Compress)) - { - def.Write(bt, startIndex, length); - } - return ms.ToArray(); - } - } - } -} diff --git a/source/HttpPacket.cs b/source/HttpPacket.cs deleted file mode 100644 index f0b958f..0000000 --- a/source/HttpPacket.cs +++ /dev/null @@ -1,84 +0,0 @@ -/* -* PROJECT: CosmosHttp Development -* CONTENT: Base Http packet class (Heavily inspered by https://github.com/2881099/TcpClientHttpRequest) -* PROGRAMMERS: Valentin Charbonnier -*/ - -using System; -using System.Collections.Generic; - -namespace CosmosHttp -{ - public class HttpPacket : IDisposable - { - internal string _domain; - internal string _ip; - internal string _method = "GET"; - internal string _charset = "us-ascii"; - internal string _data; - internal string _head; - internal Dictionary _headers; - - public HttpPacket() - { - _headers = new Dictionary(); - } - - public string Method - { - get - { - return _method; - } - set - { - _method = value.ToUpper(); - } - } - - public string Domain - { - get - { - return _domain; - } - set - { - _domain = value; - } - } - - public string IP - { - get - { - return _ip; - } - set - { - _ip = value; - } - } - - public string Charset - { - get - { - return _charset; - } - set - { - _charset = value; - } - } - - public Dictionary Headers - { - get { return _headers; } - } - - public void Dispose() - { - } - } -} diff --git a/src/Cosmos.Network.Http/ChunkedBody.cs b/src/Cosmos.Network.Http/ChunkedBody.cs new file mode 100644 index 0000000..d5fb09b --- /dev/null +++ b/src/Cosmos.Network.Http/ChunkedBody.cs @@ -0,0 +1,150 @@ +// This code is licensed under the BSD 3-Clause license (see LICENSE.txt for details) + +using System.IO; + +namespace Cosmos.Network.Http; + +/// +/// The chunked transfer coding (RFC 9112, section 7.1): chunks, each after +/// its size in hexadecimal, up to a chunk of size 0 and a trailer section. +/// Chunk extensions and trailers carry nothing a client needs and are +/// skipped. +/// +internal static class ChunkedBody +{ + /// + /// Whether the chunked body that starts at + /// has arrived whole: its last chunk and the trailer section that ends it. + /// moves past each whole chunk, so the next + /// call resumes there rather than walking the body again. + /// + /// A chunk size is malformed. + public static bool FindEnd(byte[] data, int length, ref int position) + { + while (true) + { + int lineEnd = IndexOfLineEnd(data, position, length); + if (lineEnd < 0) + { + return false; + } + + int size = ParseSize(data, position, lineEnd); + if (size == 0) + { + // The trailer section: field lines up to an empty one. + int line = lineEnd + 2; + while (true) + { + int end = IndexOfLineEnd(data, line, length); + if (end < 0) + { + return false; + } + + if (end == line) + { + return true; + } + + line = end + 2; + } + } + + long next = (long)lineEnd + 2 + size + 2; + if (next > length) + { + return false; + } + + position = (int)next; + } + } + + /// The data of the chunked body that starts at , chunk sizes, extensions and trailers left out. + /// The body ends before its last chunk, or a chunk size is malformed. + public static byte[] Decode(byte[] data, int start, int length) + { + MemoryStream body = new(); + int position = start; + while (true) + { + int lineEnd = IndexOfLineEnd(data, position, length); + if (lineEnd < 0) + { + throw Truncated(); + } + + int size = ParseSize(data, position, lineEnd); + if (size == 0) + { + return body.ToArray(); + } + + int chunkStart = lineEnd + 2; + if ((long)chunkStart + size > length) + { + throw Truncated(); + } + + body.Write(data, chunkStart, size); + position = chunkStart + size + 2; + } + } + + /// The size on a chunk line, ignoring the extensions after it. + private static int ParseSize(byte[] data, int start, int end) + { + int size = 0; + int digits = 0; + int i = start; + for (; i < end; i++) + { + int digit = HexValue(data[i]); + if (digit < 0) + { + break; + } + + if (size > (int.MaxValue >> 4)) + { + throw new HttpException("The response has a chunk larger than 2 GB."); + } + + size = (size << 4) | digit; + digits++; + } + + // What follows the digits, if anything, is whitespace or a ";name=value" extension. + if (digits == 0 || (i < end && data[i] is not ((byte)';' or (byte)' ' or (byte)'\t'))) + { + throw new HttpException("The response has a malformed chunk size."); + } + + return size; + } + + private static int HexValue(byte c) => c switch + { + >= (byte)'0' and <= (byte)'9' => c - '0', + >= (byte)'a' and <= (byte)'f' => c - 'a' + 10, + >= (byte)'A' and <= (byte)'F' => c - 'A' + 10, + _ => -1, + }; + + /// The index of the next CRLF from , or -1. + private static int IndexOfLineEnd(byte[] data, int start, int length) + { + for (int i = start; i + 1 < length; i++) + { + if (data[i] == '\r' && data[i + 1] == '\n') + { + return i; + } + } + + return -1; + } + + private static HttpException Truncated() => new("The server closed the connection in the middle of a chunked body."); +} diff --git a/src/Cosmos.Network.Http/Cosmos.Network.Http.csproj b/src/Cosmos.Network.Http/Cosmos.Network.Http.csproj new file mode 100644 index 0000000..c766746 --- /dev/null +++ b/src/Cosmos.Network.Http/Cosmos.Network.Http.csproj @@ -0,0 +1,44 @@ + + + + + net10.0 + latest + enable + disable + true + true + Cosmos.Network.Http + + + + Cosmos.Network.Http + 2.0.0 + Cosmos.Network.Http + Cosmos + Cosmos + HTTP client for Cosmos Gen3 kernels. + cosmos;http;client;kernel + icon.png + LICENSE.txt + README.md + https://github.com/CosmosOS/Cosmos.Network.Http + https://github.com/CosmosOS/Cosmos.Network.Http + + + + + + + + + + + + + diff --git a/src/Cosmos.Network.Http/HttpConnection.cs b/src/Cosmos.Network.Http/HttpConnection.cs new file mode 100644 index 0000000..4333766 --- /dev/null +++ b/src/Cosmos.Network.Http/HttpConnection.cs @@ -0,0 +1,177 @@ +// This code is licensed under the BSD 3-Clause license (see LICENSE.txt for details) + +using System; +using System.Diagnostics; +using System.Net; +using System.Net.Sockets; + +namespace Cosmos.Network.Http; + +/// +/// One request and its response, over a connection of their own: requests +/// ask the server to close it once it has answered. +/// +/// +/// Written for the Cosmos socket plugs, in a way a desktop runs as well: +/// +/// Host names are resolved with before connecting: +/// the plugged host name overloads of Connect do not resolve. +/// Bytes come in through the byte[] Receive overload, for what +/// reports: the Span overloads lose what they +/// receive, and a Receive with nothing waiting returns 0 on an open +/// connection. +/// The waiting is done by , +/// never by Thread.Sleep. Poll blocks for its timeout where sockets +/// block, and returns at once on a Cosmos kernel: a request may run on any +/// thread there, the kernel's main loop included, which must never +/// block. +/// The socket is closed after the try, not in a finally: +/// a Cosmos kernel skips finally blocks while an exception +/// unwinds. +/// +/// +internal static class HttpConnection +{ + /// The most taken from the socket at a time, in bytes. + private const int ReceiveChunkSize = 16 * 1024; + + /// How long a Poll waits for data where it can wait, in microseconds. + private const int PollIntervalUs = 10_000; + + /// Sends to and reads the response. + /// Where to connect. + /// The request, head and body. + /// Whether the request is a HEAD one, whose response has no body. + /// How long the server may stay silent, in milliseconds. + /// The host could not be resolved or reached, or the response did not come whole. + public static HttpResponse Exchange(HttpUrl url, byte[] request, bool isHead, int timeout) + { + IPAddress address = Resolve(url.Host); + + Socket socket = new(address.AddressFamily, SocketType.Stream, ProtocolType.Tcp); + HttpResponse? response = null; + Exception? failure = null; + bool connected = false; + try + { + socket.Connect(address, url.Port); + connected = true; + Send(socket, request); + response = Receive(socket, url, isHead, timeout); + } + catch (Exception exception) + { + failure = exception; + } + + try + { + socket.Close(); + } + catch (Exception) + { + // Nothing left to do with a connection that would not close. + } + + if (failure is HttpException) + { + throw failure; + } + + if (failure is not null) + { + // Not narrowed to SocketException: the Cosmos socket plugs report + // a refused or dropped connection as a bare Exception. + string message = connected + ? $"The connection to {url.Authority} failed: {failure.Message}" + : $"Could not connect to {url.Authority}: {failure.Message}"; + throw new HttpException(message, failure); + } + + return response!; + } + + private static IPAddress Resolve(string host) + { + if (IPAddress.TryParse(host, out IPAddress? literal)) + { + return literal; + } + + IPAddress[] addresses; + try + { + addresses = Dns.GetHostAddresses(host); + } + catch (Exception exception) + { + throw new HttpException($"Could not resolve {host}: {exception.Message}", exception); + } + + // An IPv4 address first: a Cosmos kernel has nothing else to connect to. + foreach (IPAddress address in addresses) + { + if (address.AddressFamily == AddressFamily.InterNetwork) + { + return address; + } + } + + if (addresses.Length == 0) + { + throw new HttpException($"Could not resolve {host}: it has no address."); + } + + return addresses[0]; + } + + private static void Send(Socket socket, byte[] request) + { + int sent = 0; + while (sent < request.Length) + { + int count = socket.Send(request, sent, request.Length - sent, SocketFlags.None); + if (count <= 0) + { + throw new HttpException("The server closed the connection before the request was sent."); + } + + sent += count; + } + } + + private static HttpResponse Receive(Socket socket, HttpUrl url, bool isHead, int timeout) + { + ResponseReader reader = new(isHead); + byte[] buffer = new byte[ReceiveChunkSize]; + long lastReceived = Stopwatch.GetTimestamp(); + + while (!reader.IsComplete) + { + int available = socket.Available; + if (available > 0) + { + int read = socket.Receive(buffer, 0, Math.Min(available, buffer.Length), SocketFlags.None); + if (read > 0) + { + reader.Append(buffer, read); + lastReceived = Stopwatch.GetTimestamp(); + continue; + } + } + + // Readable with nothing to read: the server closed its end. + if (socket.Poll(PollIntervalUs, SelectMode.SelectRead) && socket.Available == 0) + { + break; + } + + if (Stopwatch.GetElapsedTime(lastReceived).TotalMilliseconds > timeout) + { + throw new HttpException($"{url.Authority} sent nothing for {timeout} ms."); + } + } + + return reader.ToResponse(url.ToString()); + } +} diff --git a/src/Cosmos.Network.Http/HttpException.cs b/src/Cosmos.Network.Http/HttpException.cs new file mode 100644 index 0000000..6308677 --- /dev/null +++ b/src/Cosmos.Network.Http/HttpException.cs @@ -0,0 +1,28 @@ +// This code is licensed under the BSD 3-Clause license (see LICENSE.txt for details) + +using System; + +namespace Cosmos.Network.Http; + +/// +/// A request that did not get a usable response: the host could not be +/// resolved or reached, the server went silent or sent something that is not +/// HTTP, a redirect led nowhere, or +/// found an error status. +/// +public sealed class HttpException : Exception +{ + /// The status of the response that failed, or 0 when there was no response. + public int StatusCode { get; } + + /// Creates an exception, with the status of the response that failed if there was one. + public HttpException(string message, int statusCode = 0) : base(message) + { + StatusCode = statusCode; + } + + /// Creates an exception for a failure underneath HTTP, a socket error for instance. + public HttpException(string message, Exception innerException) : base(message, innerException) + { + } +} diff --git a/src/Cosmos.Network.Http/HttpRequest.cs b/src/Cosmos.Network.Http/HttpRequest.cs new file mode 100644 index 0000000..10cb50b --- /dev/null +++ b/src/Cosmos.Network.Http/HttpRequest.cs @@ -0,0 +1,262 @@ +// This code is licensed under the BSD 3-Clause license (see LICENSE.txt for details) + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Cosmos.Network.Http; + +/// +/// An HTTP/1.1 request to an http:// URL. runs it on the +/// calling thread, follows redirects, and returns the response whole. +/// +/// +/// Each request opens a connection of its own and asks the server to close +/// it once it has answered. There is no TLS, so no https://. Responses are +/// asked for without content coding (Accept-Encoding: identity): the +/// body comes as the server stores it. +/// +public sealed class HttpRequest +{ + /// How long the server may stay silent by default, in milliseconds. + public const int DefaultTimeout = 15_000; + + /// How many redirects are followed by default. + public const int DefaultMaxRedirects = 5; + + /// The User-Agent sent unless names one. + public const string DefaultUserAgent = "Cosmos.Network.Http/2.0"; + + /// + /// Headers the request sets itself: Connection tells where a response + /// without a length ends, Content-Length frames the body. + /// + private static readonly string[] s_reservedHeaders = ["Connection", "Content-Length", "Transfer-Encoding"]; + + private readonly HttpUrl _url; + private readonly string _method = "GET"; + private readonly int _timeout = DefaultTimeout; + private readonly int _maxRedirects = DefaultMaxRedirects; + + /// Creates a request; sends it. + /// An http:// URL. One without a scheme is taken as an http:// one, as wget takes it. + /// is empty. + /// is not an http:// URL (https:// needs TLS, which there is none of), or it carries credentials. + /// names no host, or a port that is not a TCP port. + public HttpRequest(string url) + { + ArgumentException.ThrowIfNullOrWhiteSpace(url); + _url = HttpUrl.Parse(url); + } + + /// The URL the request goes to, written out in full. + public string Url => _url.ToString(); + + /// The method, GET unless set: HEAD, POST, PUT, DELETE and so on. + public string Method + { + get => _method; + init + { + ArgumentException.ThrowIfNullOrEmpty(value); + if (!IsToken(value)) + { + throw new ArgumentException($"'{value}' is not an HTTP method.", nameof(value)); + } + + _method = value; + } + } + + /// The body sent after the head, if any; a POST, PUT or PATCH without one sends an empty body. + public byte[]? Body { get; init; } + + /// + /// Headers sent with the request, looked up case-insensitively. They + /// replace the ones the request sends by default: Host, User-Agent, + /// Accept and Accept-Encoding. Connection, Content-Length and + /// Transfer-Encoding are the request's own, and cannot be set. + /// + public Dictionary Headers { get; } = new(StringComparer.OrdinalIgnoreCase); + + /// How long the server may stay silent while it answers before the request fails, in milliseconds. + public int Timeout + { + get => _timeout; + init + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(value); + _timeout = value; + } + } + + /// + /// How many redirects (301, 302, 303, 307, 308) are followed. With 0, + /// returns the redirect itself. + /// + public int MaxRedirects + { + get => _maxRedirects; + init + { + ArgumentOutOfRangeException.ThrowIfNegative(value); + _maxRedirects = value; + } + } + + /// Receives a line for every response and redirect, when set. + public Action? Log { get; init; } + + /// + /// Sends the request on the calling thread, follows redirects, and + /// returns the response once it has arrived whole. Any status is + /// returned, error ones included. + /// + /// + /// A 303, and a 301 or 302 to a POST, turn the request into a GET without + /// a body, as browsers do. The other redirects send it again as it was. + /// + /// sets a header the request sets itself, or holds a name or a value that cannot be sent. + /// The host could not be resolved or reached, the server went silent for longer than , the response did not come whole or is not HTTP, or the redirects went on for longer than or to a URL that cannot be followed. + public HttpResponse Send() + { + CheckHeaders(); + + HttpUrl url = _url; + string method = _method; + byte[]? body = Body; + + for (int redirects = 0; ; redirects++) + { + HttpResponse response = HttpConnection.Exchange(url, BuildRequest(url, method, body), method == "HEAD", _timeout); + Log?.Invoke($"{method} {url} {response.StatusCode} {response.ReasonPhrase} ({response.Content.Length} bytes)"); + + string? location = response.GetHeader("Location"); + if (!IsRedirect(response.StatusCode) || location is null || _maxRedirects == 0) + { + return response; + } + + if (redirects == _maxRedirects) + { + throw new HttpException($"{_url} redirected more than {_maxRedirects} times.", response.StatusCode); + } + + HttpUrl next; + try + { + next = url.Resolve(location); + } + catch (Exception exception) + { + // NotSupportedException (https://) or FormatException. + throw new HttpException($"{url} redirected to a URL that cannot be followed: {exception.Message}", response.StatusCode); + } + + if ((response.StatusCode == 303 && method != "HEAD") || (response.StatusCode is 301 or 302 && method == "POST")) + { + method = "GET"; + body = null; + } + + Log?.Invoke($"Redirected to {next}"); + url = next; + } + } + + private void CheckHeaders() + { + foreach (KeyValuePair header in Headers) + { + foreach (string reserved in s_reservedHeaders) + { + if (header.Key.Equals(reserved, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException($"The {reserved} header is set by the request itself."); + } + } + + if (!IsToken(header.Key)) + { + throw new InvalidOperationException($"'{header.Key}' is not a header name."); + } + + if (header.Value is null || header.Value.AsSpan().IndexOfAny('\r', '\n', '\0') >= 0) + { + throw new InvalidOperationException($"The value of the {header.Key} header cannot be sent: it is null or breaks the line."); + } + } + } + + private byte[] BuildRequest(HttpUrl url, string method, byte[]? body) + { + StringBuilder head = new(); + head.Append(method).Append(' ').Append(url.Target).Append(" HTTP/1.1\r\n"); + AppendHeader(head, "Host", url.Authority); + AppendHeader(head, "User-Agent", DefaultUserAgent); + AppendHeader(head, "Accept", "*/*"); + // Nothing gzip or deflate would have to undo: a Cosmos kernel has no System.IO.Compression. + AppendHeader(head, "Accept-Encoding", "identity"); + head.Append("Connection: close\r\n"); + + if (body is not null || method is "POST" or "PUT" or "PATCH") + { + head.Append("Content-Length: ").Append(body is null ? 0 : body.Length).Append("\r\n"); + } + + foreach (KeyValuePair header in Headers) + { + if (!IsDefaultHeader(header.Key)) + { + head.Append(header.Key).Append(": ").Append(header.Value).Append("\r\n"); + } + } + + head.Append("\r\n"); + + byte[] headBytes = Encoding.UTF8.GetBytes(head.ToString()); + if (body is null || body.Length == 0) + { + return headBytes; + } + + byte[] request = new byte[headBytes.Length + body.Length]; + Buffer.BlockCopy(headBytes, 0, request, 0, headBytes.Length); + Buffer.BlockCopy(body, 0, request, headBytes.Length, body.Length); + return request; + } + + /// Appends a header the request sends by default, with the value gives it if any. + private void AppendHeader(StringBuilder head, string name, string defaultValue) + { + string value = Headers.TryGetValue(name, out string? set) ? set : defaultValue; + head.Append(name).Append(": ").Append(value).Append("\r\n"); + } + + private static bool IsDefaultHeader(string name) => + name.Equals("Host", StringComparison.OrdinalIgnoreCase) + || name.Equals("User-Agent", StringComparison.OrdinalIgnoreCase) + || name.Equals("Accept", StringComparison.OrdinalIgnoreCase) + || name.Equals("Accept-Encoding", StringComparison.OrdinalIgnoreCase); + + private static bool IsRedirect(int statusCode) => statusCode is 301 or 302 or 303 or 307 or 308; + + /// Whether is a token (RFC 9110, section 5.6.2), as methods and header names are. + private static bool IsToken(string value) + { + if (value.Length == 0) + { + return false; + } + + foreach (char c in value) + { + if (!char.IsAsciiLetterOrDigit(c) && "!#$%&'*+-.^_`|~".IndexOf(c) < 0) + { + return false; + } + } + + return true; + } +} diff --git a/src/Cosmos.Network.Http/HttpResponse.cs b/src/Cosmos.Network.Http/HttpResponse.cs new file mode 100644 index 0000000..849f742 --- /dev/null +++ b/src/Cosmos.Network.Http/HttpResponse.cs @@ -0,0 +1,142 @@ +// This code is licensed under the BSD 3-Clause license (see LICENSE.txt for details) + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Cosmos.Network.Http; + +/// +/// What a server answered: the status, the headers and the whole body. +/// returns any status, error ones included; +/// turns those into an exception. +/// +public sealed class HttpResponse +{ + /// The URL that answered: the one requested, or where its redirects led. + public string Url { get; } + + /// The HTTP version the server answered with, such as HTTP/1.1. + public string Version { get; } + + /// The status code, such as 200 or 404. + public int StatusCode { get; } + + /// The text after the status code, such as OK; it may be empty. + public string ReasonPhrase { get; } + + /// + /// The headers, looked up case-insensitively. A header the server sent + /// more than once holds its values joined by ", ". + /// + public IReadOnlyDictionary Headers { get; } + + /// + /// The body, with any chunked transfer coding removed. It is empty for a + /// HEAD request and for a 204 or 304. + /// + public byte[] Content { get; } + + /// Whether the status is a 2xx one. + public bool IsSuccessStatusCode => StatusCode is >= 200 and <= 299; + + /// The Content-Type header, or null when the server sent none. + public string? ContentType => GetHeader("Content-Type"); + + internal HttpResponse(string url, string version, int statusCode, string reasonPhrase, Dictionary headers, byte[] content) + { + Url = url; + Version = version; + StatusCode = statusCode; + ReasonPhrase = reasonPhrase; + Headers = headers; + Content = content; + } + + /// A header's value, or null when the server did not send it. + public string? GetHeader(string name) => Headers.TryGetValue(name, out string? value) ? value : null; + + /// + /// The body as text, decoded with the charset of the Content-Type header: + /// UTF-8, US-ASCII, ISO-8859-1 or UTF-16. Any other charset, or none, is + /// read as UTF-8. A byte order mark is not part of the text. + /// + public string GetString() + { + byte[] content = Content; + if (content.Length >= 3 && content[0] == 0xEF && content[1] == 0xBB && content[2] == 0xBF) + { + return Encoding.UTF8.GetString(content, 3, content.Length - 3); + } + + return GetEncoding(GetCharset(ContentType)).GetString(content); + } + + /// Throws unless the status is a 2xx one. + /// This response, so the call can be chained. + /// The status is not a 2xx one; the exception carries it. + public HttpResponse EnsureSuccessStatusCode() + { + if (!IsSuccessStatusCode) + { + string status = ReasonPhrase.Length == 0 ? $"{StatusCode}" : $"{StatusCode} {ReasonPhrase}"; + throw new HttpException($"{Url} answered {status}.", StatusCode); + } + + return this; + } + + private static string? GetCharset(string? contentType) + { + if (contentType is null) + { + return null; + } + + foreach (string parameter in contentType.Split(';')) + { + string trimmed = parameter.Trim(); + if (trimmed.StartsWith("charset=", StringComparison.OrdinalIgnoreCase)) + { + return trimmed.Substring("charset=".Length).Trim('"', ' '); + } + } + + return null; + } + + /// + /// Only the encodings every .NET runtime has built in: a Cosmos kernel has + /// no code page provider for to + /// find the others in. + /// + private static Encoding GetEncoding(string? charset) + { + if (charset is null) + { + return Encoding.UTF8; + } + + if (charset.Equals("us-ascii", StringComparison.OrdinalIgnoreCase) || charset.Equals("ascii", StringComparison.OrdinalIgnoreCase)) + { + return Encoding.ASCII; + } + + if (charset.Equals("iso-8859-1", StringComparison.OrdinalIgnoreCase) || charset.Equals("latin1", StringComparison.OrdinalIgnoreCase)) + { + return Encoding.Latin1; + } + + if (charset.Equals("utf-16", StringComparison.OrdinalIgnoreCase) || charset.Equals("utf-16le", StringComparison.OrdinalIgnoreCase)) + { + return Encoding.Unicode; + } + + if (charset.Equals("utf-16be", StringComparison.OrdinalIgnoreCase)) + { + return Encoding.BigEndianUnicode; + } + + return Encoding.UTF8; + } +} diff --git a/src/Cosmos.Network.Http/HttpUrl.cs b/src/Cosmos.Network.Http/HttpUrl.cs new file mode 100644 index 0000000..ca725d3 --- /dev/null +++ b/src/Cosmos.Network.Http/HttpUrl.cs @@ -0,0 +1,249 @@ +// This code is licensed under the BSD 3-Clause license (see LICENSE.txt for details) + +using System; +using System.Globalization; +using System.Text; + +namespace Cosmos.Network.Http; + +/// +/// An http:// URL taken apart into what a request needs: the host and port to +/// connect to, and the target its request line names. The fragment is dropped, +/// as it never leaves the client. +/// +internal sealed class HttpUrl +{ + /// The port of an http:// URL that names none. + public const int DefaultPort = 80; + + private const string HexDigits = "0123456789ABCDEF"; + + /// The host name or address, an IPv6 literal without its brackets. + public string Host { get; } + + /// The TCP port to connect to. + public int Port { get; } + + /// The path and query the request line names: at least /, percent-encoded where it has to be. + public string Target { get; } + + /// The host, and the port when it is not 80: what the Host header names. + public string Authority { get; } + + private HttpUrl(string host, int port, string target) + { + Host = host; + Port = port; + Target = target; + + string name = host.Contains(':') ? "[" + host + "]" : host; + Authority = port == DefaultPort ? name : $"{name}:{port}"; + } + + public override string ToString() => "http://" + Authority + Target; + + /// + /// Takes apart an absolute URL. One without a scheme is an http:// URL, as + /// wget takes it. + /// + /// The URL is not an http:// one (there is no TLS, so not an https:// one either), or it carries credentials. + /// The URL names no host, or a port that is not a TCP port. + public static HttpUrl Parse(string url) + { + string rest = url.Trim(); + + int fragment = rest.IndexOf('#'); + if (fragment >= 0) + { + rest = rest.Substring(0, fragment); + } + + int schemeLength = SchemeLength(rest); + if (schemeLength > 0) + { + string scheme = rest.Substring(0, schemeLength); + if (scheme.Equals("https", StringComparison.OrdinalIgnoreCase)) + { + throw new NotSupportedException($"{url} needs TLS, which is not supported: use http://."); + } + + if (!scheme.Equals("http", StringComparison.OrdinalIgnoreCase)) + { + throw new NotSupportedException($"{url} is not an http:// URL."); + } + + rest = rest.Substring(schemeLength + 3); + } + + int authorityEnd = rest.IndexOfAny(['/', '?']); + string authority = authorityEnd < 0 ? rest : rest.Substring(0, authorityEnd); + string target = authorityEnd < 0 ? "/" : rest.Substring(authorityEnd); + + if (authority.Contains('@')) + { + throw new NotSupportedException($"{url} carries credentials, which are not supported."); + } + + string host; + string? portText = null; + if (authority.StartsWith('[')) + { + int close = authority.IndexOf(']'); + if (close < 0) + { + throw new FormatException($"{url} has an IPv6 address with no closing bracket."); + } + + host = authority.Substring(1, close - 1); + string afterHost = authority.Substring(close + 1); + if (afterHost.Length > 0) + { + if (afterHost[0] != ':') + { + throw new FormatException($"{url} has something other than a port after its IPv6 address."); + } + + portText = afterHost.Substring(1); + } + } + else + { + int colon = authority.IndexOf(':'); + host = colon < 0 ? authority : authority.Substring(0, colon); + portText = colon < 0 ? null : authority.Substring(colon + 1); + } + + if (host.Length == 0) + { + throw new FormatException($"{url} names no host."); + } + + // An empty port ("host:/") is the default one (RFC 3986, section 3.2.3). + int port = DefaultPort; + if (!string.IsNullOrEmpty(portText) + && (!int.TryParse(portText, NumberStyles.None, CultureInfo.InvariantCulture, out port) || port < 1 || port > 65535)) + { + throw new FormatException($"{url} names port {portText}, which is not a TCP port."); + } + + if (target.StartsWith('?')) + { + target = "/" + target; + } + + return new HttpUrl(host, port, EncodeTarget(target)); + } + + /// + /// Resolves the Location of a redirect against this URL: an absolute URL, + /// a URL without a scheme (//host/path), an absolute path, a query, + /// or a path relative to this URL's directory. + /// + /// The location is an absolute URL does not support. + /// The location is an absolute URL cannot take apart. + public HttpUrl Resolve(string location) + { + string reference = location.Trim(); + + if (SchemeLength(reference) > 0) + { + return Parse(reference); + } + + if (reference.StartsWith("//", StringComparison.Ordinal)) + { + return Parse("http:" + reference); + } + + int fragment = reference.IndexOf('#'); + if (fragment >= 0) + { + reference = reference.Substring(0, fragment); + } + + if (reference.Length == 0) + { + return this; + } + + string path = Target; + int query = path.IndexOf('?'); + if (query >= 0) + { + path = path.Substring(0, query); + } + + string target = reference[0] switch + { + '/' => reference, + '?' => path + reference, + // The target always starts with '/', so there is a directory to resolve against. + _ => path.Substring(0, path.LastIndexOf('/') + 1) + reference, + }; + + return new HttpUrl(Host, Port, EncodeTarget(target)); + } + + /// + /// The length of the scheme starts with, when it + /// starts with scheme:// (RFC 3986, section 3.1); otherwise 0. + /// + private static int SchemeLength(string url) + { + int end = url.IndexOf("://", StringComparison.Ordinal); + if (end <= 0) + { + return 0; + } + + for (int i = 0; i < end; i++) + { + char c = url[i]; + bool valid = char.IsAsciiLetter(c) || (i > 0 && (char.IsAsciiDigit(c) || c is '+' or '-' or '.')); + if (!valid) + { + return 0; + } + } + + return end; + } + + /// + /// Percent-encodes what a request line cannot carry as is: spaces, control + /// characters and anything beyond ASCII, as UTF-8. What is already + /// percent-encoded is left alone. + /// + private static string EncodeTarget(string target) + { + bool clean = true; + foreach (char c in target) + { + if (c <= ' ' || c > '~') + { + clean = false; + break; + } + } + + if (clean) + { + return target; + } + + StringBuilder encoded = new(target.Length + 16); + foreach (byte b in Encoding.UTF8.GetBytes(target)) + { + if (b <= ' ' || b > '~') + { + encoded.Append('%').Append(HexDigits[b >> 4]).Append(HexDigits[b & 0xF]); + } + else + { + encoded.Append((char)b); + } + } + + return encoded.ToString(); + } +} diff --git a/src/Cosmos.Network.Http/ResponseReader.cs b/src/Cosmos.Network.Http/ResponseReader.cs new file mode 100644 index 0000000..ef6f566 --- /dev/null +++ b/src/Cosmos.Network.Http/ResponseReader.cs @@ -0,0 +1,298 @@ +// This code is licensed under the BSD 3-Clause license (see LICENSE.txt for details) + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; + +namespace Cosmos.Network.Http; + +/// +/// Takes a response apart as its bytes arrive, and tells when it has arrived +/// whole (RFC 9112, section 6.3), so the connection need not be read to its +/// end: a response with a Content-Length or a chunked body is complete +/// before the server closes. One without either ends where the connection +/// does. +/// +internal sealed class ResponseReader +{ + /// The longest response head taken, in bytes. + internal const int MaxHeadLength = 64 * 1024; + + /// The largest Content-Length the receive buffer is sized for up front, in bytes. + private const int MaxPreallocatedLength = 64 * 1024 * 1024; + + private enum Framing + { + /// No body: a response to HEAD, a 1xx, a 204 or a 304. + None, + + /// As many bytes as Content-Length says. + Length, + + /// The chunked transfer coding. + Chunked, + + /// Everything up to the end of the connection. + UntilClose, + } + + private readonly MemoryStream _received = new(); + private readonly bool _isHead; + + private Head? _head; + private Framing _framing; + private long _contentLength; + + /// Where the head being looked for starts: past any 1xx interim response. + private int _headStart; + + /// Where the search for the end of that head resumes. + private int _scanFrom; + + private int _bodyStart; + + /// Where the search for the end of a chunked body resumes. + private int _chunkScan; + + /// Whether the request was a HEAD one, whose response has no body whatever its headers say. + public ResponseReader(bool isHead) + { + _isHead = isHead; + } + + /// Whether the whole response has arrived. + public bool IsComplete { get; private set; } + + /// Takes the next bytes of the response. + /// The bytes are not an HTTP response. + public void Append(byte[] buffer, int count) + { + _received.Write(buffer, 0, count); + + if (_head is null && !TryReadHead()) + { + return; + } + + IsComplete = _framing switch + { + Framing.None => true, + Framing.Length => _received.Length - _bodyStart >= _contentLength, + Framing.Chunked => ChunkedBody.FindEnd(_received.GetBuffer(), (int)_received.Length, ref _chunkScan), + _ => false, + }; + } + + /// + /// The response, once it is complete or the server has closed the + /// connection, which completes a response that runs until it does. + /// + /// The URL that answered. + /// The connection closed before the response was whole. + public HttpResponse ToResponse(string url) + { + if (_head is null) + { + throw new HttpException(_received.Length == _headStart + ? "The server closed the connection without answering." + : "The server closed the connection in the middle of the response head."); + } + + byte[] data = _received.GetBuffer(); + int received = (int)_received.Length - _bodyStart; + byte[] content; + switch (_framing) + { + case Framing.None: + content = []; + break; + + case Framing.Length: + if (received < _contentLength) + { + throw new HttpException($"The server closed the connection after {received} of {_contentLength} bytes.", _head.StatusCode); + } + + content = Slice(data, _bodyStart, (int)_contentLength); + break; + + case Framing.Chunked: + content = ChunkedBody.Decode(data, _bodyStart, (int)_received.Length); + break; + + default: + content = Slice(data, _bodyStart, received); + break; + } + + return new HttpResponse(url, _head.Version, _head.StatusCode, _head.ReasonPhrase, _head.Headers, content); + } + + /// Looks for the end of the head, and takes the head apart once it arrived. + /// Whether the head of the final response arrived. + private bool TryReadHead() + { + byte[] data = _received.GetBuffer(); + int length = (int)_received.Length; + + while (true) + { + int end = IndexOfHeadEnd(data, _scanFrom, length); + if (end < 0) + { + if (length - _headStart > MaxHeadLength) + { + throw new HttpException($"The response head is longer than {MaxHeadLength} bytes."); + } + + // The blank line may straddle what arrives next. + _scanFrom = Math.Max(_headStart, length - 3); + return false; + } + + Head head = ParseHead(data, _headStart, end - _headStart); + if (head.StatusCode is >= 100 and <= 199 && head.StatusCode != 101) + { + // An interim response, such as 100 Continue: the final one follows. + _headStart = _scanFrom = end; + continue; + } + + _head = head; + _bodyStart = _chunkScan = end; + ChooseFraming(head); + return true; + } + } + + private void ChooseFraming(Head head) + { + if (_isHead || head.StatusCode is 204 or 304 or (>= 100 and <= 199)) + { + _framing = Framing.None; + return; + } + + // Transfer-Encoding wins over Content-Length. A coding other than + // chunked last leaves nothing to tell the end of the body by but the + // end of the connection. + if (head.Headers.TryGetValue("Transfer-Encoding", out string? transferEncoding)) + { + _framing = transferEncoding.Trim().EndsWith("chunked", StringComparison.OrdinalIgnoreCase) ? Framing.Chunked : Framing.UntilClose; + return; + } + + if (head.Headers.TryGetValue("Content-Length", out string? contentLength)) + { + _framing = Framing.Length; + _contentLength = ParseContentLength(contentLength); + if (_contentLength <= MaxPreallocatedLength) + { + _received.Capacity = Math.Max(_received.Capacity, _bodyStart + (int)_contentLength); + } + + return; + } + + _framing = Framing.UntilClose; + } + + /// + /// A Content-Length, which a server that sent it more than once joined + /// with commas: taken if every copy agrees (RFC 9110, section 8.6). + /// + private static long ParseContentLength(string value) + { + long length = -1; + foreach (string part in value.Split(',')) + { + if (!long.TryParse(part.Trim(), NumberStyles.None, CultureInfo.InvariantCulture, out long parsed) + || parsed > int.MaxValue + || (length >= 0 && parsed != length)) + { + throw new HttpException($"The response has an invalid Content-Length: {value}."); + } + + length = parsed; + } + + return length; + } + + /// Takes apart a status line and the header lines after it (RFC 9112, sections 4 and 5). + private static Head ParseHead(byte[] data, int start, int length) + { + // Headers are ASCII but for the odd value, which UTF-8 reads as well as anything. + string[] lines = Encoding.UTF8.GetString(data, start, length).Split("\r\n"); + string statusLine = lines[0]; + + // HTTP-version SP 3DIGIT [SP reason-phrase] + int space = statusLine.IndexOf(' '); + int statusCode = 0; + if (!statusLine.StartsWith("HTTP/", StringComparison.Ordinal) + || space < 0 + || statusLine.Length < space + 4 + || (statusLine.Length > space + 4 && statusLine[space + 4] != ' ') + || !int.TryParse(statusLine.Substring(space + 1, 3), NumberStyles.None, CultureInfo.InvariantCulture, out statusCode)) + { + string shown = statusLine.Length > 64 ? statusLine.Substring(0, 64) + "..." : statusLine; + throw new HttpException($"The server answered with something that is not HTTP: {shown}"); + } + + string version = statusLine.Substring(0, space); + string reasonPhrase = statusLine.Length > space + 5 ? statusLine.Substring(space + 5) : string.Empty; + + Dictionary headers = new(StringComparer.OrdinalIgnoreCase); + for (int i = 1; i < lines.Length; i++) + { + string line = lines[i]; + int colon = line.IndexOf(':'); + + // Lines with no name, and the obsolete continuations that start with whitespace, hold nothing to keep. + if (colon <= 0 || line[0] is ' ' or '\t') + { + continue; + } + + string name = line.Substring(0, colon).Trim(); + string value = line.Substring(colon + 1).Trim(); + headers[name] = headers.TryGetValue(name, out string? previous) ? previous + ", " + value : value; + } + + return new Head(version, statusCode, reasonPhrase, headers); + } + + /// The index just past the blank line that ends a head, or -1. + private static int IndexOfHeadEnd(byte[] data, int start, int length) + { + for (int i = start; i + 3 < length; i++) + { + if (data[i] == '\r' && data[i + 1] == '\n' && data[i + 2] == '\r' && data[i + 3] == '\n') + { + return i + 4; + } + } + + return -1; + } + + private static byte[] Slice(byte[] data, int start, int length) + { + byte[] slice = new byte[length]; + Buffer.BlockCopy(data, start, slice, 0, length); + return slice; + } + + private sealed class Head(string version, int statusCode, string reasonPhrase, Dictionary headers) + { + public string Version { get; } = version; + + public int StatusCode { get; } = statusCode; + + public string ReasonPhrase { get; } = reasonPhrase; + + public Dictionary Headers { get; } = headers; + } +} diff --git a/tests/Cosmos.Network.Http.Tests/Cosmos.Network.Http.Tests.csproj b/tests/Cosmos.Network.Http.Tests/Cosmos.Network.Http.Tests.csproj new file mode 100644 index 0000000..4193a30 --- /dev/null +++ b/tests/Cosmos.Network.Http.Tests/Cosmos.Network.Http.Tests.csproj @@ -0,0 +1,21 @@ + + + + net10.0 + latest + enable + disable + false + + + + + + + + + + + + + diff --git a/tests/Cosmos.Network.Http.Tests/HttpRequestTests.cs b/tests/Cosmos.Network.Http.Tests/HttpRequestTests.cs new file mode 100644 index 0000000..8330b48 --- /dev/null +++ b/tests/Cosmos.Network.Http.Tests/HttpRequestTests.cs @@ -0,0 +1,390 @@ +// This code is licensed under the BSD 3-Clause license (see LICENSE.txt for details) + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Text; +using NUnit.Framework; + +namespace Cosmos.Network.Http.Tests; + +/// +/// Sends requests to a loopback server that answers with bytes the tests +/// write out, and checks what went over the wire both ways. +/// +[TestFixture] +public class HttpRequestTests +{ + // Short, so a client that waits for a close it should not need fails fast. + private const int Timeout = 2000; + + [Test] + public void Get_SendsAMinimalRequestAndReturnsTheBody() + { + using TestServer server = new(); + server.Then("HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: 5\r\n\r\nhello"); + + HttpResponse response = new HttpRequest(server.Url("/page?x=1")) { Timeout = Timeout }.Send(); + + Assert.That(response.StatusCode, Is.EqualTo(200)); + Assert.That(response.GetString(), Is.EqualTo("hello")); + Assert.That(response.Url, Is.EqualTo(server.Url("/page?x=1"))); + + TestRequest request = server.Requests.Single(); + Assert.That(request.Head, Does.StartWith("GET /page?x=1 HTTP/1.1\r\n")); + Assert.That(request.Headers["Host"], Is.EqualTo($"127.0.0.1:{server.Port}")); + Assert.That(request.Headers["Connection"], Is.EqualTo("close")); + Assert.That(request.Headers["Accept-Encoding"], Is.EqualTo("identity")); + Assert.That(request.Headers["User-Agent"], Is.EqualTo(HttpRequest.DefaultUserAgent)); + Assert.That(request.Headers.ContainsKey("Content-Length"), Is.False); + } + + [Test] + public void ContentLength_DoesNotWaitForTheServerToClose() + { + using TestServer server = new(); + server.Then(connection => + { + connection.ReadRequest(); + connection.Write("HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok"); + connection.WaitForClientClose(); + }); + + HttpResponse response = new HttpRequest(server.Url("/")) { Timeout = Timeout }.Send(); + + Assert.That(response.GetString(), Is.EqualTo("ok")); + } + + [Test] + public void Chunked_DoesNotWaitForTheServerToClose() + { + using TestServer server = new(); + server.Then(connection => + { + connection.ReadRequest(); + connection.WriteSlowly("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n3\r\nabc\r\n2\r\nde\r\n0\r\n\r\n"); + connection.WaitForClientClose(); + }); + + HttpResponse response = new HttpRequest(server.Url("/")) { Timeout = Timeout }.Send(); + + Assert.That(response.GetString(), Is.EqualTo("abcde")); + } + + [Test] + public void NoLength_ReadsUntilTheServerCloses() + { + using TestServer server = new(); + server.Then(connection => + { + connection.ReadRequest(); + connection.WriteSlowly("HTTP/1.0 200 OK\r\n\r\n"); + connection.Write(new string('x', 100_000)); + }); + + HttpResponse response = new HttpRequest(server.Url("/")) { Timeout = Timeout }.Send(); + + Assert.That(response.Content, Has.Length.EqualTo(100_000)); + } + + [Test] + public void LargeBody_ArrivesWhole() + { + byte[] body = new byte[3 * 1024 * 1024]; + new Random(42).NextBytes(body); + using TestServer server = new(); + server.Then(connection => + { + connection.ReadRequest(); + connection.Write($"HTTP/1.1 200 OK\r\nContent-Length: {body.Length}\r\n\r\n"); + connection.Write(body); + }); + + HttpResponse response = new HttpRequest(server.Url("/big.bin")) { Timeout = Timeout }.Send(); + + Assert.That(response.Content, Is.EqualTo(body)); + } + + [Test] + public void Head_ReturnsNoBody() + { + using TestServer server = new(); + server.Then(connection => + { + connection.ReadRequest(); + connection.Write("HTTP/1.1 200 OK\r\nContent-Length: 1234\r\n\r\n"); + connection.WaitForClientClose(); + }); + + HttpResponse response = new HttpRequest(server.Url("/")) { Method = "HEAD", Timeout = Timeout }.Send(); + + Assert.That(response.Content, Is.Empty); + Assert.That(response.GetHeader("Content-Length"), Is.EqualTo("1234")); + Assert.That(server.Requests.Single().Method, Is.EqualTo("HEAD")); + } + + [Test] + public void Post_SendsTheBodyWithItsLength() + { + using TestServer server = new(); + server.Then("HTTP/1.1 201 Created\r\nContent-Length: 0\r\n\r\n"); + + HttpResponse response = new HttpRequest(server.Url("/api")) + { + Method = "POST", + Body = Encoding.UTF8.GetBytes("{\"a\":1}"), + Headers = { ["Content-Type"] = "application/json" }, + Timeout = Timeout, + }.Send(); + + Assert.That(response.StatusCode, Is.EqualTo(201)); + TestRequest request = server.Requests.Single(); + Assert.That(request.Method, Is.EqualTo("POST")); + Assert.That(request.Headers["Content-Length"], Is.EqualTo("7")); + Assert.That(request.Headers["Content-Type"], Is.EqualTo("application/json")); + Assert.That(Encoding.UTF8.GetString(request.Body), Is.EqualTo("{\"a\":1}")); + } + + [Test] + public void PostWithoutBody_SendsAZeroLength() + { + using TestServer server = new(); + server.Then("HTTP/1.1 204 No Content\r\n\r\n"); + + new HttpRequest(server.Url("/")) { Method = "POST", Timeout = Timeout }.Send(); + + Assert.That(server.Requests.Single().Headers["Content-Length"], Is.EqualTo("0")); + } + + [Test] + public void Headers_ReplaceTheDefaultsAndAddToThem() + { + using TestServer server = new(); + server.Then("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"); + + new HttpRequest(server.Url("/")) + { + Headers = + { + ["host"] = "virtual.example", + ["User-Agent"] = "AuraOS/1.0", + ["X-Extra"] = "yes", + }, + Timeout = Timeout, + }.Send(); + + TestRequest request = server.Requests.Single(); + Assert.That(request.Headers["Host"], Is.EqualTo("virtual.example")); + Assert.That(request.Headers["User-Agent"], Is.EqualTo("AuraOS/1.0")); + Assert.That(request.Headers["X-Extra"], Is.EqualTo("yes")); + Assert.That(request.Head.Split("\r\n").Count(line => line.StartsWith("Host:", StringComparison.OrdinalIgnoreCase)), Is.EqualTo(1)); + } + + [TestCase("Connection")] + [TestCase("content-length")] + [TestCase("Transfer-Encoding")] + public void Headers_TheRequestSetsItselfAreRefused(string name) + { + HttpRequest request = new("http://127.0.0.1:1/") { Headers = { [name] = "x" } }; + + Assert.Throws(() => request.Send()); + } + + [Test] + public void Headers_ThatBreakTheLineAreRefused() + { + HttpRequest request = new("http://127.0.0.1:1/") { Headers = { ["X-Evil"] = "a\r\nInjected: yes" } }; + + Assert.Throws(() => request.Send()); + } + + [Test] + public void ErrorStatus_IsReturned_AndEnsureSuccessThrows() + { + using TestServer server = new(); + server.Then("HTTP/1.1 404 Not Found\r\nContent-Length: 9\r\n\r\nnot found"); + + HttpResponse response = new HttpRequest(server.Url("/missing")) { Timeout = Timeout }.Send(); + + Assert.That(response.StatusCode, Is.EqualTo(404)); + Assert.That(response.GetString(), Is.EqualTo("not found")); + HttpException exception = Assert.Throws(() => response.EnsureSuccessStatusCode())!; + Assert.That(exception.StatusCode, Is.EqualTo(404)); + Assert.That(exception.Message, Does.Contain("404 Not Found")); + } + + [Test] + public void Redirects_AreFollowed() + { + using TestServer server = new(); + server.Then("HTTP/1.1 301 Moved Permanently\r\nLocation: /next\r\nContent-Length: 0\r\n\r\n") + .Then($"HTTP/1.1 302 Found\r\nLocation: {server.Url("/last")}\r\nContent-Length: 0\r\n\r\n") + .Then("HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ndone"); + List log = []; + + HttpResponse response = new HttpRequest(server.Url("/first")) { Timeout = Timeout, Log = log.Add }.Send(); + + Assert.That(response.GetString(), Is.EqualTo("done")); + Assert.That(response.Url, Is.EqualTo(server.Url("/last"))); + Assert.That(server.Requests.Select(request => request.Target), Is.EqualTo(new[] { "/first", "/next", "/last" })); + Assert.That(log, Has.Count.EqualTo(5)); + } + + [Test] + public void SeeOther_TurnsAPostIntoAGet() + { + using TestServer server = new(); + server.Then("HTTP/1.1 303 See Other\r\nLocation: /result\r\nContent-Length: 0\r\n\r\n") + .Then("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"); + + new HttpRequest(server.Url("/form")) { Method = "POST", Body = [1, 2, 3], Timeout = Timeout }.Send(); + + TestRequest second = server.Requests.Last(); + Assert.That(second.Method, Is.EqualTo("GET")); + Assert.That(second.Body, Is.Empty); + Assert.That(second.Headers.ContainsKey("Content-Length"), Is.False); + } + + [Test] + public void TemporaryRedirect_KeepsThePost() + { + using TestServer server = new(); + server.Then("HTTP/1.1 307 Temporary Redirect\r\nLocation: /elsewhere\r\nContent-Length: 0\r\n\r\n") + .Then("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"); + + new HttpRequest(server.Url("/form")) { Method = "POST", Body = [1, 2, 3], Timeout = Timeout }.Send(); + + TestRequest second = server.Requests.Last(); + Assert.That(second.Method, Is.EqualTo("POST")); + Assert.That(second.Body, Is.EqualTo(new byte[] { 1, 2, 3 })); + } + + [Test] + public void Redirects_StopAtTheLimit() + { + using TestServer server = new(); + for (int i = 0; i < 3; i++) + { + server.Then("HTTP/1.1 302 Found\r\nLocation: /again\r\nContent-Length: 0\r\n\r\n"); + } + + HttpException exception = Assert.Throws(() => new HttpRequest(server.Url("/")) { MaxRedirects = 2, Timeout = Timeout }.Send())!; + Assert.That(exception.StatusCode, Is.EqualTo(302)); + } + + [Test] + public void Redirects_AreReturnedWithoutLimit() + { + using TestServer server = new(); + server.Then("HTTP/1.1 301 Moved Permanently\r\nLocation: /next\r\nContent-Length: 0\r\n\r\n"); + + HttpResponse response = new HttpRequest(server.Url("/")) { MaxRedirects = 0, Timeout = Timeout }.Send(); + + Assert.That(response.StatusCode, Is.EqualTo(301)); + Assert.That(response.GetHeader("Location"), Is.EqualTo("/next")); + } + + [Test] + public void RedirectToHttps_Throws() + { + using TestServer server = new(); + server.Then("HTTP/1.1 301 Moved Permanently\r\nLocation: https://secure.example/\r\nContent-Length: 0\r\n\r\n"); + + HttpException exception = Assert.Throws(() => new HttpRequest(server.Url("/")) { Timeout = Timeout }.Send())!; + Assert.That(exception.StatusCode, Is.EqualTo(301)); + Assert.That(exception.Message, Does.Contain("TLS")); + } + + [Test] + public void SilentServer_TimesOut() + { + using TestServer server = new(); + server.Then(connection => + { + connection.ReadRequest(); + connection.Hang(); + }); + + HttpException exception = Assert.Throws(() => new HttpRequest(server.Url("/")) { Timeout = 300 }.Send())!; + Assert.That(exception.Message, Does.Contain("sent nothing")); + } + + [Test] + public void TruncatedBody_Throws() + { + using TestServer server = new(); + server.Then("HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nabc"); + + HttpException exception = Assert.Throws(() => new HttpRequest(server.Url("/")) { Timeout = Timeout }.Send())!; + Assert.That(exception.Message, Does.Contain("3 of 10")); + } + + [Test] + public void ClosedWithoutAnswer_Throws() + { + using TestServer server = new(); + server.Then(connection => connection.ReadRequest()); + + HttpException exception = Assert.Throws(() => new HttpRequest(server.Url("/")) { Timeout = Timeout }.Send())!; + Assert.That(exception.Message, Does.Contain("without answering")); + } + + [Test] + public void RefusedConnection_Throws() + { + // A port nothing listens on: bound, then released. + TcpListener listener = new(IPAddress.Loopback, 0); + listener.Start(); + int port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + + HttpException exception = Assert.Throws(() => new HttpRequest($"http://127.0.0.1:{port}/") { Timeout = Timeout }.Send())!; + Assert.That(exception.Message, Does.StartWith($"Could not connect to 127.0.0.1:{port}")); + } + + [Test] + public void UnknownHost_Throws() + { + HttpException exception = Assert.Throws(() => new HttpRequest("http://no-such-host.invalid/") { Timeout = Timeout }.Send())!; + Assert.That(exception.Message, Does.StartWith("Could not resolve no-such-host.invalid")); + } + + [Test] + public void GetString_UsesTheCharsetAndSkipsTheByteOrderMark() + { + using TestServer server = new(); + server.Then(connection => + { + connection.ReadRequest(); + connection.Write("HTTP/1.1 200 OK\r\nContent-Type: text/plain; charset=\"ISO-8859-1\"\r\nContent-Length: 1\r\n\r\n"); + connection.Write([0xE9]); + }).Then(connection => + { + connection.ReadRequest(); + connection.Write("HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\n"); + connection.Write([0xEF, 0xBB, 0xBF, (byte)'{']); + }); + + Assert.That(new HttpRequest(server.Url("/latin1")) { Timeout = Timeout }.Send().GetString(), Is.EqualTo("é")); + Assert.That(new HttpRequest(server.Url("/bom")) { Timeout = Timeout }.Send().GetString(), Is.EqualTo("{")); + } + + [TestCase("GET POST")] + [TestCase("G\r\nET")] + [TestCase("")] + public void Method_MustBeAToken(string method) + { + Assert.Throws(() => _ = new HttpRequest("http://host/") { Method = method }); + } + + [Test] + public void Settings_AreValidated() + { + Assert.Throws(() => _ = new HttpRequest("http://host/") { Timeout = 0 }); + Assert.Throws(() => _ = new HttpRequest("http://host/") { MaxRedirects = -1 }); + Assert.Throws(() => _ = new HttpRequest(" ")); + Assert.Throws(() => _ = new HttpRequest("https://host/")); + } +} diff --git a/tests/Cosmos.Network.Http.Tests/HttpUrlTests.cs b/tests/Cosmos.Network.Http.Tests/HttpUrlTests.cs new file mode 100644 index 0000000..7dd46f5 --- /dev/null +++ b/tests/Cosmos.Network.Http.Tests/HttpUrlTests.cs @@ -0,0 +1,95 @@ +// This code is licensed under the BSD 3-Clause license (see LICENSE.txt for details) + +using System; +using NUnit.Framework; + +namespace Cosmos.Network.Http.Tests; + +[TestFixture] +public class HttpUrlTests +{ + [TestCase("http://httpforever.com/", "httpforever.com", 80, "/")] + [TestCase("http://httpforever.com", "httpforever.com", 80, "/")] + [TestCase("HTTP://Example.com/a/b.html", "Example.com", 80, "/a/b.html")] + [TestCase("httpforever.com/page", "httpforever.com", 80, "/page")] + [TestCase("http://10.0.2.2:8080/x?y=1", "10.0.2.2", 8080, "/x?y=1")] + [TestCase("http://host?q=1", "host", 80, "/?q=1")] + [TestCase("http://host/a#section", "host", 80, "/a")] + [TestCase("http://host:/a", "host", 80, "/a")] + [TestCase(" http://host/a ", "host", 80, "/a")] + [TestCase("http://[::1]:8000/", "::1", 8000, "/")] + [TestCase("host/search?u=http://other/", "host", 80, "/search?u=http://other/")] + public void Parse_TakesTheUrlApart(string url, string host, int port, string target) + { + HttpUrl parsed = HttpUrl.Parse(url); + + Assert.Multiple(() => + { + Assert.That(parsed.Host, Is.EqualTo(host)); + Assert.That(parsed.Port, Is.EqualTo(port)); + Assert.That(parsed.Target, Is.EqualTo(target)); + }); + } + + [TestCase("http://host/", "host")] + [TestCase("http://host:8080/", "host:8080")] + [TestCase("http://[::1]:8080/", "[::1]:8080")] + public void Authority_LeavesOutTheDefaultPort(string url, string authority) + { + Assert.That(HttpUrl.Parse(url).Authority, Is.EqualTo(authority)); + } + + [Test] + public void Parse_PercentEncodesWhatARequestLineCannotCarry() + { + Assert.That(HttpUrl.Parse("http://host/a file/é?x=a b").Target, Is.EqualTo("/a%20file/%C3%A9?x=a%20b")); + Assert.That(HttpUrl.Parse("http://host/a%20b").Target, Is.EqualTo("/a%20b")); + } + + [Test] + public void Parse_RefusesHttps() + { + NotSupportedException exception = Assert.Throws(() => HttpUrl.Parse("https://host/"))!; + Assert.That(exception.Message, Does.Contain("TLS")); + } + + [TestCase("ftp://host/")] + [TestCase("http://user:pass@host/")] + public void Parse_RefusesWhatItDoesNotSupport(string url) + { + Assert.Throws(() => HttpUrl.Parse(url)); + } + + [TestCase("http://")] + [TestCase("http:///path")] + [TestCase("http://host:0/")] + [TestCase("http://host:65536/")] + [TestCase("http://host:http/")] + [TestCase("http://[::1/")] + [TestCase("http://[::1]x/")] + public void Parse_RefusesMalformedUrls(string url) + { + Assert.Throws(() => HttpUrl.Parse(url)); + } + + [TestCase("http://other:81/x", "http://other:81/x")] + [TestCase("//other/x", "http://other/x")] + [TestCase("/x/y", "http://host:8080/x/y")] + [TestCase("?page=2", "http://host:8080/dir/file?page=2")] + [TestCase("other", "http://host:8080/dir/other")] + [TestCase("sub/other?a=1", "http://host:8080/dir/sub/other?a=1")] + [TestCase("#top", "http://host:8080/dir/file?q=1")] + [TestCase("/a b", "http://host:8080/a%20b")] + public void Resolve_FollowsEveryKindOfLocation(string location, string expected) + { + HttpUrl url = HttpUrl.Parse("http://host:8080/dir/file?q=1"); + + Assert.That(url.Resolve(location).ToString(), Is.EqualTo(expected)); + } + + [Test] + public void Resolve_RefusesHttps() + { + Assert.Throws(() => HttpUrl.Parse("http://host/").Resolve("https://host/")); + } +} diff --git a/tests/Cosmos.Network.Http.Tests/ResponseReaderTests.cs b/tests/Cosmos.Network.Http.Tests/ResponseReaderTests.cs new file mode 100644 index 0000000..9c7f6c0 --- /dev/null +++ b/tests/Cosmos.Network.Http.Tests/ResponseReaderTests.cs @@ -0,0 +1,243 @@ +// This code is licensed under the BSD 3-Clause license (see LICENSE.txt for details) + +using System.Text; +using NUnit.Framework; + +namespace Cosmos.Network.Http.Tests; + +/// +/// Feeds responses to the reader in every split a connection could deliver +/// them in, and checks it tells where they end. +/// +[TestFixture] +public class ResponseReaderTests +{ + private const string Url = "http://host/"; + + [Test] + public void ContentLength_CompletesWithoutTheConnectionClosing() + { + const string response = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 5\r\n\r\nhello"; + + foreach (ResponseReader reader in FeedInEverySplit(response)) + { + Assert.That(reader.IsComplete, Is.True); + HttpResponse parsed = reader.ToResponse(Url); + Assert.That(parsed.StatusCode, Is.EqualTo(200)); + Assert.That(parsed.ReasonPhrase, Is.EqualTo("OK")); + Assert.That(parsed.Version, Is.EqualTo("HTTP/1.1")); + Assert.That(parsed.ContentType, Is.EqualTo("text/plain")); + Assert.That(Encoding.ASCII.GetString(parsed.Content), Is.EqualTo("hello")); + } + } + + [Test] + public void ContentLength_IsNotCompleteBeforeTheLastByte() + { + ResponseReader reader = Feed("HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhell"); + + Assert.That(reader.IsComplete, Is.False); + HttpException exception = Assert.Throws(() => reader.ToResponse(Url))!; + Assert.That(exception.Message, Does.Contain("4 of 5")); + } + + [Test] + public void ContentLength_IgnoresBytesPastTheBody() + { + HttpResponse response = Feed("HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nhello").ToResponse(Url); + + Assert.That(Encoding.ASCII.GetString(response.Content), Is.EqualTo("he")); + } + + [Test] + public void Chunked_CompletesAtTheLastChunkAndTrailers() + { + const string response = "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n" + + "5;name=value\r\nhello\r\n" + + "1\r\n,\r\n" + + "6 \r\n world\r\n" + + "0\r\nExpires: never\r\n\r\n"; + + foreach (ResponseReader reader in FeedInEverySplit(response)) + { + Assert.That(reader.IsComplete, Is.True); + Assert.That(Encoding.ASCII.GetString(reader.ToResponse(Url).Content), Is.EqualTo("hello, world")); + } + } + + [Test] + public void Chunked_IsNotCompleteBeforeTheBlankLineAfterTheLastChunk() + { + ResponseReader reader = Feed("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n3\r\nabc\r\n0\r\n"); + + Assert.That(reader.IsComplete, Is.False); + } + + [Test] + public void Chunked_TruncatedBodyThrows() + { + ResponseReader reader = Feed("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\nA\r\nabc"); + + Assert.Throws(() => reader.ToResponse(Url)); + } + + [TestCase("zz")] + [TestCase("")] + [TestCase("5x")] + public void Chunked_MalformedSizeThrows(string size) + { + Assert.Throws(() => Feed($"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n{size}\r\nhello\r\n0\r\n\r\n")); + } + + [Test] + public void TransferEncoding_WinsOverContentLength() + { + ResponseReader reader = Feed("HTTP/1.1 200 OK\r\nContent-Length: 100\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nok\r\n0\r\n\r\n"); + + Assert.That(reader.IsComplete, Is.True); + Assert.That(Encoding.ASCII.GetString(reader.ToResponse(Url).Content), Is.EqualTo("ok")); + } + + [Test] + public void NoLength_RunsUntilTheConnectionCloses() + { + ResponseReader reader = Feed("HTTP/1.0 200 OK\r\n\r\neverything until the end"); + + Assert.That(reader.IsComplete, Is.False); + Assert.That(Encoding.ASCII.GetString(reader.ToResponse(Url).Content), Is.EqualTo("everything until the end")); + } + + [TestCase(204)] + [TestCase(304)] + public void StatusesWithoutBody_CompleteAtTheHead(int status) + { + ResponseReader reader = Feed($"HTTP/1.1 {status} Whatever\r\nContent-Length: 10\r\n\r\n"); + + Assert.That(reader.IsComplete, Is.True); + Assert.That(reader.ToResponse(Url).Content, Is.Empty); + } + + [Test] + public void HeadRequest_CompletesAtTheHead() + { + ResponseReader reader = new(isHead: true); + byte[] bytes = Encoding.ASCII.GetBytes("HTTP/1.1 200 OK\r\nContent-Length: 1234\r\n\r\n"); + reader.Append(bytes, bytes.Length); + + Assert.That(reader.IsComplete, Is.True); + HttpResponse response = reader.ToResponse(Url); + Assert.That(response.Content, Is.Empty); + Assert.That(response.GetHeader("content-length"), Is.EqualTo("1234")); + } + + [Test] + public void InterimResponses_AreSkipped() + { + const string response = "HTTP/1.1 100 Continue\r\n\r\nHTTP/1.1 103 Early Hints\r\nLink: \r\n\r\n" + + "HTTP/1.1 201 Created\r\nContent-Length: 2\r\n\r\nok"; + + foreach (ResponseReader reader in FeedInEverySplit(response)) + { + HttpResponse parsed = reader.ToResponse(Url); + Assert.That(parsed.StatusCode, Is.EqualTo(201)); + Assert.That(parsed.GetHeader("Link"), Is.Null); + Assert.That(Encoding.ASCII.GetString(parsed.Content), Is.EqualTo("ok")); + } + } + + [Test] + public void Headers_AreCaseInsensitiveAndRepeatsAreJoined() + { + HttpResponse response = Feed("HTTP/1.1 200 OK\r\nX-Thing: a\r\nx-thing: b\r\nSpaced : value \r\nContent-Length: 0\r\n\r\n").ToResponse(Url); + + Assert.That(response.GetHeader("X-THING"), Is.EqualTo("a, b")); + Assert.That(response.Headers["spaced"], Is.EqualTo("value")); + } + + [Test] + public void StatusLine_WithoutReasonPhrase() + { + HttpResponse response = Feed("HTTP/1.1 404\r\nContent-Length: 0\r\n\r\n").ToResponse(Url); + + Assert.That(response.StatusCode, Is.EqualTo(404)); + Assert.That(response.ReasonPhrase, Is.Empty); + Assert.That(response.IsSuccessStatusCode, Is.False); + } + + [TestCase("SSH-2.0-OpenSSH_9.6\r\n\r\n")] + [TestCase("HTTP/1.1 2000 OK\r\n\r\n")] + [TestCase("HTTP/1.1 abc OK\r\n\r\n")] + public void NotHttp_Throws(string response) + { + Assert.Throws(() => Feed(response)); + } + + [TestCase("abc")] + [TestCase("5, 6")] + [TestCase("-1")] + public void InvalidContentLength_Throws(string length) + { + Assert.Throws(() => Feed($"HTTP/1.1 200 OK\r\nContent-Length: {length}\r\n\r\n")); + } + + [Test] + public void RepeatedContentLength_IsTakenWhenTheCopiesAgree() + { + ResponseReader reader = Feed("HTTP/1.1 200 OK\r\nContent-Length: 2\r\nContent-Length: 2\r\n\r\nok"); + + Assert.That(reader.IsComplete, Is.True); + } + + [Test] + public void OversizedHead_Throws() + { + string response = "HTTP/1.1 200 OK\r\nX-Big: " + new string('a', ResponseReader.MaxHeadLength) + "\r\n"; + + Assert.Throws(() => Feed(response)); + } + + [Test] + public void NoAnswer_Throws() + { + HttpException exception = Assert.Throws(() => new ResponseReader(isHead: false).ToResponse(Url))!; + Assert.That(exception.Message, Does.Contain("without answering")); + } + + [Test] + public void PartialHead_Throws() + { + ResponseReader reader = Feed("HTTP/1.1 200 OK\r\nContent-"); + + HttpException exception = Assert.Throws(() => reader.ToResponse(Url))!; + Assert.That(exception.Message, Does.Contain("middle of the response head")); + } + + private static ResponseReader Feed(string response) + { + ResponseReader reader = new(isHead: false); + byte[] bytes = Encoding.ASCII.GetBytes(response); + reader.Append(bytes, bytes.Length); + return reader; + } + + /// The reader after the response arrived in two parts, for every place it can be split, and then a byte at a time. + private static System.Collections.Generic.IEnumerable FeedInEverySplit(string response) + { + byte[] bytes = Encoding.ASCII.GetBytes(response); + for (int split = 0; split <= bytes.Length; split++) + { + ResponseReader reader = new(isHead: false); + reader.Append(bytes[..split], split); + reader.Append(bytes[split..], bytes.Length - split); + yield return reader; + } + + ResponseReader slow = new(isHead: false); + foreach (byte b in bytes) + { + slow.Append([b], 1); + } + + yield return slow; + } +} diff --git a/tests/Cosmos.Network.Http.Tests/TestServer.cs b/tests/Cosmos.Network.Http.Tests/TestServer.cs new file mode 100644 index 0000000..e3111ab --- /dev/null +++ b/tests/Cosmos.Network.Http.Tests/TestServer.cs @@ -0,0 +1,215 @@ +// This code is licensed under the BSD 3-Clause license (see LICENSE.txt for details) + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Sockets; +using System.Text; +using System.Threading; + +namespace Cosmos.Network.Http.Tests; + +/// +/// A loopback server that answers each connection with the next handler a +/// test queued, so a test writes the response bytes exactly as a server +/// would send them. +/// +internal sealed class TestServer : IDisposable +{ + private readonly TcpListener _listener; + private readonly Thread _thread; + private readonly ConcurrentQueue> _handlers = new(); + private volatile bool _disposed; + + public TestServer() + { + _listener = new TcpListener(IPAddress.Loopback, 0); + _listener.Start(); + _thread = new Thread(Serve) { IsBackground = true }; + _thread.Start(); + } + + public int Port => ((IPEndPoint)_listener.LocalEndpoint).Port; + + /// The requests received, in order. + public ConcurrentQueue Requests { get; } = new(); + + /// The first failure of a handler, rethrown by so the test sees it. + private Exception? _failure; + + public string Url(string target) => $"http://127.0.0.1:{Port}{target}"; + + /// Queues the handler for the next connection. + public TestServer Then(Action handler) + { + _handlers.Enqueue(handler); + return this; + } + + /// Queues a handler that reads the request and sends , then closes. + public TestServer Then(string response) => Then(connection => + { + connection.ReadRequest(); + connection.Write(response); + }); + + public void Dispose() + { + _disposed = true; + _listener.Stop(); + _thread.Join(5000); + if (_failure is not null) + { + throw new InvalidOperationException("A test server handler failed.", _failure); + } + } + + private void Serve() + { + while (!_disposed) + { + TcpClient client; + try + { + client = _listener.AcceptTcpClient(); + } + catch (SocketException) + { + return; + } + + using (client) + { + if (!_handlers.TryDequeue(out Action? handler)) + { + continue; + } + + try + { + handler(new TestConnection(this, client)); + } + catch (Exception exception) when (exception is IOException or SocketException) + { + // The client went away, which some tests make it do. + } + catch (Exception exception) + { + _failure ??= exception; + } + } + } + } + + internal void Record(TestRequest request) => Requests.Enqueue(request); +} + +/// A request as the test server received it. +internal sealed record TestRequest(string Method, string Target, Dictionary Headers, byte[] Body, string Head); + +/// One connection to the test server. +internal sealed class TestConnection +{ + private readonly TestServer _server; + private readonly NetworkStream _stream; + private readonly MemoryStream _pending = new(); + + public TestConnection(TestServer server, TcpClient client) + { + _server = server; + _stream = client.GetStream(); + _stream.ReadTimeout = 10_000; + } + + /// Reads a request head and the body its Content-Length announces. + public TestRequest ReadRequest() + { + byte[] buffer = new byte[4096]; + int headEnd; + while ((headEnd = IndexOfHeadEnd(_pending.GetBuffer(), (int)_pending.Length)) < 0) + { + int read = _stream.Read(buffer, 0, buffer.Length); + if (read == 0) + { + throw new IOException("The client closed before sending a whole head."); + } + + _pending.Write(buffer, 0, read); + } + + string head = Encoding.UTF8.GetString(_pending.GetBuffer(), 0, headEnd); + string[] lines = head.Split("\r\n", StringSplitOptions.RemoveEmptyEntries); + string[] requestLine = lines[0].Split(' '); + Dictionary headers = new(StringComparer.OrdinalIgnoreCase); + for (int i = 1; i < lines.Length; i++) + { + int colon = lines[i].IndexOf(':'); + headers[lines[i].Substring(0, colon).Trim()] = lines[i].Substring(colon + 1).Trim(); + } + + int length = headers.TryGetValue("Content-Length", out string? value) ? int.Parse(value) : 0; + while (_pending.Length < headEnd + length) + { + int read = _stream.Read(buffer, 0, buffer.Length); + if (read == 0) + { + throw new IOException("The client closed before sending a whole body."); + } + + _pending.Write(buffer, 0, read); + } + + byte[] body = new byte[length]; + Buffer.BlockCopy(_pending.GetBuffer(), headEnd, body, 0, length); + + TestRequest request = new(requestLine[0], requestLine[1], headers, body, head); + _server.Record(request); + return request; + } + + public void Write(string text) => Write(Encoding.UTF8.GetBytes(text)); + + public void Write(byte[] data) + { + _stream.Write(data, 0, data.Length); + _stream.Flush(); + } + + /// Writes a byte at a time, so the client sees every split there is. + public void WriteSlowly(string text) + { + foreach (byte b in Encoding.UTF8.GetBytes(text)) + { + _stream.WriteByte(b); + _stream.Flush(); + Thread.Sleep(1); + } + } + + /// Keeps the connection open until the client closes it, as a server that ignores Connection: close would. + public void WaitForClientClose() + { + byte[] buffer = new byte[256]; + while (_stream.Read(buffer, 0, buffer.Length) > 0) + { + } + } + + /// Stays silent until the client gives up. + public void Hang() => WaitForClientClose(); + + private static int IndexOfHeadEnd(byte[] data, int length) + { + for (int i = 0; i + 3 < length; i++) + { + if (data[i] == '\r' && data[i + 1] == '\n' && data[i + 2] == '\r' && data[i + 3] == '\n') + { + return i + 4; + } + } + + return -1; + } +}