From 03c29f67a968d2d1b021e074bba2f7662f386c80 Mon Sep 17 00:00:00 2001 From: adamw Date: Mon, 25 May 2026 08:17:37 +0000 Subject: [PATCH 1/8] Implement Add safe `byteBufferToArray` helper in `sttp.client4.internal` --- .claude/orca-system-prompt.md | 24 +++++++++ .../scala/sttp/client4/internal/package.scala | 13 +++++ .../internal/ByteBufferToArrayTest.scala | 53 +++++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 .claude/orca-system-prompt.md create mode 100644 core/src/test/scala/sttp/client4/internal/ByteBufferToArrayTest.scala diff --git a/.claude/orca-system-prompt.md b/.claude/orca-system-prompt.md new file mode 100644 index 0000000000..d4a83b4cc2 --- /dev/null +++ b/.claude/orca-system-prompt.md @@ -0,0 +1,24 @@ +## Scope + +Correctness only — what the code does and how it fails. Other dimensions (style, +performance, tests, structure) belong to other reviewers. + +## Aspects + +- **Intent vs. behaviour**: trace the code; does it produce the + documented/intended result for typical inputs? +- **Edge cases**: empty collections, zero, negative, max/min, boundary indices, + unicode, missing/null, malformed input. Pick the ones that apply. +- **Failure modes**: every external call, parse, or shell-out has a sad path — + is it caught at the right boundary, logged with enough context, surfaced to + the caller, or deliberately ignored with a reason? +- **Error swallowing**: a `try/catch` that drops the exception or returns a + default silently is almost always wrong. Flag it. +- **Concurrent access**: if shared state crosses threads, are the invariants + still safe? +iddle of logic. + Suggest named constants. +- **Local consistency**: similar things named or structured differently across + the change. + +Don't manufacture problems — when the code reads well, report no issues. diff --git a/core/src/main/scala/sttp/client4/internal/package.scala b/core/src/main/scala/sttp/client4/internal/package.scala index b00751cc53..ee5f941ffc 100644 --- a/core/src/main/scala/sttp/client4/internal/package.scala +++ b/core/src/main/scala/sttp/client4/internal/package.scala @@ -40,6 +40,16 @@ package object internal { private[client4] def emptyInputStream(): InputStream = new ByteArrayInputStream(Array[Byte]()) + /** Returns the bytes between the buffer's current `position` and `limit` as a fresh `Array[Byte]`. Safe for direct, + * read-only, partially-consumed, and sliced buffers. The source buffer is not mutated (its `position` and `limit` + * are preserved), and the result never aliases the buffer's storage. + */ + private[client4] def byteBufferToArray(b: ByteBuffer): Array[Byte] = { + val out = new Array[Byte](b.remaining()) + b.duplicate().get(out) + out + } + private[client4] def enqueueBytes( queue: Queue[Array[Byte]], bytes: ByteBuffer @@ -81,6 +91,9 @@ package object internal { private[client4] val IOBufferSize = 1024 implicit class RichByteBuffer(byteBuffer: ByteBuffer) { + /** Reads the buffer's remaining bytes into a fresh array, advancing the buffer's `position` to `limit`. Use + * [[byteBufferToArray]] instead when the buffer's position should be preserved. + */ def safeRead(): Array[Byte] = { val array = new Array[Byte](byteBuffer.remaining()) byteBuffer.get(array) diff --git a/core/src/test/scala/sttp/client4/internal/ByteBufferToArrayTest.scala b/core/src/test/scala/sttp/client4/internal/ByteBufferToArrayTest.scala new file mode 100644 index 0000000000..df9747c2df --- /dev/null +++ b/core/src/test/scala/sttp/client4/internal/ByteBufferToArrayTest.scala @@ -0,0 +1,53 @@ +package sttp.client4.internal + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.ByteBuffer + +class ByteBufferToArrayTest extends AnyFlatSpec with Matchers { + + it should "extract bytes from a direct buffer" in { + val direct = ByteBuffer.allocateDirect(5) + direct.put("ABCDE".getBytes) + direct.flip() + byteBufferToArray(direct) shouldBe "ABCDE".getBytes + } + + it should "extract bytes from a read-only buffer" in { + val readOnly = ByteBuffer.wrap("ABCDE".getBytes).asReadOnlyBuffer() + byteBufferToArray(readOnly) shouldBe "ABCDE".getBytes + } + + it should "return only the remaining slice when position > 0 and limit < capacity" in { + val partial = ByteBuffer.wrap("ABCDE".getBytes) + partial.position(2) + partial.limit(4) + byteBufferToArray(partial) shouldBe "CD".getBytes + } + + it should "respect arrayOffset for sliced heap buffers" in { + val original = ByteBuffer.wrap("ABCDE".getBytes) + original.position(2) + val sliced = original.slice() + sliced.arrayOffset() should be > 0 + byteBufferToArray(sliced) shouldBe "CDE".getBytes + } + + it should "return a fresh array that does not alias the source buffer's storage" in { + val data = "ABCDE".getBytes + val full = ByteBuffer.wrap(data) + val out = byteBufferToArray(full) + out(0) = 'Z'.toByte + full.get(0) shouldBe 'A'.toByte + } + + it should "not mutate the source buffer's position or limit" in { + val partial = ByteBuffer.wrap("ABCDE".getBytes) + partial.position(1) + partial.limit(4) + val _ = byteBufferToArray(partial) + partial.position() shouldBe 1 + partial.limit() shouldBe 4 + } +} From a18176cbc7dd9f3e3334e6d926f9636f8dad9706 Mon Sep 17 00:00:00 2001 From: adamw Date: Mon, 25 May 2026 08:34:54 +0000 Subject: [PATCH 2/8] Implement Use helper in `DigestAuthenticator.calculateHa2` (auth-int) --- .claude/orca-system-prompt.md | 27 +++++++------- .../internal/DigestAuthenticator.scala | 2 +- .../internal/DigestAuthenticatorTest.scala | 35 +++++++++++++++++++ 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/.claude/orca-system-prompt.md b/.claude/orca-system-prompt.md index d4a83b4cc2..dfebb9883b 100644 --- a/.claude/orca-system-prompt.md +++ b/.claude/orca-system-prompt.md @@ -1,24 +1,23 @@ ## Scope -Correctness only — what the code does and how it fails. Other dimensions (style, -performance, tests, structure) belong to other reviewers. +Micro-level clarity only — names, comments, control flow, magic values. Other +dimensions (structure, correctness, performance, tests) belong to other +reviewers. Don't chase formatting the project's formatter handles. ## Aspects -- **Intent vs. behaviour**: trace the code; does it produce the - documented/intended result for typical inputs? -- **Edge cases**: empty collections, zero, negative, max/min, boundary indices, - unicode, missing/null, malformed input. Pick the ones that apply. -- **Failure modes**: every external call, parse, or shell-out has a sad path — - is it caught at the right boundary, logged with enough context, surfaced to - the caller, or deliberately ignored with a reason? -- **Error swallowing**: a `try/catch` that drops the exception or returns a - default silently is almost always wrong. Flag it. -- **Concurrent access**: if shared state crosses threads, are the invariants - still safe? -iddle of logic. +- **Names**: variables, functions, types — do they say what they are? Flag + ambiguous, misleading, or overly abstract names. Flag boolean parameters whose + meaning isn't obvious at the call site. +- **Comments**: explain *why*, not *what*. Flag comments that restate the code, + are out of date, or could be replaced by a better name. Flag absent comments + where non-obvious reasoning is needed. +- **Control flow**: deep nesting, long methods, dense conditionals. Suggest + early returns, named helpers, or pattern matching when they'd clarify. +- **Magic values**: unexplained literals/strings/numbers in the middle of logic. Suggest named constants. - **Local consistency**: similar things named or structured differently across the change. Don't manufacture problems — when the code reads well, report no issues. +t the 3–5 most valuable improvements when the change is large. diff --git a/core/src/main/scala/sttp/client4/internal/DigestAuthenticator.scala b/core/src/main/scala/sttp/client4/internal/DigestAuthenticator.scala index 96d19bb164..07540fd653 100644 --- a/core/src/main/scala/sttp/client4/internal/DigestAuthenticator.scala +++ b/core/src/main/scala/sttp/client4/internal/DigestAuthenticator.scala @@ -165,7 +165,7 @@ private[client4] class DigestAuthenticator private ( brb match { case StringBody(s, e, _) => s.getBytes(Charset.forName(e)) case ByteArrayBody(b, _) => b - case ByteBufferBody(b, _) => b.array() + case ByteBufferBody(b, _) => byteBufferToArray(b) case InputStreamBody(b, _) => toByteArray(b) case _: FileBody => throw new IllegalStateException("Qop auth-int cannot be used with a file body") } diff --git a/core/src/test/scala/sttp/client4/internal/DigestAuthenticatorTest.scala b/core/src/test/scala/sttp/client4/internal/DigestAuthenticatorTest.scala index aac521f212..33175f4553 100644 --- a/core/src/test/scala/sttp/client4/internal/DigestAuthenticatorTest.scala +++ b/core/src/test/scala/sttp/client4/internal/DigestAuthenticatorTest.scala @@ -5,6 +5,7 @@ import sttp.client4.internal.DigestAuthenticator.DigestAuthData import sttp.client4._ import sttp.model.{Header, HeaderNames, StatusCode} +import java.nio.ByteBuffer import scala.util.{Failure, Try} import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers @@ -155,6 +156,40 @@ class DigestAuthenticatorTest extends AnyFreeSpec with Matchers with OptionValue header.value.value should fullyMatch regex """Digest username="admin", realm="myrealm", uri="/", nonce="BBBBBB", qop=auth, response="[0-9a-f]+", cnonce="[0-9a-f]+", nc=000000\d\d, algorithm=MD5""" } + "auth-int" - { + val payload = "hello".getBytes("UTF-8") + val fixedCnonce = () => "0123456789abcdef" + val authIntChallenge = """Digest realm="myrealm", nonce="BBBBBB", algorithm=MD5, qop="auth-int"""" + + def authHeaderValueFor(setBody: Request[Either[String, String]] => Request[Either[String, String]]): String = { + val baseRequest = basicRequest.get(uri"http://google.com/").auth.digest("admin", "password") + val r = responseWithAuthenticateHeader(authIntChallenge) + DigestAuthenticator(DigestAuthData("admin", "password"), fixedCnonce) + .authenticate(setBody(baseRequest), r) + .value + .value + } + + "compute the digest for a direct ByteBufferBody (previously threw UnsupportedOperationException)" in { + val direct = ByteBuffer.allocateDirect(payload.length) + direct.put(payload) + direct.flip() + val viaDirectBuffer = authHeaderValueFor(_.body(direct)) + val viaArray = authHeaderValueFor(_.body(payload)) + viaDirectBuffer shouldBe viaArray + } + + "compute the digest over the partial buffer's remaining slice, not the full backing array" in { + val backing = ("XX".getBytes ++ payload ++ "YY".getBytes) + val partial = ByteBuffer.wrap(backing) + partial.position(2) + partial.limit(2 + payload.length) + val viaPartialBuffer = authHeaderValueFor(_.body(partial)) + val viaArray = authHeaderValueFor(_.body(payload)) + viaPartialBuffer shouldBe viaArray + } + } + "proxy" - { "should work" in { val request = basicRequest From 2e0cf9024d76e47751bdc167ed50824c18b84f0c Mon Sep 17 00:00:00 2001 From: adamw Date: Mon, 25 May 2026 08:43:49 +0000 Subject: [PATCH 3/8] Implement Use helper in `testing.forceBodyAsString` and `forceBodyAsByteArray` --- .claude/orca-system-prompt.md | 35 +++++++------- .../scala/sttp/client4/testing/package.scala | 6 +-- .../client4/testing/ForceBodyAsTest.scala | 46 +++++++++++++++++++ 3 files changed, 67 insertions(+), 20 deletions(-) create mode 100644 core/src/test/scala/sttp/client4/testing/ForceBodyAsTest.scala diff --git a/.claude/orca-system-prompt.md b/.claude/orca-system-prompt.md index dfebb9883b..0818d096bf 100644 --- a/.claude/orca-system-prompt.md +++ b/.claude/orca-system-prompt.md @@ -1,23 +1,24 @@ ## Scope -Micro-level clarity only — names, comments, control flow, magic values. Other -dimensions (structure, correctness, performance, tests) belong to other -reviewers. Don't chase formatting the project's formatter handles. +Correctness only — what the code does and how it fails. Other dimensions (style, +performance, tests, structure) belong to other reviewers. ## Aspects -- **Names**: variables, functions, types — do they say what they are? Flag - ambiguous, misleading, or overly abstract names. Flag boolean parameters whose - meaning isn't obvious at the call site. -- **Comments**: explain *why*, not *what*. Flag comments that restate the code, - are out of date, or could be replaced by a better name. Flag absent comments - where non-obvious reasoning is needed. -- **Control flow**: deep nesting, long methods, dense conditionals. Suggest - early returns, named helpers, or pattern matching when they'd clarify. -- **Magic values**: unexplained literals/strings/numbers in the middle of logic. - Suggest named constants. -- **Local consistency**: similar things named or structured differently across - the change. +- **Intent vs. behaviour**: trace the code; does it produce the + documented/intended result for typical inputs? +- **Edge cases**: empty collections, zero, negative, max/min, boundary indices, + unicode, missing/null, malformed input. Pick the ones that apply. +- **Failure modes**: every external call, parse, or shell-out has a sad path — + is it caught at the right boundary, logged with enough context, surfaced to + the caller, or deliberately ignored with a reason? +- **Error swallowing**: a `try/catch` that drops the exception or returns a + default silently is almost always wrong. Flag it. +- **Concurrent access**: if shared state crosses threads, are the invariants + still safe? +ouldn't depend on transport/IO concretely). +- **Symbol visibility**: top-level constructs and helpers should be + `private[xxx]` to the smallest enclosing scope that uses them. Flag + over-exposed symbols. -Don't manufacture problems — when the code reads well, report no issues. -t the 3–5 most valuable improvements when the change is large. +Cap at the 3–5 most valuable improvements when the change is large. diff --git a/core/src/main/scala/sttp/client4/testing/package.scala b/core/src/main/scala/sttp/client4/testing/package.scala index 08a6d2f9db..0595178b8e 100644 --- a/core/src/main/scala/sttp/client4/testing/package.scala +++ b/core/src/main/scala/sttp/client4/testing/package.scala @@ -1,6 +1,6 @@ package sttp.client4 -import sttp.client4.internal.toByteArray +import sttp.client4.internal.{byteBufferToArray, toByteArray} package object testing { implicit class RichTestingRequest[T, R](r: GenericRequest[T, R]) { @@ -13,7 +13,7 @@ package object testing { case NoBody => "" case StringBody(s, _, _) => s case ByteArrayBody(b, _) => new String(b) - case ByteBufferBody(b, _) => new String(b.array()) + case ByteBufferBody(b, _) => new String(byteBufferToArray(b)) case InputStreamBody(b, _) => new String(toByteArray(b)) case FileBody(f, _) => f.readAsString() case StreamBody(_) => @@ -30,7 +30,7 @@ package object testing { case NoBody => Array.emptyByteArray case StringBody(s, encoding, _) => s.getBytes(encoding) case ByteArrayBody(b, _) => b - case ByteBufferBody(b, _) => b.array() + case ByteBufferBody(b, _) => byteBufferToArray(b) case InputStreamBody(b, _) => toByteArray(b) case FileBody(f, _) => f.readAsByteArray() case StreamBody(_) => diff --git a/core/src/test/scala/sttp/client4/testing/ForceBodyAsTest.scala b/core/src/test/scala/sttp/client4/testing/ForceBodyAsTest.scala new file mode 100644 index 0000000000..8e1a1cde00 --- /dev/null +++ b/core/src/test/scala/sttp/client4/testing/ForceBodyAsTest.scala @@ -0,0 +1,46 @@ +package sttp.client4.testing + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import sttp.client4._ + +import java.nio.ByteBuffer + +class ForceBodyAsTest extends AnyFlatSpec with Matchers { + + private val payload = "hello".getBytes("UTF-8") + + it should "forceBodyAsString: extract a direct ByteBuffer body" in { + val direct = ByteBuffer.allocateDirect(payload.length) + direct.put(payload) + direct.flip() + val request = basicRequest.post(uri"http://example.com/").body(direct) + request.forceBodyAsString shouldBe "hello" + } + + it should "forceBodyAsByteArray: extract a direct ByteBuffer body" in { + val direct = ByteBuffer.allocateDirect(payload.length) + direct.put(payload) + direct.flip() + val request = basicRequest.post(uri"http://example.com/").body(direct) + request.forceBodyAsByteArray shouldBe payload + } + + it should "forceBodyAsString: return only the remaining slice of a partial heap buffer" in { + val backing = "XX".getBytes ++ payload ++ "YY".getBytes + val partial = ByteBuffer.wrap(backing) + partial.position(2) + partial.limit(2 + payload.length) + val request = basicRequest.post(uri"http://example.com/").body(partial) + request.forceBodyAsString shouldBe "hello" + } + + it should "forceBodyAsByteArray: return only the remaining slice of a partial heap buffer" in { + val backing = "XX".getBytes ++ payload ++ "YY".getBytes + val partial = ByteBuffer.wrap(backing) + partial.position(2) + partial.limit(2 + payload.length) + val request = basicRequest.post(uri"http://example.com/").body(partial) + request.forceBodyAsByteArray shouldBe payload + } +} From 883418cef3a9e4495f82178c4762f4ae5196b93f Mon Sep 17 00:00:00 2001 From: adamw Date: Mon, 25 May 2026 09:32:58 +0000 Subject: [PATCH 4/8] Implement Use helper in `internal.enqueueBytes` --- .claude/orca-system-prompt.md | 27 +++++++++---------- .../scala/sttp/client4/internal/package.scala | 3 ++- .../client4/internal/EnqueueBytesTest.scala | 26 ++++++++++++++++++ 3 files changed, 41 insertions(+), 15 deletions(-) create mode 100644 core/src/test/scala/sttp/client4/internal/EnqueueBytesTest.scala diff --git a/.claude/orca-system-prompt.md b/.claude/orca-system-prompt.md index 0818d096bf..21036af6cb 100644 --- a/.claude/orca-system-prompt.md +++ b/.claude/orca-system-prompt.md @@ -1,22 +1,21 @@ ## Scope -Correctness only — what the code does and how it fails. Other dimensions (style, -performance, tests, structure) belong to other reviewers. +Structure only — how the pieces fit together. Other dimensions (correctness, +naming, performance, tests) belong to other reviewers. ## Aspects -- **Intent vs. behaviour**: trace the code; does it produce the - documented/intended result for typical inputs? -- **Edge cases**: empty collections, zero, negative, max/min, boundary indices, - unicode, missing/null, malformed input. Pick the ones that apply. -- **Failure modes**: every external call, parse, or shell-out has a sad path — - is it caught at the right boundary, logged with enough context, surfaced to - the caller, or deliberately ignored with a reason? -- **Error swallowing**: a `try/catch` that drops the exception or returns a - default silently is almost always wrong. Flag it. -- **Concurrent access**: if shared state crosses threads, are the invariants - still safe? -ouldn't depend on transport/IO concretely). +- **Duplication**: semantic duplication (same logic, different syntax) repeated + 2+ times. Suggest a name and a home for the extracted unit. +- **Abstraction quality**: extractions that genuinely simplify vs. premature + ones that just add indirection. An extracted unit should have a coherent + single responsibility. Don't propose abstractions for one-off code. +- **Module boundaries**: do new symbols sit in the right package? Are + types/helpers grouped by feature, not by category (`utils`, `helpers`, + `common` are smells)? +- **Dependency direction**: no circular package dependencies; downstream modules + don't reach into internals of upstream ones; layering is respected (domain + logic shouldn't depend on transport/IO concretely). - **Symbol visibility**: top-level constructs and helpers should be `private[xxx]` to the smallest enclosing scope that uses them. Flag over-exposed symbols. diff --git a/core/src/main/scala/sttp/client4/internal/package.scala b/core/src/main/scala/sttp/client4/internal/package.scala index ee5f941ffc..a67baca908 100644 --- a/core/src/main/scala/sttp/client4/internal/package.scala +++ b/core/src/main/scala/sttp/client4/internal/package.scala @@ -53,10 +53,11 @@ package object internal { private[client4] def enqueueBytes( queue: Queue[Array[Byte]], bytes: ByteBuffer - ): Queue[Array[Byte]] = queue.enqueue(bytes.array()) + ): Queue[Array[Byte]] = queue.enqueue(byteBufferToArray(bytes)) private[client4] def concatBytes(queue: Queue[Array[Byte]]): Array[Byte] = { val size = queue.map(_.length).sum + // Heap buffer of exact size, fully filled and never read partially: array() is safe here. val bytes = ByteBuffer.allocate(size) queue.foreach(bytes.put) bytes.array() diff --git a/core/src/test/scala/sttp/client4/internal/EnqueueBytesTest.scala b/core/src/test/scala/sttp/client4/internal/EnqueueBytesTest.scala new file mode 100644 index 0000000000..506a6574b2 --- /dev/null +++ b/core/src/test/scala/sttp/client4/internal/EnqueueBytesTest.scala @@ -0,0 +1,26 @@ +package sttp.client4.internal + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.ByteBuffer +import scala.collection.immutable.Queue + +class EnqueueBytesTest extends AnyFlatSpec with Matchers { + + it should "append the bytes of a direct buffer" in { + val direct = ByteBuffer.allocateDirect(5) + direct.put("hello".getBytes) + direct.flip() + val result = enqueueBytes(Queue.empty[Array[Byte]], direct) + result.head shouldBe "hello".getBytes + } + + it should "append only the remaining slice of a partial heap buffer" in { + val partial = ByteBuffer.wrap("ABCDE".getBytes) + partial.position(2) + partial.limit(4) + val result = enqueueBytes(Queue.empty[Array[Byte]], partial) + result.head shouldBe "CD".getBytes + } +} From fc54140c69c7032b1629dabfdb01ee1690fce9dd Mon Sep 17 00:00:00 2001 From: adamw Date: Mon, 25 May 2026 10:45:57 +0000 Subject: [PATCH 5/8] Route Compressor.byteBufferToArray through the shared internal helper The old Compressor.byteBufferToArray had two bugs: the hasArray branch ignored position/limit, and the else branch mutated the buffer's position. Replace it with the safe internal.byteBufferToArray helper. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../main/scala/sttp/client4/compression/Compressor.scala | 9 --------- .../sttp/client4/compression/defaultCompressors.scala | 1 + 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/core/src/main/scala/sttp/client4/compression/Compressor.scala b/core/src/main/scala/sttp/client4/compression/Compressor.scala index a5e5b59c87..6fcc9991dd 100644 --- a/core/src/main/scala/sttp/client4/compression/Compressor.scala +++ b/core/src/main/scala/sttp/client4/compression/Compressor.scala @@ -1,7 +1,6 @@ package sttp.client4.compression import sttp.client4._ -import java.nio.ByteBuffer /** Allows compressing bodies, using the supported encoding. The compressed bodies might use `R` capabilities (e.g. * streaming). @@ -53,12 +52,4 @@ object Compressor extends CompressorExtensions { private[compression] def streamsNotSupported: Nothing = throw new IllegalArgumentException("Streams are not supported") - private[compression] def byteBufferToArray(inputBuffer: ByteBuffer): Array[Byte] = - if (inputBuffer.hasArray()) { - inputBuffer.array() - } else { - val inputBytes = new Array[Byte](inputBuffer.remaining()) - inputBuffer.get(inputBytes) - inputBytes - } } diff --git a/core/src/main/scalajvm/sttp/client4/compression/defaultCompressors.scala b/core/src/main/scalajvm/sttp/client4/compression/defaultCompressors.scala index c9b9d351fa..371dd0ff28 100644 --- a/core/src/main/scalajvm/sttp/client4/compression/defaultCompressors.scala +++ b/core/src/main/scalajvm/sttp/client4/compression/defaultCompressors.scala @@ -4,6 +4,7 @@ import sttp.client4._ import sttp.model.Encodings import Compressor._ +import sttp.client4.internal.byteBufferToArray import java.util.zip.Deflater import java.util.zip.DeflaterInputStream import java.io.ByteArrayOutputStream From 795a4a417b40a643b4c9067745d834f88419a6fe Mon Sep 17 00:00:00 2001 From: adamw Date: Mon, 25 May 2026 11:09:33 +0000 Subject: [PATCH 6/8] Add fast path for fresh full heap buffers in byteBufferToArray When the buffer has a backing array with offset 0, position 0, and remaining == array length, return the backing array directly (no copy). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../scala/sttp/client4/internal/package.scala | 19 +++++++++++-------- .../internal/ByteBufferToArrayTest.scala | 12 +++++++++--- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/core/src/main/scala/sttp/client4/internal/package.scala b/core/src/main/scala/sttp/client4/internal/package.scala index a67baca908..3f16ac23a2 100644 --- a/core/src/main/scala/sttp/client4/internal/package.scala +++ b/core/src/main/scala/sttp/client4/internal/package.scala @@ -40,15 +40,18 @@ package object internal { private[client4] def emptyInputStream(): InputStream = new ByteArrayInputStream(Array[Byte]()) - /** Returns the bytes between the buffer's current `position` and `limit` as a fresh `Array[Byte]`. Safe for direct, - * read-only, partially-consumed, and sliced buffers. The source buffer is not mutated (its `position` and `limit` - * are preserved), and the result never aliases the buffer's storage. + /** Returns the bytes between the buffer's current `position` and `limit` as an `Array[Byte]`. Safe for direct, + * read-only, partially-consumed, and sliced buffers. The source buffer's `position` and `limit` are preserved. For + * a fresh, full heap buffer the backing array is returned directly (no copy); otherwise a new array is allocated. */ - private[client4] def byteBufferToArray(b: ByteBuffer): Array[Byte] = { - val out = new Array[Byte](b.remaining()) - b.duplicate().get(out) - out - } + private[client4] def byteBufferToArray(b: ByteBuffer): Array[Byte] = + if (b.hasArray && b.arrayOffset() == 0 && b.position() == 0 && b.remaining() == b.array().length) + b.array() + else { + val out = new Array[Byte](b.remaining()) + b.duplicate().get(out) + out + } private[client4] def enqueueBytes( queue: Queue[Array[Byte]], diff --git a/core/src/test/scala/sttp/client4/internal/ByteBufferToArrayTest.scala b/core/src/test/scala/sttp/client4/internal/ByteBufferToArrayTest.scala index df9747c2df..43b7d9f0ca 100644 --- a/core/src/test/scala/sttp/client4/internal/ByteBufferToArrayTest.scala +++ b/core/src/test/scala/sttp/client4/internal/ByteBufferToArrayTest.scala @@ -34,12 +34,18 @@ class ByteBufferToArrayTest extends AnyFlatSpec with Matchers { byteBufferToArray(sliced) shouldBe "CDE".getBytes } - it should "return a fresh array that does not alias the source buffer's storage" in { + it should "return the backing array directly for a fresh full heap buffer" in { val data = "ABCDE".getBytes val full = ByteBuffer.wrap(data) - val out = byteBufferToArray(full) + byteBufferToArray(full) should be theSameInstanceAs data + } + + it should "return a fresh array that does not alias storage for a partial buffer" in { + val partial = ByteBuffer.wrap("ABCDE".getBytes) + partial.position(1) + val out = byteBufferToArray(partial) out(0) = 'Z'.toByte - full.get(0) shouldBe 'A'.toByte + partial.get(1) shouldBe 'B'.toByte } it should "not mutate the source buffer's position or limit" in { From 7fc2f74e0e06d57fe8ce60634966c0f9ca531587 Mon Sep 17 00:00:00 2001 From: adamw Date: Mon, 25 May 2026 13:38:31 +0000 Subject: [PATCH 7/8] Use shared byteBufferToArray in remaining call sites Replace unsafe ByteBuffer.array() calls in BodyToHttpClient, MultipartBodyBuilder, FinagleBackend, and AbstractFetchBackend with the shared internal.byteBufferToArray helper. Remove the now-unused ByteBufferBackedInputStream class. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../client4/fetch/AbstractFetchBackend.scala | 7 ----- .../httpclient/BodyToHttpClient.scala | 4 +-- .../httpclient/MultipartBodyBuilder.scala | 28 ++----------------- .../sttp/client4/finagle/FinagleBackend.scala | 4 +-- 4 files changed, 7 insertions(+), 36 deletions(-) diff --git a/core/src/main/scalajs/sttp/client4/fetch/AbstractFetchBackend.scala b/core/src/main/scalajs/sttp/client4/fetch/AbstractFetchBackend.scala index b4fe9c2465..37720a111b 100644 --- a/core/src/main/scalajs/sttp/client4/fetch/AbstractFetchBackend.scala +++ b/core/src/main/scalajs/sttp/client4/fetch/AbstractFetchBackend.scala @@ -244,13 +244,6 @@ abstract class AbstractFetchBackend[F[_], S <: Streams[S]]( f.toDomFile } - // https://stackoverflow.com/questions/679298/gets-byte-array-from-a-bytebuffer-in-java - private def byteBufferToArray(bb: ByteBuffer): Array[Byte] = { - val b = new Array[Byte](bb.remaining()) - bb.get(b) - b - } - private def sendWebSocket[T](request: GenericRequest[T, R]): F[Response[T]] = { val queue = new JSSimpleQueue[F, WebSocketEvent] val ws = new JSWebSocket(request.uri.toString) diff --git a/core/src/main/scalajvm/sttp/client4/internal/httpclient/BodyToHttpClient.scala b/core/src/main/scalajvm/sttp/client4/internal/httpclient/BodyToHttpClient.scala index 69ea75d57b..c4405f8e12 100644 --- a/core/src/main/scalajvm/sttp/client4/internal/httpclient/BodyToHttpClient.scala +++ b/core/src/main/scalajvm/sttp/client4/internal/httpclient/BodyToHttpClient.scala @@ -5,6 +5,7 @@ import sttp.client4._ import sttp.client4.compression.Compressor import sttp.client4.httpclient.BodyProgressCallback import sttp.client4.internal.SttpToJavaConverters.toJavaSupplier +import sttp.client4.internal.byteBufferToArray import sttp.model.HeaderNames import sttp.monad.MonadError import sttp.monad.syntax._ @@ -31,8 +32,7 @@ private[client4] trait BodyToHttpClient[F[_], S, R] { case StringBody(b, _, _) => BodyPublishers.ofString(b).unit case ByteArrayBody(b, _) => BodyPublishers.ofByteArray(b).unit case ByteBufferBody(b, _) => - if (b.hasArray) BodyPublishers.ofByteArray(b.array(), 0, b.limit()).unit - else { val a = new Array[Byte](b.remaining()); b.get(a); BodyPublishers.ofByteArray(a).unit } + BodyPublishers.ofByteArray(byteBufferToArray(b)).unit case InputStreamBody(b, _) => BodyPublishers.ofInputStream(toJavaSupplier(() => b)).unit case FileBody(f, _) => BodyPublishers.ofFile(f.toFile.toPath).unit case StreamBody(s) => streamToPublisher(s.asInstanceOf[streams.BinaryStream]) diff --git a/core/src/main/scalajvm/sttp/client4/internal/httpclient/MultipartBodyBuilder.scala b/core/src/main/scalajvm/sttp/client4/internal/httpclient/MultipartBodyBuilder.scala index b35540dae8..01129ae6be 100644 --- a/core/src/main/scalajvm/sttp/client4/internal/httpclient/MultipartBodyBuilder.scala +++ b/core/src/main/scalajvm/sttp/client4/internal/httpclient/MultipartBodyBuilder.scala @@ -5,6 +5,7 @@ import sttp.client4.internal.httpclient import sttp.model.Part import sttp.model.Header import sttp.model.HeaderNames +import sttp.client4.internal.byteBufferToArray import sttp.client4.internal.throwNestedMultipartNotAllowed import sttp.client4.internal.Utf8 import sttp.monad.MonadError @@ -14,7 +15,6 @@ import java.net.http.HttpRequest import java.util.function.Supplier import java.io.File import java.io.InputStream -import java.nio.Buffer import java.nio.ByteBuffer import java.io.ByteArrayInputStream import java.util.UUID @@ -44,10 +44,7 @@ trait NonStreamMultipartBodyBuilder[BinaryStream, F[_]] extends MultipartBodyBui case ByteArrayBody(b, _) => multipartBuilder.addPart(p.name, supplier(new ByteArrayInputStream(b)), partHeaders) case ByteBufferBody(b, _) => - if ((b: Buffer).isReadOnly) - multipartBuilder.addPart(p.name, supplier(new ByteBufferBackedInputStream(b)), partHeaders) - else - multipartBuilder.addPart(p.name, supplier(new ByteArrayInputStream(b.array())), partHeaders) + multipartBuilder.addPart(p.name, supplier(new ByteArrayInputStream(byteBufferToArray(b))), partHeaders) case InputStreamBody(b, _) => multipartBuilder.addPart(p.name, supplier(b), partHeaders) case StreamBody(_) => throw new IllegalArgumentException("Multipart streaming bodies are not supported with this backend") @@ -96,11 +93,7 @@ trait StreamMultipartBodyBuilder[BinaryStream, F[_]] extends MultipartBodyBuilde case ByteArrayBody(b, _) => concatBytesToStream(accumulatedStream, encodeBytes(b, partHeaders, boundary)) case ByteBufferBody(b, _) => - if ((b: Buffer).isReadOnly) { - val buffer = new ByteBufferBackedInputStream(b) - concatStreams(accumulatedStream, encodeStream(inputStreamToStream(buffer), partHeaders, boundary)) - } else - concatBytesToStream(accumulatedStream, encodeBytes(b.array(), partHeaders, boundary)) + concatBytesToStream(accumulatedStream, encodeBytes(byteBufferToArray(b), partHeaders, boundary)) case InputStreamBody(b, _) => concatStreams(accumulatedStream, encodeStream(inputStreamToStream(b), partHeaders, boundary)) case StreamBody(s) => @@ -143,18 +136,3 @@ trait StreamMultipartBodyBuilder[BinaryStream, F[_]] extends MultipartBodyBuilde lastPart.getBytes(StandardCharsets.UTF_8) } } - -// https://stackoverflow.com/a/6603018/362531 -private[httpclient] class ByteBufferBackedInputStream(buf: ByteBuffer) extends InputStream { - override def read: Int = { - if (!buf.hasRemaining) return -1 - buf.get & 0xff - } - - override def read(bytes: Array[Byte], off: Int, len: Int): Int = { - if (!buf.hasRemaining) return -1 - val len2 = Math.min(len, buf.remaining) - buf.get(bytes, off, len2) - len2 - } -} diff --git a/finagle-backend/src/main/scala/sttp/client4/finagle/FinagleBackend.scala b/finagle-backend/src/main/scala/sttp/client4/finagle/FinagleBackend.scala index 8a031ab1f6..0fb54fad6e 100644 --- a/finagle-backend/src/main/scala/sttp/client4/finagle/FinagleBackend.scala +++ b/finagle-backend/src/main/scala/sttp/client4/finagle/FinagleBackend.scala @@ -15,7 +15,7 @@ import com.twitter.io.Buf.{ByteArray, ByteBuffer} import com.twitter.util import com.twitter.util.{Duration, Future => TFuture} import sttp.capabilities.Effect -import sttp.client4.internal.{BodyFromResponseAs, FileHelpers, SttpFile, Utf8} +import sttp.client4.internal.{BodyFromResponseAs, FileHelpers, SttpFile, Utf8, byteBufferToArray} import sttp.client4.testing.BackendStub import sttp.client4.ws.{GotAWebSocketException, NotAWebSocketException} import sttp.client4.{wrappers, _} @@ -126,7 +126,7 @@ class FinagleBackend(client: Option[Client] = None) extends Backend[TFuture] { case StringBody(s, e, _) if e.equalsIgnoreCase(Utf8) => s case StringBody(s, e, _) => Source.fromBytes(s.getBytes(e)).mkString case ByteArrayBody(b, _) => Source.fromBytes(b).mkString - case ByteBufferBody(b, _) => Source.fromBytes(b.array()).mkString + case ByteBufferBody(b, _) => Source.fromBytes(byteBufferToArray(b)).mkString case InputStreamBody(is, _) => Source.fromInputStream(is).mkString case FileBody(f, _) => Source.fromFile(f.toFile).mkString case StreamBody(_) => throw new IllegalArgumentException("Streaming is not supported") From 6c9c02f8bad84e62aadfe14d73c346853b4569aa Mon Sep 17 00:00:00 2001 From: adamw Date: Mon, 25 May 2026 15:29:21 +0000 Subject: [PATCH 8/8] Stream ByteBufferBody in StreamMultipartBodyBuilder instead of materializing Use ByteBufferBackedInputStream(b.duplicate()) to stream ByteBuffer content lazily, avoiding eager materialization of potentially large buffers. The duplicate() preserves the original buffer's position. Also remove stale .claude/orca-system-prompt.md. Co-Authored-By: Claude Opus 4.7 (1M context) --- .claude/orca-system-prompt.md | 23 ------------------- .../httpclient/MultipartBodyBuilder.scala | 19 ++++++++++++++- 2 files changed, 18 insertions(+), 24 deletions(-) delete mode 100644 .claude/orca-system-prompt.md diff --git a/.claude/orca-system-prompt.md b/.claude/orca-system-prompt.md deleted file mode 100644 index 21036af6cb..0000000000 --- a/.claude/orca-system-prompt.md +++ /dev/null @@ -1,23 +0,0 @@ -## Scope - -Structure only — how the pieces fit together. Other dimensions (correctness, -naming, performance, tests) belong to other reviewers. - -## Aspects - -- **Duplication**: semantic duplication (same logic, different syntax) repeated - 2+ times. Suggest a name and a home for the extracted unit. -- **Abstraction quality**: extractions that genuinely simplify vs. premature - ones that just add indirection. An extracted unit should have a coherent - single responsibility. Don't propose abstractions for one-off code. -- **Module boundaries**: do new symbols sit in the right package? Are - types/helpers grouped by feature, not by category (`utils`, `helpers`, - `common` are smells)? -- **Dependency direction**: no circular package dependencies; downstream modules - don't reach into internals of upstream ones; layering is respected (domain - logic shouldn't depend on transport/IO concretely). -- **Symbol visibility**: top-level constructs and helpers should be - `private[xxx]` to the smallest enclosing scope that uses them. Flag - over-exposed symbols. - -Cap at the 3–5 most valuable improvements when the change is large. diff --git a/core/src/main/scalajvm/sttp/client4/internal/httpclient/MultipartBodyBuilder.scala b/core/src/main/scalajvm/sttp/client4/internal/httpclient/MultipartBodyBuilder.scala index 01129ae6be..8cdfdbf321 100644 --- a/core/src/main/scalajvm/sttp/client4/internal/httpclient/MultipartBodyBuilder.scala +++ b/core/src/main/scalajvm/sttp/client4/internal/httpclient/MultipartBodyBuilder.scala @@ -93,7 +93,10 @@ trait StreamMultipartBodyBuilder[BinaryStream, F[_]] extends MultipartBodyBuilde case ByteArrayBody(b, _) => concatBytesToStream(accumulatedStream, encodeBytes(b, partHeaders, boundary)) case ByteBufferBody(b, _) => - concatBytesToStream(accumulatedStream, encodeBytes(byteBufferToArray(b), partHeaders, boundary)) + concatStreams( + accumulatedStream, + encodeStream(inputStreamToStream(new ByteBufferBackedInputStream(b.duplicate())), partHeaders, boundary) + ) case InputStreamBody(b, _) => concatStreams(accumulatedStream, encodeStream(inputStreamToStream(b), partHeaders, boundary)) case StreamBody(s) => @@ -136,3 +139,17 @@ trait StreamMultipartBodyBuilder[BinaryStream, F[_]] extends MultipartBodyBuilde lastPart.getBytes(StandardCharsets.UTF_8) } } + +private[httpclient] class ByteBufferBackedInputStream(buf: ByteBuffer) extends InputStream { + override def read: Int = { + if (!buf.hasRemaining) return -1 + buf.get & 0xff + } + + override def read(bytes: Array[Byte], off: Int, len: Int): Int = { + if (!buf.hasRemaining) return -1 + val len2 = Math.min(len, buf.remaining) + buf.get(bytes, off, len2) + len2 + } +}