From 7d914968dc592b62615de3f7d5e861e190fb156d Mon Sep 17 00:00:00 2001 From: Juliusz Sosinowicz Date: Tue, 14 Jul 2026 10:08:29 +0000 Subject: [PATCH 1/2] Zero-copy encrypt of AEAD application data BuildMessage copied the application data into the output buffer and encrypted it in place. For AEAD suites the cipher input is exactly the plaintext, so Encrypt can read it straight from the caller's buffer and write the record to the output buffer, skipping the copy. - EncryptDo AES-GCM/CCM: input == out keeps the in-place record layout [explicit IV | plaintext | tag space]; input != out means input is the plaintext itself. ChaCha20-Poly1305 already reads only the plaintext from input. Other suites keep the copy: CBC/stream append MAC/pad to the cipher input, CID appends the real content type. - Disabled for WOLFSSL_ASYNC_CRYPT and WOLFSSL_THREADED_CRYPT, where encryption can run after BuildMessage returns, and for debug hooks that expect the in-place layout. Define WOLFSSL_BUILD_MSG_NO_ZERO_COPY to force the old behavior. Measured with examples/benchmark/dtls_bench -z (DTLS 1.2, 1300 byte records, aesni + intelasm): +2.9% AES-256-GCM, +1.0% ChaCha20-Poly1305. The copy was ~3% of send-path CPU in perf. Also fix two bugs found while benchmarking with a larger STATIC_BUFFER_LEN: - The STATIC_BUFFER_LEN bounds check had an unbalanced paren and compared enum constants in a preprocessor #if, where they evaluate to 0, so any user supplied value failed to compile. Keep the lower bound as a static assert. Drop the upper bound: values past the largest record are never used by the record layer but can be useful for memory layout reasons. - ShrinkInputBuffer could run in the middle of record processing via FreeHandshakeResources on the first DTLS app data record, moving the record data and invalidating the idx based bookkeeping. The stale curStartIdx made the more-messages-per-record check dispatch a spurious second message over the AEAD tag bytes, handing garbage app data to the caller and corrupting the buffer state for the next read. Only reachable when the remaining record data fits in the static buffer, which it never does with the default STATIC_BUFFER_LEN. Only shrink between records; the deferred shrink happens at the next wolfSSL_read. --- src/internal.c | 85 ++++++++++++++++++++++++++++++++++++++++++---- wolfssl/internal.h | 8 ++--- 2 files changed, 82 insertions(+), 11 deletions(-) diff --git a/src/internal.c b/src/internal.c index 0ea1b04f2b7..e5fe44f2a6e 100644 --- a/src/internal.c +++ b/src/internal.c @@ -11616,7 +11616,10 @@ void ShrinkInputBuffer(WOLFSSL* ssl, int forcedFree) int usedLength = (int)(ssl->buffers.inputBuffer.length - ssl->buffers.inputBuffer.idx); if (!forcedFree && (usedLength > STATIC_BUFFER_LEN || - ssl->buffers.clearOutputBuffer.length > 0)) + ssl->buffers.clearOutputBuffer.length > 0 || + /* Moving the data would invalidate the idx based bookkeeping + * of in-flight record processing. */ + ssl->options.processReply != doProcessInit)) return; WOLFSSL_MSG("Shrinking input buffer"); @@ -21167,6 +21170,29 @@ int writeAeadAuthData(WOLFSSL* ssl, word16 sz, byte type, return (int)idx; } +/* Encrypt application data records straight from the caller's buffer + * instead of copying the plaintext into the output buffer first. Only used + * with AEAD suites where the cipher input is exactly the plaintext. + * Excluded configs: + * - WOLFSSL_ASYNC_CRYPT and WOLFSSL_THREADED_CRYPT: encryption may run + * after BuildMessage returns, when the caller's buffer may be gone. + * - WOLFSSL_CIPHER_TEXT_CHECK, WOLFSSL_CHECK_MEM_ZERO, HAVE_FUZZER and + * CHACHA_AEAD_TEST: debug hooks that expect the record layout at the + * cipher input. + * Define WOLFSSL_BUILD_MSG_NO_ZERO_COPY to always copy. */ +#if !defined(WOLFSSL_BUILD_MSG_NO_ZERO_COPY) && \ + !defined(WOLFSSL_NO_TLS12) && !defined(WOLFSSL_ASYNC_CRYPT) && \ + !defined(WOLFSSL_THREADED_CRYPT) && \ + !defined(WOLFSSL_CIPHER_TEXT_CHECK) && \ + !defined(WOLFSSL_CHECK_MEM_ZERO) && !defined(HAVE_FUZZER) && \ + !defined(CHACHA_AEAD_TEST) + #define WOLFSSL_BUILD_MSG_ZERO_COPY +#endif + +/* out always holds the record layout: [explicit IV | ciphertext | tag]. + * In-place (input == out): input holds the same layout with the plaintext + * where the ciphertext goes. Zero-copy (input != out): input points at the + * plaintext itself; only the plaintext length is read from it. */ static WC_INLINE int EncryptDo(WOLFSSL* ssl, byte* out, const byte* input, word16 sz, int asyncOkay, byte type) { @@ -21236,6 +21262,8 @@ static WC_INLINE int EncryptDo(WOLFSSL* ssl, byte* out, const byte* input, { AES_AUTH_ENCRYPT_FUNC aes_auth_fn; int additionalSz; + const byte* plain = (input == out) ? input + AESGCM_EXP_IV_SZ + : input; #ifdef WOLFSSL_ASYNC_CRYPT /* initialize event */ @@ -21276,7 +21304,7 @@ static WC_INLINE int EncryptDo(WOLFSSL* ssl, byte* out, const byte* input, ret = NOT_COMPILED_IN; if (ssl->ctx && ssl->ctx->PerformTlsRecordProcessingCb) { ret = ssl->ctx->PerformTlsRecordProcessingCb(ssl, 1, - out + AESGCM_EXP_IV_SZ, input + AESGCM_EXP_IV_SZ, + out + AESGCM_EXP_IV_SZ, plain, sz - AESGCM_EXP_IV_SZ - ssl->specs.aead_mac_size, ssl->encrypt.nonce, AESGCM_NONCE_SZ, out + sz - ssl->specs.aead_mac_size, @@ -21288,7 +21316,7 @@ static WC_INLINE int EncryptDo(WOLFSSL* ssl, byte* out, const byte* input, #endif /* HAVE_PK_CALLBACKS */ { ret = aes_auth_fn(ssl->encrypt.aes, - out + AESGCM_EXP_IV_SZ, input + AESGCM_EXP_IV_SZ, + out + AESGCM_EXP_IV_SZ, plain, sz - (word16)(AESGCM_EXP_IV_SZ) - ssl->specs.aead_mac_size, ssl->encrypt.nonce, AESGCM_NONCE_SZ, out + sz - ssl->specs.aead_mac_size, @@ -21365,6 +21393,7 @@ static WC_INLINE int EncryptDo(WOLFSSL* ssl, byte* out, const byte* input, #if defined(HAVE_CHACHA) && defined(HAVE_POLY1305) && \ !defined(NO_CHAPOL_AEAD) case wolfssl_chacha: + /* No explicit IV: the plaintext is at input in both layouts. */ ret = ChachaAEADEncrypt(ssl, out, input, sz, type); break; #endif @@ -25366,6 +25395,33 @@ void FreeBuildMsgArgs(WOLFSSL* ssl, BuildMsgArgs* args) #endif /* Build SSL Message, encrypted */ +#ifdef WOLFSSL_BUILD_MSG_ZERO_COPY +/* Can the plaintext be encrypted straight from the caller's buffer? Only + * for AEAD suites whose cipher input is exactly the plaintext: no trailing + * MAC/pad (CBC/stream) and no appended content type (CID). */ +static int BuildMsgZeroCopyOk(WOLFSSL* ssl, int type) +{ + if (type != application_data) + return 0; + if (ssl->specs.cipher_type != aead) + return 0; + if (ssl->specs.bulk_cipher_algorithm != wolfssl_aes_gcm && + ssl->specs.bulk_cipher_algorithm != wolfssl_aes_ccm && + ssl->specs.bulk_cipher_algorithm != wolfssl_chacha) + return 0; +#if defined(WOLFSSL_DTLS) && defined(WOLFSSL_DTLS_CID) + if (ssl->options.dtls && DtlsGetCidTxSize(ssl) > 0) + return 0; +#endif +#ifdef ATOMIC_USER + /* MacEncryptCb reads the plaintext from the output buffer. */ + if (ssl->ctx->MacEncryptCb != NULL) + return 0; +#endif + return 1; +} +#endif /* WOLFSSL_BUILD_MSG_ZERO_COPY */ + int BuildMessage(WOLFSSL* ssl, byte* output, int outSz, const byte* input, int inSz, int type, int hashOutput, int sizeOnly, int asyncOkay, int epochOrder) @@ -25375,6 +25431,9 @@ int BuildMessage(WOLFSSL* ssl, byte* output, int outSz, const byte* input, BuildMsgArgs* args; BuildMsgArgs lcl_args; #endif +#ifdef WOLFSSL_BUILD_MSG_ZERO_COPY + const byte* encInput = NULL; +#endif #ifdef WOLFSSL_DTLS_CID byte cidSz = 0; @@ -25654,7 +25713,15 @@ int BuildMessage(WOLFSSL* ssl, byte* output, int outSz, const byte* input, min(args->ivSz, MAX_IV_SZ)); args->idx += min(args->ivSz, MAX_IV_SZ); } - XMEMCPY(output + args->idx, input, (size_t)(inSz)); +#ifdef WOLFSSL_BUILD_MSG_ZERO_COPY + if (BuildMsgZeroCopyOk(ssl, type)) { + encInput = input; + } + else +#endif + { + XMEMCPY(output + args->idx, input, (size_t)(inSz)); + } args->idx += (word32)inSz; #if defined(WOLFSSL_DTLS) && defined(WOLFSSL_DTLS_CID) if (ssl->options.dtls && DtlsGetCidTxSize(ssl) > 0) { @@ -25833,9 +25900,13 @@ int BuildMessage(WOLFSSL* ssl, byte* output, int outSz, const byte* input, else #endif { - ret = Encrypt(ssl, output + args->headerSz, - output + args->headerSz, args->size, asyncOkay, - args->type); + const byte* encIn = output + args->headerSz; +#ifdef WOLFSSL_BUILD_MSG_ZERO_COPY + if (encInput != NULL) + encIn = encInput; +#endif + ret = Encrypt(ssl, output + args->headerSz, encIn, + args->size, asyncOkay, args->type); } #if defined(HAVE_SECURE_RENEGOTIATION) && defined(WOLFSSL_DTLS) /* Restore sequence numbers */ diff --git a/wolfssl/internal.h b/wolfssl/internal.h index d54496e0f9c..80e0f7aaa1a 100644 --- a/wolfssl/internal.h +++ b/wolfssl/internal.h @@ -2363,10 +2363,6 @@ enum { */ #ifdef STATIC_BUFFER_LEN /* user supplied option */ - #if STATIC_BUFFER_LEN < 5 || STATIC_BUFFER_LEN > (RECORD_HEADER_SZ + \ - RECORD_SIZE + COMP_EXTRA + MTU_EXTRA + MAX_MSG_EXTRA)) - #error Invalid static buffer length - #endif #elif defined(LARGE_STATIC_BUFFERS) #define STATIC_BUFFER_LEN (RECORD_HEADER_SZ + RECORD_SIZE + COMP_EXTRA + \ MTU_EXTRA + MAX_MSG_EXTRA) @@ -2375,6 +2371,10 @@ enum { #define STATIC_BUFFER_LEN RECORD_HEADER_SZ #endif +/* RECORD_HEADER_SZ is an enum constant, so the preprocessor can't check + * this bound. */ +wc_static_assert(STATIC_BUFFER_LEN >= RECORD_HEADER_SZ); + typedef struct { ALIGN16 byte staticBuffer[STATIC_BUFFER_LEN]; byte* buffer; /* place holder for static or dynamic buffer */ From 1cd4f18a0babb00970ac007d31f892cbe57c4243 Mon Sep 17 00:00:00 2001 From: Juliusz Sosinowicz Date: Tue, 14 Jul 2026 13:25:26 +0000 Subject: [PATCH 2/2] Add WOLFSSL_BUILD_MSG_NO_ZERO_COPY to known macros check-source-text requires macros that are used but never defined in the tree to be listed in .wolfssl_known_macro_extras. --- .wolfssl_known_macro_extras | 1 + 1 file changed, 1 insertion(+) diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras index 707c68cb51d..3714f6899f4 100644 --- a/.wolfssl_known_macro_extras +++ b/.wolfssl_known_macro_extras @@ -798,6 +798,7 @@ WOLFSSL_ATMEL_TIME WOLFSSL_BEFORE_DATE_CLOCK_SKEW WOLFSSL_BIGINT_TYPES WOLFSSL_BIO_NO_FLOW_STATS +WOLFSSL_BUILD_MSG_NO_ZERO_COPY WOLFSSL_BYTESWAP32_ASM WOLFSSL_CAAM_BLACK_KEY_AESCCM WOLFSSL_CAAM_BLACK_KEY_SM