diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras index 707c68cb51..3714f6899f 100644 --- a/.wolfssl_known_macro_extras +++ b/.wolfssl_known_macro_extras @@ -798,6 +798,7 @@ WOLFSSL_ATMEL_TIME WOLFSSL_BEFORE_DATE_CLOCK_SKEW WOLFSSL_BIGINT_TYPES WOLFSSL_BIO_NO_FLOW_STATS +WOLFSSL_BUILD_MSG_NO_ZERO_COPY WOLFSSL_BYTESWAP32_ASM WOLFSSL_CAAM_BLACK_KEY_AESCCM WOLFSSL_CAAM_BLACK_KEY_SM diff --git a/src/internal.c b/src/internal.c index 0ea1b04f2b..e5fe44f2a6 100644 --- a/src/internal.c +++ b/src/internal.c @@ -11616,7 +11616,10 @@ void ShrinkInputBuffer(WOLFSSL* ssl, int forcedFree) int usedLength = (int)(ssl->buffers.inputBuffer.length - ssl->buffers.inputBuffer.idx); if (!forcedFree && (usedLength > STATIC_BUFFER_LEN || - ssl->buffers.clearOutputBuffer.length > 0)) + ssl->buffers.clearOutputBuffer.length > 0 || + /* Moving the data would invalidate the idx based bookkeeping + * of in-flight record processing. */ + ssl->options.processReply != doProcessInit)) return; WOLFSSL_MSG("Shrinking input buffer"); @@ -21167,6 +21170,29 @@ int writeAeadAuthData(WOLFSSL* ssl, word16 sz, byte type, return (int)idx; } +/* Encrypt application data records straight from the caller's buffer + * instead of copying the plaintext into the output buffer first. Only used + * with AEAD suites where the cipher input is exactly the plaintext. + * Excluded configs: + * - WOLFSSL_ASYNC_CRYPT and WOLFSSL_THREADED_CRYPT: encryption may run + * after BuildMessage returns, when the caller's buffer may be gone. + * - WOLFSSL_CIPHER_TEXT_CHECK, WOLFSSL_CHECK_MEM_ZERO, HAVE_FUZZER and + * CHACHA_AEAD_TEST: debug hooks that expect the record layout at the + * cipher input. + * Define WOLFSSL_BUILD_MSG_NO_ZERO_COPY to always copy. */ +#if !defined(WOLFSSL_BUILD_MSG_NO_ZERO_COPY) && \ + !defined(WOLFSSL_NO_TLS12) && !defined(WOLFSSL_ASYNC_CRYPT) && \ + !defined(WOLFSSL_THREADED_CRYPT) && \ + !defined(WOLFSSL_CIPHER_TEXT_CHECK) && \ + !defined(WOLFSSL_CHECK_MEM_ZERO) && !defined(HAVE_FUZZER) && \ + !defined(CHACHA_AEAD_TEST) + #define WOLFSSL_BUILD_MSG_ZERO_COPY +#endif + +/* out always holds the record layout: [explicit IV | ciphertext | tag]. + * In-place (input == out): input holds the same layout with the plaintext + * where the ciphertext goes. Zero-copy (input != out): input points at the + * plaintext itself; only the plaintext length is read from it. */ static WC_INLINE int EncryptDo(WOLFSSL* ssl, byte* out, const byte* input, word16 sz, int asyncOkay, byte type) { @@ -21236,6 +21262,8 @@ static WC_INLINE int EncryptDo(WOLFSSL* ssl, byte* out, const byte* input, { AES_AUTH_ENCRYPT_FUNC aes_auth_fn; int additionalSz; + const byte* plain = (input == out) ? input + AESGCM_EXP_IV_SZ + : input; #ifdef WOLFSSL_ASYNC_CRYPT /* initialize event */ @@ -21276,7 +21304,7 @@ static WC_INLINE int EncryptDo(WOLFSSL* ssl, byte* out, const byte* input, ret = NOT_COMPILED_IN; if (ssl->ctx && ssl->ctx->PerformTlsRecordProcessingCb) { ret = ssl->ctx->PerformTlsRecordProcessingCb(ssl, 1, - out + AESGCM_EXP_IV_SZ, input + AESGCM_EXP_IV_SZ, + out + AESGCM_EXP_IV_SZ, plain, sz - AESGCM_EXP_IV_SZ - ssl->specs.aead_mac_size, ssl->encrypt.nonce, AESGCM_NONCE_SZ, out + sz - ssl->specs.aead_mac_size, @@ -21288,7 +21316,7 @@ static WC_INLINE int EncryptDo(WOLFSSL* ssl, byte* out, const byte* input, #endif /* HAVE_PK_CALLBACKS */ { ret = aes_auth_fn(ssl->encrypt.aes, - out + AESGCM_EXP_IV_SZ, input + AESGCM_EXP_IV_SZ, + out + AESGCM_EXP_IV_SZ, plain, sz - (word16)(AESGCM_EXP_IV_SZ) - ssl->specs.aead_mac_size, ssl->encrypt.nonce, AESGCM_NONCE_SZ, out + sz - ssl->specs.aead_mac_size, @@ -21365,6 +21393,7 @@ static WC_INLINE int EncryptDo(WOLFSSL* ssl, byte* out, const byte* input, #if defined(HAVE_CHACHA) && defined(HAVE_POLY1305) && \ !defined(NO_CHAPOL_AEAD) case wolfssl_chacha: + /* No explicit IV: the plaintext is at input in both layouts. */ ret = ChachaAEADEncrypt(ssl, out, input, sz, type); break; #endif @@ -25366,6 +25395,33 @@ void FreeBuildMsgArgs(WOLFSSL* ssl, BuildMsgArgs* args) #endif /* Build SSL Message, encrypted */ +#ifdef WOLFSSL_BUILD_MSG_ZERO_COPY +/* Can the plaintext be encrypted straight from the caller's buffer? Only + * for AEAD suites whose cipher input is exactly the plaintext: no trailing + * MAC/pad (CBC/stream) and no appended content type (CID). */ +static int BuildMsgZeroCopyOk(WOLFSSL* ssl, int type) +{ + if (type != application_data) + return 0; + if (ssl->specs.cipher_type != aead) + return 0; + if (ssl->specs.bulk_cipher_algorithm != wolfssl_aes_gcm && + ssl->specs.bulk_cipher_algorithm != wolfssl_aes_ccm && + ssl->specs.bulk_cipher_algorithm != wolfssl_chacha) + return 0; +#if defined(WOLFSSL_DTLS) && defined(WOLFSSL_DTLS_CID) + if (ssl->options.dtls && DtlsGetCidTxSize(ssl) > 0) + return 0; +#endif +#ifdef ATOMIC_USER + /* MacEncryptCb reads the plaintext from the output buffer. */ + if (ssl->ctx->MacEncryptCb != NULL) + return 0; +#endif + return 1; +} +#endif /* WOLFSSL_BUILD_MSG_ZERO_COPY */ + int BuildMessage(WOLFSSL* ssl, byte* output, int outSz, const byte* input, int inSz, int type, int hashOutput, int sizeOnly, int asyncOkay, int epochOrder) @@ -25375,6 +25431,9 @@ int BuildMessage(WOLFSSL* ssl, byte* output, int outSz, const byte* input, BuildMsgArgs* args; BuildMsgArgs lcl_args; #endif +#ifdef WOLFSSL_BUILD_MSG_ZERO_COPY + const byte* encInput = NULL; +#endif #ifdef WOLFSSL_DTLS_CID byte cidSz = 0; @@ -25654,7 +25713,15 @@ int BuildMessage(WOLFSSL* ssl, byte* output, int outSz, const byte* input, min(args->ivSz, MAX_IV_SZ)); args->idx += min(args->ivSz, MAX_IV_SZ); } - XMEMCPY(output + args->idx, input, (size_t)(inSz)); +#ifdef WOLFSSL_BUILD_MSG_ZERO_COPY + if (BuildMsgZeroCopyOk(ssl, type)) { + encInput = input; + } + else +#endif + { + XMEMCPY(output + args->idx, input, (size_t)(inSz)); + } args->idx += (word32)inSz; #if defined(WOLFSSL_DTLS) && defined(WOLFSSL_DTLS_CID) if (ssl->options.dtls && DtlsGetCidTxSize(ssl) > 0) { @@ -25833,9 +25900,13 @@ int BuildMessage(WOLFSSL* ssl, byte* output, int outSz, const byte* input, else #endif { - ret = Encrypt(ssl, output + args->headerSz, - output + args->headerSz, args->size, asyncOkay, - args->type); + const byte* encIn = output + args->headerSz; +#ifdef WOLFSSL_BUILD_MSG_ZERO_COPY + if (encInput != NULL) + encIn = encInput; +#endif + ret = Encrypt(ssl, output + args->headerSz, encIn, + args->size, asyncOkay, args->type); } #if defined(HAVE_SECURE_RENEGOTIATION) && defined(WOLFSSL_DTLS) /* Restore sequence numbers */ diff --git a/wolfssl/internal.h b/wolfssl/internal.h index d54496e0f9..80e0f7aaa1 100644 --- a/wolfssl/internal.h +++ b/wolfssl/internal.h @@ -2363,10 +2363,6 @@ enum { */ #ifdef STATIC_BUFFER_LEN /* user supplied option */ - #if STATIC_BUFFER_LEN < 5 || STATIC_BUFFER_LEN > (RECORD_HEADER_SZ + \ - RECORD_SIZE + COMP_EXTRA + MTU_EXTRA + MAX_MSG_EXTRA)) - #error Invalid static buffer length - #endif #elif defined(LARGE_STATIC_BUFFERS) #define STATIC_BUFFER_LEN (RECORD_HEADER_SZ + RECORD_SIZE + COMP_EXTRA + \ MTU_EXTRA + MAX_MSG_EXTRA) @@ -2375,6 +2371,10 @@ enum { #define STATIC_BUFFER_LEN RECORD_HEADER_SZ #endif +/* RECORD_HEADER_SZ is an enum constant, so the preprocessor can't check + * this bound. */ +wc_static_assert(STATIC_BUFFER_LEN >= RECORD_HEADER_SZ); + typedef struct { ALIGN16 byte staticBuffer[STATIC_BUFFER_LEN]; byte* buffer; /* place holder for static or dynamic buffer */