Zero-copy encrypt of AEAD application data

.wolfssl_known_macro_extras

@@ -798,6 +798,7 @@ WOLFSSL_ATMEL_TIME
 WOLFSSL_BEFORE_DATE_CLOCK_SKEW
 WOLFSSL_BIGINT_TYPES
 WOLFSSL_BIO_NO_FLOW_STATS
+WOLFSSL_BUILD_MSG_NO_ZERO_COPY
 WOLFSSL_BYTESWAP32_ASM
 WOLFSSL_CAAM_BLACK_KEY_AESCCM
 WOLFSSL_CAAM_BLACK_KEY_SM

src/internal.c

@@ -11616,7 +11616,10 @@ void ShrinkInputBuffer(WOLFSSL* ssl, int forcedFree)
     int usedLength = (int)(ssl->buffers.inputBuffer.length -
                      ssl->buffers.inputBuffer.idx);
     if (!forcedFree && (usedLength > STATIC_BUFFER_LEN ||
-            ssl->buffers.clearOutputBuffer.length > 0))
+            ssl->buffers.clearOutputBuffer.length > 0 ||
+            /* Moving the data would invalidate the idx based bookkeeping
+             * of in-flight record processing. */
+            ssl->options.processReply != doProcessInit))
         return;
 
     WOLFSSL_MSG("Shrinking input buffer");
@@ -21167,6 +21170,29 @@ int writeAeadAuthData(WOLFSSL* ssl, word16 sz, byte type,
     return (int)idx;
 }
 
+/* Encrypt application data records straight from the caller's buffer
+ * instead of copying the plaintext into the output buffer first. Only used
+ * with AEAD suites where the cipher input is exactly the plaintext.
+ * Excluded configs:
+ * - WOLFSSL_ASYNC_CRYPT and WOLFSSL_THREADED_CRYPT: encryption may run
+ *   after BuildMessage returns, when the caller's buffer may be gone.
+ * - WOLFSSL_CIPHER_TEXT_CHECK, WOLFSSL_CHECK_MEM_ZERO, HAVE_FUZZER and
+ *   CHACHA_AEAD_TEST: debug hooks that expect the record layout at the
+ *   cipher input.
+ * Define WOLFSSL_BUILD_MSG_NO_ZERO_COPY to always copy. */
+#if !defined(WOLFSSL_BUILD_MSG_NO_ZERO_COPY) && \
+    !defined(WOLFSSL_NO_TLS12) && !defined(WOLFSSL_ASYNC_CRYPT) && \
+    !defined(WOLFSSL_THREADED_CRYPT) && \
+    !defined(WOLFSSL_CIPHER_TEXT_CHECK) && \
+    !defined(WOLFSSL_CHECK_MEM_ZERO) && !defined(HAVE_FUZZER) && \
+    !defined(CHACHA_AEAD_TEST)
+    #define WOLFSSL_BUILD_MSG_ZERO_COPY
+#endif
+
+/* out always holds the record layout: [explicit IV | ciphertext | tag].
+ * In-place (input == out): input holds the same layout with the plaintext
+ * where the ciphertext goes. Zero-copy (input != out): input points at the
+ * plaintext itself; only the plaintext length is read from it. */
 static WC_INLINE int EncryptDo(WOLFSSL* ssl, byte* out, const byte* input,
     word16 sz, int asyncOkay, byte type)
 {
@@ -21236,6 +21262,8 @@ static WC_INLINE int EncryptDo(WOLFSSL* ssl, byte* out, const byte* input,
         {
             AES_AUTH_ENCRYPT_FUNC aes_auth_fn;
             int additionalSz;
+            const byte* plain = (input == out) ? input + AESGCM_EXP_IV_SZ
+                                               : input;
 
         #ifdef WOLFSSL_ASYNC_CRYPT
             /* initialize event */
@@ -21276,7 +21304,7 @@ static WC_INLINE int EncryptDo(WOLFSSL* ssl, byte* out, const byte* input,
             ret = NOT_COMPILED_IN;
             if (ssl->ctx && ssl->ctx->PerformTlsRecordProcessingCb) {
                 ret = ssl->ctx->PerformTlsRecordProcessingCb(ssl, 1,
-                         out + AESGCM_EXP_IV_SZ, input + AESGCM_EXP_IV_SZ,
+                         out + AESGCM_EXP_IV_SZ, plain,
                          sz - AESGCM_EXP_IV_SZ - ssl->specs.aead_mac_size,
                          ssl->encrypt.nonce, AESGCM_NONCE_SZ,
                          out + sz - ssl->specs.aead_mac_size,
@@ -21288,7 +21316,7 @@ static WC_INLINE int EncryptDo(WOLFSSL* ssl, byte* out, const byte* input,
         #endif /* HAVE_PK_CALLBACKS */
             {
                 ret = aes_auth_fn(ssl->encrypt.aes,
-                    out + AESGCM_EXP_IV_SZ, input + AESGCM_EXP_IV_SZ,
+                    out + AESGCM_EXP_IV_SZ, plain,
                     sz - (word16)(AESGCM_EXP_IV_SZ) - ssl->specs.aead_mac_size,
                     ssl->encrypt.nonce, AESGCM_NONCE_SZ,
                     out + sz - ssl->specs.aead_mac_size,
@@ -21365,6 +21393,7 @@ static WC_INLINE int EncryptDo(WOLFSSL* ssl, byte* out, const byte* input,
     #if defined(HAVE_CHACHA) && defined(HAVE_POLY1305) && \
         !defined(NO_CHAPOL_AEAD)
         case wolfssl_chacha:
+            /* No explicit IV: the plaintext is at input in both layouts. */
             ret = ChachaAEADEncrypt(ssl, out, input, sz, type);
             break;
     #endif
@@ -25366,6 +25395,33 @@ void FreeBuildMsgArgs(WOLFSSL* ssl, BuildMsgArgs* args)
 #endif
 
 /* Build SSL Message, encrypted */
+#ifdef WOLFSSL_BUILD_MSG_ZERO_COPY
+/* Can the plaintext be encrypted straight from the caller's buffer? Only
+ * for AEAD suites whose cipher input is exactly the plaintext: no trailing
+ * MAC/pad (CBC/stream) and no appended content type (CID). */
+static int BuildMsgZeroCopyOk(WOLFSSL* ssl, int type)
+{
+    if (type != application_data)
+        return 0;
+    if (ssl->specs.cipher_type != aead)
+        return 0;
+    if (ssl->specs.bulk_cipher_algorithm != wolfssl_aes_gcm &&
+            ssl->specs.bulk_cipher_algorithm != wolfssl_aes_ccm &&
+            ssl->specs.bulk_cipher_algorithm != wolfssl_chacha)
+        return 0;
+#if defined(WOLFSSL_DTLS) && defined(WOLFSSL_DTLS_CID)
+    if (ssl->options.dtls && DtlsGetCidTxSize(ssl) > 0)
+        return 0;
+#endif
+#ifdef ATOMIC_USER
+    /* MacEncryptCb reads the plaintext from the output buffer. */
+    if (ssl->ctx->MacEncryptCb != NULL)
+        return 0;
+#endif
+    return 1;
+}
+#endif /* WOLFSSL_BUILD_MSG_ZERO_COPY */
+
 int BuildMessage(WOLFSSL* ssl, byte* output, int outSz, const byte* input,
              int inSz, int type, int hashOutput, int sizeOnly, int asyncOkay,
              int epochOrder)
@@ -25375,6 +25431,9 @@ int BuildMessage(WOLFSSL* ssl, byte* output, int outSz, const byte* input,
     BuildMsgArgs* args;
     BuildMsgArgs  lcl_args;
 #endif
+#ifdef WOLFSSL_BUILD_MSG_ZERO_COPY
+    const byte* encInput = NULL;
+#endif
 
 #ifdef WOLFSSL_DTLS_CID
     byte cidSz = 0;
@@ -25654,7 +25713,15 @@ int BuildMessage(WOLFSSL* ssl, byte* output, int outSz, const byte* input,
                                         min(args->ivSz, MAX_IV_SZ));
                 args->idx += min(args->ivSz, MAX_IV_SZ);
             }
-            XMEMCPY(output + args->idx, input, (size_t)(inSz));
+#ifdef WOLFSSL_BUILD_MSG_ZERO_COPY
+            if (BuildMsgZeroCopyOk(ssl, type)) {
+                encInput = input;
+            }
+            else
+#endif
+            {
+                XMEMCPY(output + args->idx, input, (size_t)(inSz));
+            }
             args->idx += (word32)inSz;
 #if defined(WOLFSSL_DTLS) && defined(WOLFSSL_DTLS_CID)
             if (ssl->options.dtls && DtlsGetCidTxSize(ssl) > 0) {
@@ -25833,9 +25900,13 @@ int BuildMessage(WOLFSSL* ssl, byte* output, int outSz, const byte* input,
             else
     #endif
             {
-                ret = Encrypt(ssl, output + args->headerSz,
-                                output + args->headerSz, args->size, asyncOkay,
-                                args->type);
+                const byte* encIn = output + args->headerSz;
+#ifdef WOLFSSL_BUILD_MSG_ZERO_COPY
+                if (encInput != NULL)
+                    encIn = encInput;
+#endif
+                ret = Encrypt(ssl, output + args->headerSz, encIn,
+                                args->size, asyncOkay, args->type);
             }
     #if defined(HAVE_SECURE_RENEGOTIATION) && defined(WOLFSSL_DTLS)
             /* Restore sequence numbers */

wolfssl/internal.h

@@ -2363,10 +2363,6 @@ enum {
 */
 #ifdef STATIC_BUFFER_LEN
     /* user supplied option */
-    #if STATIC_BUFFER_LEN < 5 || STATIC_BUFFER_LEN > (RECORD_HEADER_SZ + \
-                          RECORD_SIZE + COMP_EXTRA + MTU_EXTRA + MAX_MSG_EXTRA))
-        #error Invalid static buffer length
-    #endif
 #elif defined(LARGE_STATIC_BUFFERS)
     #define STATIC_BUFFER_LEN (RECORD_HEADER_SZ + RECORD_SIZE + COMP_EXTRA + \
              MTU_EXTRA + MAX_MSG_EXTRA)
@@ -2375,6 +2371,10 @@ enum {
     #define STATIC_BUFFER_LEN RECORD_HEADER_SZ
 #endif
 
+/* RECORD_HEADER_SZ is an enum constant, so the preprocessor can't check
+ * this bound. */
+wc_static_assert(STATIC_BUFFER_LEN >= RECORD_HEADER_SZ);
+
 typedef struct {
     ALIGN16 byte staticBuffer[STATIC_BUFFER_LEN];
     byte*  buffer;       /* place holder for static or dynamic buffer */