server: fix data races found by the nightly ThreadSanitizer job

src/internal.h

@@ -44,6 +44,7 @@
 #include <wolfssl/wolfcrypt/ecc.h>
 #include <wolfssl/wolfcrypt/rsa.h>
 #include <wolfssl/wolfcrypt/random.h>
+#include <wolfssl/wolfcrypt/wc_port.h>
 #include <wolfssl/ssl.h>
 
 /* ---- tunable stack-buffer sizes ----------------------------------------- *
@@ -140,7 +141,13 @@ struct WolfCertServer {
     size_t                  cfg_csr_attrs_len;
     WolfCertCa              ca;
     int                     listen_fd;
-    volatile int            stopping;
+    /* Shutdown flag. Set by wolfcert_server_stop() -- which may run on a
+     * different thread than the accept loop (test harness) or from a signal
+     * handler (wolfcert-server CLI) -- and polled by wolfcert_server_run().
+     * wolfSSL_Atomic_Int gives the cross-thread access a happens-before edge
+     * (and async-signal-safe store); it degrades to volatile int on targets
+     * that wolfSSL builds WOLFSSL_NO_ATOMICS. */
+    wolfSSL_Atomic_Int      stopping;
     const WolfCertServerOps* ops;
     void*                   priv;        /* protocol-specific state */
     /* TLS terminator. `tls_ctx` is created in wolfcert_server_start() when

src/server.c

@@ -34,13 +34,21 @@
 #include <arpa/inet.h>
 #include <errno.h>
 #include <netinet/in.h>
+#include <poll.h>
 #include <string.h>
 #include <sys/socket.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <wolfssl/ssl.h>
 
+/* accept() poll cadence: how often wolfcert_server_run() wakes to re-check the
+ * stopping flag while idle. Bounds shutdown latency; not performance-critical.
+ */
+#ifndef WOLFCERT_SERVER_POLL_MS
+#define WOLFCERT_SERVER_POLL_MS 200
+#endif
+
 ssize_t wolfcert_io_recv(WolfCertServer* srv, int fd, void* buf, size_t len)
 {
     if (srv != NULL && srv->tls_current != NULL) {
@@ -174,6 +182,11 @@ int wolfcert_server_start(const WolfCertServerCfgSrv* cfg, WolfCertServer** out)
     if (s == NULL)
         return WOLFCERT_ERR_MEMORY;
 
+    /* Zero-init covers every field, including the wolfSSL_Atomic_Int `stopping`
+     * flag: it is a lock-free integer atomic, so an all-zero representation is
+     * a valid initialized value of 0. Nothing else touches `s` until the caller
+     * publishes it to the serving thread (pthread_create is the happens-before
+     * edge), so no atomic_init()/barrier is needed here. */
     memset(s, 0, sizeof(*s));
     s->cfg       = *cfg;
     s->listen_fd = -1;
@@ -260,22 +273,58 @@ int wolfcert_server_start(const WolfCertServerCfgSrv* cfg, WolfCertServer** out)
 
 int wolfcert_server_run(WolfCertServer* srv)
 {
+    struct pollfd pfd;
     int ret;
+    int pr;
+    int cs;
 
     if (srv == NULL)
         return WOLFCERT_ERR_BAD_ARG;
 
-    while (!srv->stopping) {
-        int cs = accept(srv->listen_fd, NULL, NULL);
-        if (cs < 0) {
-            if (srv->stopping)
-                break;
-
+    /* Poll the listener with a short timeout rather than blocking in accept(),
+     * so the loop re-checks the stopping flag on its own. This keeps shutdown
+     * portable -- neither shutdown() nor close() from another thread reliably
+     * wakes a blocked accept() on BSD/macOS -- and race-free: listen_fd is
+     * touched only by start() (before the serving thread exists) and free()
+     * (after it has been joined), never concurrently with this loop. */
+    while (!WOLFSSL_ATOMIC_LOAD(srv->stopping)) {
+        pfd.fd     = srv->listen_fd;
+        pfd.events = POLLIN;
+
+        pr = poll(&pfd, 1, WOLFCERT_SERVER_POLL_MS);
+        if (pr < 0) {
             if (errno == EINTR)
                 continue;
 
             return WOLFCERT_ERR_IO;
         }
+        if (pr == 0)
+            continue;   /* timed out -- re-check stopping */
+
+        /* poll() reports the listener ready for either a pending connection
+         * (POLLIN) or an error/hangup condition (POLLERR/POLLHUP/POLLNVAL).
+         * Accept only on POLLIN: an error condition on the listener is fatal,
+         * and accepting on it could block or spin the loop. */
+        if ((pfd.revents & POLLIN) == 0)
+            return WOLFCERT_ERR_IO;
+
+        /* stopping may have been set between poll() returning and here; honour
+         * it now rather than serving one more connection. */
+        if (WOLFSSL_ATOMIC_LOAD(srv->stopping))
+            break;
+
+        cs = accept(srv->listen_fd, NULL, NULL);
+        if (cs < 0) {
+            /* A single misbehaving peer must not take the listener down. A
+             * reset between poll() reporting POLLIN and accept() running
+             * surfaces as ECONNABORTED (ECONNRESET on some systems); EINTR is
+             * a delivered signal. Retry those -- only a genuine listener
+             * failure is fatal. */
+            if (errno == EINTR || errno == ECONNABORTED || errno == ECONNRESET)
+                continue;
+
+            return WOLFCERT_ERR_IO;
+        }
 
         if (srv->tls_ctx != NULL) {
             /* Terminate TLS on this accepted fd. The protocol handler sees
@@ -299,7 +348,7 @@ int wolfcert_server_run(WolfCertServer* srv)
                         if (srv->ops->serve_fd(srv, cs) != WOLFCERT_OK)
                             break;
                     }
-                    while (srv->keep_alive && !srv->stopping);
+                    while (srv->keep_alive && !WOLFSSL_ATOMIC_LOAD(srv->stopping));
 
                     srv->tls_current = NULL;
                     wolfSSL_shutdown(ssl);
@@ -318,7 +367,7 @@ int wolfcert_server_run(WolfCertServer* srv)
                 if (srv->ops->serve_fd(srv, cs) != WOLFCERT_OK)
                     break;
             }
-            while (srv->keep_alive && !srv->stopping);
+            while (srv->keep_alive && !WOLFSSL_ATOMIC_LOAD(srv->stopping));
         }
 
         close(cs);
@@ -340,13 +389,12 @@ int wolfcert_server_stop(WolfCertServer* srv)
     if (srv == NULL)
         return WOLFCERT_ERR_BAD_ARG;
 
-    srv->stopping = 1;
-
-    if (srv->listen_fd >= 0) {
-        shutdown(srv->listen_fd, SHUT_RDWR);
-        close(srv->listen_fd);
-        srv->listen_fd = -1;
-    }
+    /* Signal the accept loop to exit. It polls the listener on a short timeout
+     * (WOLFCERT_SERVER_POLL_MS) and re-checks this flag, so no fd surgery is
+     * needed here -- wolfcert_server_free() closes listen_fd after the serving
+     * thread is joined. Setting the flag from another thread (test harness) or
+     * a signal handler (wolfcert-server CLI) is safe: the store is atomic. */
+    WOLFSSL_ATOMIC_STORE(srv->stopping, 1);
 
     return WOLFCERT_OK;
 }

tests/integration/test_tls_http.c

@@ -35,6 +35,7 @@
 #include <wolfssl/wolfcrypt/asn_public.h>
 #include <wolfssl/wolfcrypt/rsa.h>
 #include <wolfssl/wolfcrypt/random.h>
+#include <wolfssl/wolfcrypt/wc_port.h>
 
 #include "tls_test_util.h"
 
@@ -57,7 +58,10 @@
     } while (0)
 
 struct srv_ctx {
-    int port;
+    /* Published by srv_thread once the ephemeral listener is bound, then
+     * polled by the main thread. Atomic so the cross-thread handoff has a
+     * happens-before edge (this is what ThreadSanitizer requires). */
+    wolfSSL_Atomic_Int port;
     uint8_t* cert_pem;
     size_t cert_pem_len;
     uint8_t* key_pem;
@@ -86,7 +90,7 @@ static void* srv_thread(void* arg)
     listen(ls, 1);
     socklen_t slen = sizeof(sa);
     getsockname(ls, (struct sockaddr*)&sa, &slen);
-    sc->port = ntohs(sa.sin_port);
+    WOLFSSL_ATOMIC_STORE(sc->port, ntohs(sa.sin_port));
 
     int cs = accept(ls, NULL, NULL);
     close(ls);
@@ -123,14 +127,14 @@ int main(void)
 
     pthread_t tid;
     REQUIRE(pthread_create(&tid, NULL, srv_thread, &sc) == 0);
-    for (int i = 0; i < 200 && sc.port == 0; ++i) {
+    for (int i = 0; i < 200 && WOLFSSL_ATOMIC_LOAD(sc.port) == 0; ++i) {
         const struct timespec ts = { 0, 5 * 1000 * 1000 };
         nanosleep(&ts, NULL);
     }
-    REQUIRE(sc.port != 0);
+    REQUIRE(WOLFSSL_ATOMIC_LOAD(sc.port) != 0);
 
     char url[128];
-    snprintf(url, sizeof(url), "https://127.0.0.1:%d/", sc.port);
+    snprintf(url, sizeof(url), "https://127.0.0.1:%d/", WOLFSSL_ATOMIC_LOAD(sc.port));
 
     WolfCertHttpRequest req = {
         .method            = "GET",