ci: add TSAN integration test job

.github/workflows/ci-tests.yaml

@@ -187,3 +187,58 @@ jobs:
             BAZEL_TEST_OPTS=--jobs=HOST_RAM*.00005 --test_timeout=900 --local_test_jobs=1 --notest_keep_going --flaky_test_attempts=1 --noshow_progress --noshow_loading_progress --test_env=MSAN_SYMBOLIZER_PATH=/usr/lib/llvm-18/bin/llvm-symbolizer
           cache-from: type=local,src=/tmp/buildx-cache
           push: false
+
+  tsan-tests:
+    timeout-minutes: 360
+    name: Run TSAN integration tests on amd64
+    runs-on: ${{ vars.PROXY_BUILD_GITHUB_RUNNER || 'oracle-vm-32cpu-128gb-x86-64' }}
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
+
+      - name: Cache Docker layers
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: /tmp/buildx-cache
+          key: docker-cache-tests-tsan
+          restore-keys: docker-cache-tests
+
+      - name: Checkout PR Source Code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+
+      - name: Prep for build
+        run: |
+          echo "${{ github.event.pull_request.head.sha }}" >SOURCE_VERSION
+          echo "BUILDER_DOCKER_HASH=$(git ls-tree --full-tree HEAD -- ./Dockerfile.builder | awk '{ print $3 }')" >> $GITHUB_ENV
+
+      - name: Wait for build image
+        uses: ./.github/workflows/wait-for-image
+        with:
+          SHA: ${{ env.BUILDER_DOCKER_HASH }}
+          repo: cilium
+          images: cilium-envoy-builder-dev
+
+      - name: Run TSAN integration tests on amd64
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        id: docker_tsan_tests_ci
+        with:
+          provenance: false
+          context: .
+          file: ./Dockerfile.tests
+          platforms: linux/amd64
+          build-args: |
+            BUILDER_BASE=quay.io/${{ github.repository_owner }}/cilium-envoy-builder-dev:${{ env.BUILDER_DOCKER_HASH }}
+            PROXYLIB_BUILDER=quay.io/${{ github.repository_owner }}/cilium-envoy-builder-dev:${{ env.BUILDER_DOCKER_HASH }}
+            PROXYLIB_CC=/usr/lib/llvm-18/bin/clang
+            PROXYLIB_CGO_CFLAGS=-fsanitize=thread
+            PROXYLIB_GO_BUILD_FLAGS=-installsuffix=tsan -buildvcs=false
+            PROXYLIB_GOCACHE=/tmp/go-build
+            ARCHIVE_IMAGE=quay.io/${{ github.repository_owner }}/cilium-envoy-builder:test-main-archive-latest
+            DEBUG=1
+            BAZEL_BUILD_OPTS=--config=tsan --remote_upload_local_results=false
+            BAZEL_TEST_OPTS=--jobs=HOST_RAM*.00005 --test_timeout=900 --local_test_jobs=1 --notest_keep_going --flaky_test_attempts=1 --noshow_progress --noshow_loading_progress --test_env=TSAN_OPTIONS=report_atomic_races=0:halt_on_error=1:external_symbolizer_path=/usr/lib/llvm-18/bin/llvm-symbolizer:strip_path_prefix=/proc/self/cwd/
+          cache-from: type=local,src=/tmp/buildx-cache
+          push: false

Dockerfile.tests

@@ -22,13 +22,15 @@ WORKDIR /go/src/github.com/cilium/proxy
 ENV PATH=/usr/local/go/bin:$PATH
 ARG TARGETARCH
 ARG PROXYLIB_CC=gcc
+ARG PROXYLIB_CGO_CFLAGS
 ARG PROXYLIB_GO_BUILD_FLAGS
 ARG PROXYLIB_GOCACHE
 RUN --mount=type=bind,target=/go/src/github.com/cilium/proxy \
     --mount=mode=0777,target=/cilium/proxy/.cache,type=cache \
     --mount=mode=0777,target=/go/pkg,type=cache \
     --mount=mode=0777,uid=1337,gid=1337,target=/tmp/go-build,type=cache \
     if [ -n "${PROXYLIB_GOCACHE}" ]; then export GOCACHE="${PROXYLIB_GOCACHE}"; fi && \
+    if [ -n "${PROXYLIB_CGO_CFLAGS}" ]; then export CGO_CFLAGS="${PROXYLIB_CGO_CFLAGS}"; fi && \
     CC="${PROXYLIB_CC}" GOARCH="${TARGETARCH}" GO_BUILD_FLAGS="${PROXYLIB_GO_BUILD_FLAGS}" make -C proxylib TARGET=/tmp/libcilium.so all
 
 FROM --platform=$BUILDPLATFORM $BUILDER_BASE AS builder-fresh

Makefile

@@ -62,6 +62,12 @@ BAZEL_MSAN_TEST_OPTS ?= --jobs=HOST_RAM*.00005 --test_timeout=900 --local_test_j
 PROXYLIB_MSAN_CC ?= clang
 PROXYLIB_MSAN_GO_BUILD_FLAGS ?= -msan -buildvcs=false
 
+BAZEL_TSAN_BUILD_OPTS ?= --config=tsan $(EXTRA_BAZEL_BUILD_OPTS) -c dbg
+BAZEL_TSAN_TEST_OPTS ?= --jobs=HOST_RAM*.00005 --test_timeout=900 --local_test_jobs=1 --flaky_test_attempts=1 --test_output=errors
+PROXYLIB_TSAN_CC ?= clang
+PROXYLIB_TSAN_CGO_CFLAGS ?= -fsanitize=thread
+PROXYLIB_TSAN_GO_BUILD_FLAGS ?= -installsuffix=tsan -buildvcs=false
+
 ifdef DEBUG
   BAZEL_BUILD_OPTS += -c dbg
 else ifdef RELEASE_DEBUG
@@ -225,6 +231,14 @@ proxylib-msan:
 envoy-msan-tests: proxylib-msan
 	$(MAKE) envoy-tests BAZEL_BUILD_OPTS="$(BAZEL_MSAN_BUILD_OPTS)" BAZEL_TEST_OPTS="$(BAZEL_MSAN_TEST_OPTS)"
 
+.PHONY: proxylib-tsan
+proxylib-tsan:
+	$(MAKE) -C proxylib all CC="$(PROXYLIB_TSAN_CC)" CGO_CFLAGS="$(PROXYLIB_TSAN_CGO_CFLAGS)" GO_BUILD_FLAGS="$(PROXYLIB_TSAN_GO_BUILD_FLAGS)"
+
+.PHONY: envoy-tsan-tests
+envoy-tsan-tests: proxylib-tsan
+	$(MAKE) envoy-tests BAZEL_BUILD_OPTS="$(BAZEL_TSAN_BUILD_OPTS)" BAZEL_TEST_OPTS="$(BAZEL_TSAN_TEST_OPTS)"
+
 .PHONY: \
 	install \
 	force \

bazel/toolchains/BUILD

@@ -12,6 +12,14 @@ config_setting(
     values = {"define": "ENVOY_CONFIG_MSAN=1"},
 )
 
+config_setting(
+    name = "tsan_target_build",
+    constraint_values = [
+        "//bazel:cilium_target_platform_enabled",
+    ],
+    values = {"define": "ENVOY_CONFIG_TSAN=1"},
+)
+
 toolchain(
     name = "aarch64_linux_cc_toolchain",
     exec_compatible_with = ["@platforms//os:linux"],
@@ -178,6 +186,7 @@ cc_toolchain_config(
     ],
     cxx_flags = ["-std=c++0x"] + select({
         "@envoy//bazel:msan_build": ["-stdlib=libc++"],
+        "@envoy//bazel:tsan_build": ["-stdlib=libc++"],
         "//conditions:default": [],
     }),
     dbg_compile_flags = ["-g"],
@@ -190,19 +199,28 @@ cc_toolchain_config(
         "-lm",
     ] + select({
         "@envoy//bazel:msan_build": ["-L/usr/lib/llvm-18/lib"],
+        "@envoy//bazel:tsan_build": ["-L/usr/lib/llvm-18/lib"],
         "//conditions:default": [],
     }),
-    # Envoy supplies instrumented libc++ for MSAN targets. Non-instrumented
-    # Bazel exec tools use the system libc++ runtime below.
+    # Envoy supplies instrumented libc++ for MSAN and TSAN targets.
+    # Non-instrumented Bazel exec tools use the system libc++ runtime below.
     link_libs = select({
         ":msan_target_build": [
             "-latomic",
         ],
+        ":tsan_target_build": [
+            "-latomic",
+        ],
         "@envoy//bazel:msan_build": [
             "-l:libc++.a",
             "-l:libc++abi.a",
             "-latomic",
         ],
+        "@envoy//bazel:tsan_build": [
+            "-l:libc++.a",
+            "-l:libc++abi.a",
+            "-latomic",
+        ],
         "//conditions:default": [
             "-l:libstdc++.a",
             "-latomic",

tests/cilium_tls_tcp_integration_test.cc

@@ -18,7 +18,6 @@
 #include "envoy/network/address.h"
 #include "envoy/network/connection.h"
 #include "envoy/network/transport_socket.h"
-#include "envoy/ssl/connection.h"
 
 #include "source/common/buffer/buffer_impl.h"
 #include "source/common/buffer/watermark_buffer.h"
@@ -119,12 +118,23 @@ class CiliumTLSIntegrationTest : public CiliumTcpIntegrationTest {
   }
 
   AssertionResult
-  waitForTlsHandshake(const FakeRawConnection& connection,
+  waitForTlsHandshake(FakeRawConnection& connection,
                       std::chrono::milliseconds timeout = TestUtility::DefaultTimeout) {
     Event::TestTimeSystem::RealTimeBound bound(timeout);
     while (true) {
-      const auto downstream_timing = connection.connection().streamInfo().downstreamTiming();
-      if (downstream_timing.downstreamHandshakeComplete().has_value()) {
+      bool handshake_complete = false;
+      const auto state_result = connection.sharedConnection().executeOnDispatcher(
+          [&handshake_complete](Network::Connection& upstream_connection) {
+            handshake_complete = upstream_connection.streamInfo()
+                                     .downstreamTiming()
+                                     .downstreamHandshakeComplete()
+                                     .has_value();
+          });
+      if (!state_result) {
+        return state_result;
+      }
+
+      if (handshake_complete) {
         return AssertionSuccess();
       }
 
@@ -134,13 +144,33 @@ class CiliumTLSIntegrationTest : public CiliumTcpIntegrationTest {
       timeSystem().advanceTimeWait(std::chrono::milliseconds(1));
 
       if (timeout != std::chrono::milliseconds::zero() && !bound.withinBound()) {
-        const Ssl::ConnectionInfoConstSharedPtr ssl = connection.connection().ssl();
-        return AssertionFailure() << "Timed out waiting for TLS handshake. ssl=" << (ssl != nullptr)
-                                  << " handshake_complete="
-                                  << downstream_timing.downstreamHandshakeComplete().has_value()
-                                  << " tls_version=" << (ssl != nullptr ? ssl->tlsVersion() : "")
-                                  << " ciphersuite="
-                                  << (ssl != nullptr ? ssl->ciphersuiteString() : "");
+        bool has_ssl = false;
+        std::string tls_version;
+        std::string ciphersuite;
+        const auto diagnostic_result = connection.sharedConnection().executeOnDispatcher(
+            [&](Network::Connection& upstream_connection) {
+              handshake_complete = upstream_connection.streamInfo()
+                                       .downstreamTiming()
+                                       .downstreamHandshakeComplete()
+                                       .has_value();
+              const auto ssl = upstream_connection.ssl();
+              has_ssl = ssl != nullptr;
+              if (ssl != nullptr) {
+                tls_version = std::string(ssl->tlsVersion());
+                ciphersuite = std::string(ssl->ciphersuiteString());
+              }
+            });
+        if (!diagnostic_result) {
+          return diagnostic_result;
+        }
+        if (handshake_complete) {
+          return AssertionSuccess();
+        }
+
+        return AssertionFailure() << "Timed out waiting for TLS handshake. ssl=" << has_ssl
+                                  << " handshake_complete=" << handshake_complete
+                                  << " tls_version=" << tls_version
+                                  << " ciphersuite=" << ciphersuite;
       }
     }
   }

tests/uds_server.cc

@@ -69,19 +69,21 @@ void UDSServer::shutdownServerThread() {
 
   if (fd2 >= 0) {
     ::shutdown(fd2, SHUT_RD);
-    ::close(fd2);
   }
   if (fd >= 0) {
     ::shutdown(fd, SHUT_RD);
-    errno = 0;
-    ::close(fd);
   }
   if (thread_ != nullptr) {
-    ENVOY_LOG(trace, "Waiting on unix domain socket server to close: {}",
-              Envoy::errorDetails(errno));
+    ENVOY_LOG(trace, "Waiting on unix domain socket server thread");
     thread_->join();
     thread_.reset();
   }
+  if (fd2 >= 0 && ::close(fd2) == -1) {
+    ENVOY_LOG(warn, "Closing UDS client socket {} failed: {}", fd2, Envoy::errorDetails(errno));
+  }
+  if (fd >= 0 && ::close(fd) == -1) {
+    ENVOY_LOG(warn, "Closing UDS server socket {} failed: {}", fd, Envoy::errorDetails(errno));
+  }
   if (!addr_->pipe()->abstractNamespace()) {
     ::unlink(addr_->asString().c_str());
   }