diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index eb08b7a9f..7c6947af3 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -187,3 +187,58 @@ jobs: BAZEL_TEST_OPTS=--jobs=HOST_RAM*.00005 --test_timeout=900 --local_test_jobs=1 --notest_keep_going --flaky_test_attempts=1 --noshow_progress --noshow_loading_progress --test_env=MSAN_SYMBOLIZER_PATH=/usr/lib/llvm-18/bin/llvm-symbolizer cache-from: type=local,src=/tmp/buildx-cache push: false + + tsan-tests: + timeout-minutes: 360 + name: Run TSAN integration tests on amd64 + runs-on: ${{ vars.PROXY_BUILD_GITHUB_RUNNER || 'oracle-vm-32cpu-128gb-x86-64' }} + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + - name: Cache Docker layers + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: /tmp/buildx-cache + key: docker-cache-tests-tsan + restore-keys: docker-cache-tests + + - name: Checkout PR Source Code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + + - name: Prep for build + run: | + echo "${{ github.event.pull_request.head.sha }}" >SOURCE_VERSION + echo "BUILDER_DOCKER_HASH=$(git ls-tree --full-tree HEAD -- ./Dockerfile.builder | awk '{ print $3 }')" >> $GITHUB_ENV + + - name: Wait for build image + uses: ./.github/workflows/wait-for-image + with: + SHA: ${{ env.BUILDER_DOCKER_HASH }} + repo: cilium + images: cilium-envoy-builder-dev + + - name: Run TSAN integration tests on amd64 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + id: docker_tsan_tests_ci + with: + provenance: false + context: . + file: ./Dockerfile.tests + platforms: linux/amd64 + build-args: | + BUILDER_BASE=quay.io/${{ github.repository_owner }}/cilium-envoy-builder-dev:${{ env.BUILDER_DOCKER_HASH }} + PROXYLIB_BUILDER=quay.io/${{ github.repository_owner }}/cilium-envoy-builder-dev:${{ env.BUILDER_DOCKER_HASH }} + PROXYLIB_CC=/usr/lib/llvm-18/bin/clang + PROXYLIB_CGO_CFLAGS=-fsanitize=thread + PROXYLIB_GO_BUILD_FLAGS=-installsuffix=tsan -buildvcs=false + PROXYLIB_GOCACHE=/tmp/go-build + ARCHIVE_IMAGE=quay.io/${{ github.repository_owner }}/cilium-envoy-builder:test-main-archive-latest + DEBUG=1 + BAZEL_BUILD_OPTS=--config=tsan --remote_upload_local_results=false + BAZEL_TEST_OPTS=--jobs=HOST_RAM*.00005 --test_timeout=900 --local_test_jobs=1 --notest_keep_going --flaky_test_attempts=1 --noshow_progress --noshow_loading_progress --test_env=TSAN_OPTIONS=report_atomic_races=0:halt_on_error=1:external_symbolizer_path=/usr/lib/llvm-18/bin/llvm-symbolizer:strip_path_prefix=/proc/self/cwd/ + cache-from: type=local,src=/tmp/buildx-cache + push: false diff --git a/Dockerfile.tests b/Dockerfile.tests index e8d751d41..14953d804 100644 --- a/Dockerfile.tests +++ b/Dockerfile.tests @@ -22,6 +22,7 @@ WORKDIR /go/src/github.com/cilium/proxy ENV PATH=/usr/local/go/bin:$PATH ARG TARGETARCH ARG PROXYLIB_CC=gcc +ARG PROXYLIB_CGO_CFLAGS ARG PROXYLIB_GO_BUILD_FLAGS ARG PROXYLIB_GOCACHE RUN --mount=type=bind,target=/go/src/github.com/cilium/proxy \ @@ -29,6 +30,7 @@ RUN --mount=type=bind,target=/go/src/github.com/cilium/proxy \ --mount=mode=0777,target=/go/pkg,type=cache \ --mount=mode=0777,uid=1337,gid=1337,target=/tmp/go-build,type=cache \ if [ -n "${PROXYLIB_GOCACHE}" ]; then export GOCACHE="${PROXYLIB_GOCACHE}"; fi && \ + if [ -n "${PROXYLIB_CGO_CFLAGS}" ]; then export CGO_CFLAGS="${PROXYLIB_CGO_CFLAGS}"; fi && \ CC="${PROXYLIB_CC}" GOARCH="${TARGETARCH}" GO_BUILD_FLAGS="${PROXYLIB_GO_BUILD_FLAGS}" make -C proxylib TARGET=/tmp/libcilium.so all FROM --platform=$BUILDPLATFORM $BUILDER_BASE AS builder-fresh diff --git a/Makefile b/Makefile index ee7cb87ee..eb5581a87 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,12 @@ BAZEL_MSAN_TEST_OPTS ?= --jobs=HOST_RAM*.00005 --test_timeout=900 --local_test_j PROXYLIB_MSAN_CC ?= clang PROXYLIB_MSAN_GO_BUILD_FLAGS ?= -msan -buildvcs=false +BAZEL_TSAN_BUILD_OPTS ?= --config=tsan $(EXTRA_BAZEL_BUILD_OPTS) -c dbg +BAZEL_TSAN_TEST_OPTS ?= --jobs=HOST_RAM*.00005 --test_timeout=900 --local_test_jobs=1 --flaky_test_attempts=1 --test_output=errors +PROXYLIB_TSAN_CC ?= clang +PROXYLIB_TSAN_CGO_CFLAGS ?= -fsanitize=thread +PROXYLIB_TSAN_GO_BUILD_FLAGS ?= -installsuffix=tsan -buildvcs=false + ifdef DEBUG BAZEL_BUILD_OPTS += -c dbg else ifdef RELEASE_DEBUG @@ -225,6 +231,14 @@ proxylib-msan: envoy-msan-tests: proxylib-msan $(MAKE) envoy-tests BAZEL_BUILD_OPTS="$(BAZEL_MSAN_BUILD_OPTS)" BAZEL_TEST_OPTS="$(BAZEL_MSAN_TEST_OPTS)" +.PHONY: proxylib-tsan +proxylib-tsan: + $(MAKE) -C proxylib all CC="$(PROXYLIB_TSAN_CC)" CGO_CFLAGS="$(PROXYLIB_TSAN_CGO_CFLAGS)" GO_BUILD_FLAGS="$(PROXYLIB_TSAN_GO_BUILD_FLAGS)" + +.PHONY: envoy-tsan-tests +envoy-tsan-tests: proxylib-tsan + $(MAKE) envoy-tests BAZEL_BUILD_OPTS="$(BAZEL_TSAN_BUILD_OPTS)" BAZEL_TEST_OPTS="$(BAZEL_TSAN_TEST_OPTS)" + .PHONY: \ install \ force \ diff --git a/bazel/toolchains/BUILD b/bazel/toolchains/BUILD index 1e626a387..2aa0b67ac 100644 --- a/bazel/toolchains/BUILD +++ b/bazel/toolchains/BUILD @@ -12,6 +12,14 @@ config_setting( values = {"define": "ENVOY_CONFIG_MSAN=1"}, ) +config_setting( + name = "tsan_target_build", + constraint_values = [ + "//bazel:cilium_target_platform_enabled", + ], + values = {"define": "ENVOY_CONFIG_TSAN=1"}, +) + toolchain( name = "aarch64_linux_cc_toolchain", exec_compatible_with = ["@platforms//os:linux"], @@ -178,6 +186,7 @@ cc_toolchain_config( ], cxx_flags = ["-std=c++0x"] + select({ "@envoy//bazel:msan_build": ["-stdlib=libc++"], + "@envoy//bazel:tsan_build": ["-stdlib=libc++"], "//conditions:default": [], }), dbg_compile_flags = ["-g"], @@ -190,19 +199,28 @@ cc_toolchain_config( "-lm", ] + select({ "@envoy//bazel:msan_build": ["-L/usr/lib/llvm-18/lib"], + "@envoy//bazel:tsan_build": ["-L/usr/lib/llvm-18/lib"], "//conditions:default": [], }), - # Envoy supplies instrumented libc++ for MSAN targets. Non-instrumented - # Bazel exec tools use the system libc++ runtime below. + # Envoy supplies instrumented libc++ for MSAN and TSAN targets. + # Non-instrumented Bazel exec tools use the system libc++ runtime below. link_libs = select({ ":msan_target_build": [ "-latomic", ], + ":tsan_target_build": [ + "-latomic", + ], "@envoy//bazel:msan_build": [ "-l:libc++.a", "-l:libc++abi.a", "-latomic", ], + "@envoy//bazel:tsan_build": [ + "-l:libc++.a", + "-l:libc++abi.a", + "-latomic", + ], "//conditions:default": [ "-l:libstdc++.a", "-latomic", diff --git a/tests/cilium_tls_tcp_integration_test.cc b/tests/cilium_tls_tcp_integration_test.cc index d3c42bd41..81e41b533 100644 --- a/tests/cilium_tls_tcp_integration_test.cc +++ b/tests/cilium_tls_tcp_integration_test.cc @@ -18,7 +18,6 @@ #include "envoy/network/address.h" #include "envoy/network/connection.h" #include "envoy/network/transport_socket.h" -#include "envoy/ssl/connection.h" #include "source/common/buffer/buffer_impl.h" #include "source/common/buffer/watermark_buffer.h" @@ -119,12 +118,23 @@ class CiliumTLSIntegrationTest : public CiliumTcpIntegrationTest { } AssertionResult - waitForTlsHandshake(const FakeRawConnection& connection, + waitForTlsHandshake(FakeRawConnection& connection, std::chrono::milliseconds timeout = TestUtility::DefaultTimeout) { Event::TestTimeSystem::RealTimeBound bound(timeout); while (true) { - const auto downstream_timing = connection.connection().streamInfo().downstreamTiming(); - if (downstream_timing.downstreamHandshakeComplete().has_value()) { + bool handshake_complete = false; + const auto state_result = connection.sharedConnection().executeOnDispatcher( + [&handshake_complete](Network::Connection& upstream_connection) { + handshake_complete = upstream_connection.streamInfo() + .downstreamTiming() + .downstreamHandshakeComplete() + .has_value(); + }); + if (!state_result) { + return state_result; + } + + if (handshake_complete) { return AssertionSuccess(); } @@ -134,13 +144,33 @@ class CiliumTLSIntegrationTest : public CiliumTcpIntegrationTest { timeSystem().advanceTimeWait(std::chrono::milliseconds(1)); if (timeout != std::chrono::milliseconds::zero() && !bound.withinBound()) { - const Ssl::ConnectionInfoConstSharedPtr ssl = connection.connection().ssl(); - return AssertionFailure() << "Timed out waiting for TLS handshake. ssl=" << (ssl != nullptr) - << " handshake_complete=" - << downstream_timing.downstreamHandshakeComplete().has_value() - << " tls_version=" << (ssl != nullptr ? ssl->tlsVersion() : "") - << " ciphersuite=" - << (ssl != nullptr ? ssl->ciphersuiteString() : ""); + bool has_ssl = false; + std::string tls_version; + std::string ciphersuite; + const auto diagnostic_result = connection.sharedConnection().executeOnDispatcher( + [&](Network::Connection& upstream_connection) { + handshake_complete = upstream_connection.streamInfo() + .downstreamTiming() + .downstreamHandshakeComplete() + .has_value(); + const auto ssl = upstream_connection.ssl(); + has_ssl = ssl != nullptr; + if (ssl != nullptr) { + tls_version = std::string(ssl->tlsVersion()); + ciphersuite = std::string(ssl->ciphersuiteString()); + } + }); + if (!diagnostic_result) { + return diagnostic_result; + } + if (handshake_complete) { + return AssertionSuccess(); + } + + return AssertionFailure() << "Timed out waiting for TLS handshake. ssl=" << has_ssl + << " handshake_complete=" << handshake_complete + << " tls_version=" << tls_version + << " ciphersuite=" << ciphersuite; } } } diff --git a/tests/uds_server.cc b/tests/uds_server.cc index 581100644..62ce5eb5c 100644 --- a/tests/uds_server.cc +++ b/tests/uds_server.cc @@ -69,19 +69,21 @@ void UDSServer::shutdownServerThread() { if (fd2 >= 0) { ::shutdown(fd2, SHUT_RD); - ::close(fd2); } if (fd >= 0) { ::shutdown(fd, SHUT_RD); - errno = 0; - ::close(fd); } if (thread_ != nullptr) { - ENVOY_LOG(trace, "Waiting on unix domain socket server to close: {}", - Envoy::errorDetails(errno)); + ENVOY_LOG(trace, "Waiting on unix domain socket server thread"); thread_->join(); thread_.reset(); } + if (fd2 >= 0 && ::close(fd2) == -1) { + ENVOY_LOG(warn, "Closing UDS client socket {} failed: {}", fd2, Envoy::errorDetails(errno)); + } + if (fd >= 0 && ::close(fd) == -1) { + ENVOY_LOG(warn, "Closing UDS server socket {} failed: {}", fd, Envoy::errorDetails(errno)); + } if (!addr_->pipe()->abstractNamespace()) { ::unlink(addr_->asString().c_str()); }