Unify end-to-end tests under pytest

AGENTS.md

@@ -11,7 +11,7 @@ with a sophisticated plugin system.
 **Key Technologies:**
 - Language: C++20
 - Build System: CMake (migrated from autotools in v10)
-- Testing: Catch2 (unit tests) + AuTest Python framework (end-to-end tests)
+- Testing: Catch2 (unit tests) + pytest (end-to-end tests, with an AuTest compatibility backend)
 - Protocols: TLS, HTTP/1.1, HTTP/2, HTTP/3 (via Quiche)
 
 ## Project Structure
@@ -80,61 +80,63 @@ Unit tests are built into executables. Find the test binary and run it directly:
 ./build/src/tscore/test_tscore
 ```
 
-### End-to-End Tests (AuTest)
+### End-to-End Tests (pytest)
 
-**Enable autests during configuration:**
+**Enable end-to-end tests during configuration:**
 ```bash
-cmake -B build -DENABLE_AUTEST=ON
+cmake -B build -DENABLE_E2ETEST=ON
 cmake --build build
 cmake --install build
 ```
 
-**Run all autests:**
+**Run all end-to-end tests:**
 ```bash
-cmake --build build -t autest
+cmake --build build -t e2etest
 ```
 
 **Run specific test(s):**
 ```bash
 cd build/tests
-./autest.sh --sandbox /tmp/sbcodex --clean=none -f <test_name_without_test_py>
+./e2etest.sh -f <test_name_without_extension>
 ```
 
 For example, to run `cache-auth.test.py`:
 ```bash
-./autest.sh --sandbox /tmp/sbcursor --clean=none -f cache-auth
+./e2etest.sh -f cache-auth
 ```
 
 To run multiple tests efficiently, pass the -j option.
 
 ```bash
 cd build/tests
-./autest.sh -j4 --sandbox /tmp/sbcodex --clean=none -f 'header_rewrite*'
+./e2etest.sh -j4 -f 'header_rewrite*'
 ```
 
 Most end-to-end test coverage is in `tests/gold_tests/`. The CI system uses the
-Docker image `ci.trafficserver.apache.org/ats/fedora:43` (Fedora version updated
-regularly).
+Docker image `ci.trafficserver.apache.org/ats/fedora:44` (Fedora version updated
+regularly). The source-tree `tests/e2etest.sh` defaults to this image. It runs
+directly instead when it detects that it is already inside a Fedora 44
+container. Use `--run-in-docker` or `--no-run-in-docker` to override.
 
-### Writing Autests
+### Writing End-to-End Tests
 
-**New tests should use the `Test.ATSReplayTest()` approach**, which references a
-`replay.yaml` file that describes the test configuration and traffic patterns
-using the Proxy Verifier format. This is simpler, more maintainable, and
-parseable by tools.
+**New tests should normally be direct pytest replay tests.** Name the Proxy
+Verifier replay `<scenario>.test.yaml`; the file's `e2etest` section describes
+DNS, server, client, and ATS setup, and pytest collects it without a companion
+`.test.py` wrapper. Run these with `cmake --build build -t e2etest-replay`.
 
-If `ATSReplayTest` is not a good fit (say, the test needs a custom client), then
+If a direct replay test is not a good fit (say, the test needs a custom client), then
 organize the test around a test class with member functions that configure any
 servers, the ATS process, and the client. See
 `tests/gold_tests/ats_probe/ats_probe.test.py` for an example of a test organized
 around a test class.
 
-In autests, launch Python helpers with `{sys.executable}` rather than a
+In compatibility tests, launch Python helpers with `{sys.executable}` rather than a
 hardcoded `python3`, so the test runs under the same interpreter the harness
 uses.
 
-**For complete details on writing autests, see:**
-- `doc/developer-guide/testing/autests.en.rst` - Comprehensive guide to autest
+**For complete details on writing end-to-end tests, see:**
+- `doc/developer-guide/testing/e2e-tests.en.rst` - Comprehensive end-to-end test guide
 - Proxy Verifier format: https://github.com/yahoo/proxy-verifier
 - AuTest framework: https://autestsuite.bitbucket.io/
 

CMakeLists.txt

@@ -161,8 +161,18 @@ option(ENABLE_DISK_FAILURE_TESTS "Build disk failure tests (enables AIO fault in
 if(ENABLE_DISK_FAILURE_TESTS)
   add_compile_definitions("AIO_FAULT_INJECTION")
 endif()
-option(ENABLE_AUTEST "Setup autest (default OFF)")
-option(ENABLE_AUTEST_UDS "Setup autest with curl using UDS (default OFF)")
+option(ENABLE_E2ETEST "Set up end-to-end tests (default OFF)")
+option(ENABLE_E2ETEST_UDS "Set up end-to-end tests with curl using UDS (default OFF)")
+option(ENABLE_AUTEST "Deprecated alias for ENABLE_E2ETEST" OFF)
+option(ENABLE_AUTEST_UDS "Deprecated alias for ENABLE_E2ETEST_UDS" OFF)
+if(ENABLE_AUTEST)
+  message(DEPRECATION "ENABLE_AUTEST is deprecated; use ENABLE_E2ETEST")
+  set(ENABLE_E2ETEST ON)
+endif()
+if(ENABLE_AUTEST_UDS)
+  message(DEPRECATION "ENABLE_AUTEST_UDS is deprecated; use ENABLE_E2ETEST_UDS")
+  set(ENABLE_E2ETEST_UDS ON)
+endif()
 option(ENABLE_BENCHMARKS "Build benchmarks (default OFF)")
 option(EXTERNAL_YAML_CPP "Use external yaml-cpp (default OFF)")
 option(EXTERNAL_LIBSWOC "Use external libswoc (default OFF)")
@@ -772,7 +782,7 @@ check_struct_has_member("struct mptcp_info" mptcpi_subflows "linux/mptcp.h" HAVE
 # find resolv library if available
 find_package(resolv)
 
-if(ENABLE_DOCS OR ENABLE_AUTEST)
+if(ENABLE_DOCS OR ENABLE_E2ETEST)
   find_package(Python3 REQUIRED)
   find_program(UV uv REQUIRED)
   find_program(NETCAT_PROGRAM nc REQUIRED)
@@ -788,27 +798,34 @@ if(ENABLE_DOCS)
   find_program(GRAPHVIZ_DOT dot REQUIRED)
 endif()
 
-if(ENABLE_AUTEST)
+if(ENABLE_E2ETEST)
   # Default the sandbox to /tmp to keep paths short. Unix domain socket paths
   # are limited to 108 characters and deep build directories (e.g. in home
   # directories) can exceed this limit, causing confusing test failures. A hash
   # of CMAKE_BINARY_DIR provides per-build isolation while keeping the path
   # deterministic across runs.
   string(MD5 _build_dir_hash "${CMAKE_BINARY_DIR}")
   string(SUBSTRING "${_build_dir_hash}" 0 8 _build_dir_hash)
-  set(AUTEST_SANDBOX
-      "/tmp/sb_${_build_dir_hash}"
-      CACHE STRING "Location for autest output (default /tmp/sb_<hash>)"
+  set(E2ETEST_SANDBOX
+      "/tmp/e2e_${_build_dir_hash}"
+      CACHE STRING "Location for end-to-end test output (default /tmp/e2e_<hash>)"
   )
-  set(AUTEST_OPTIONS
+  if(AUTEST_SANDBOX)
+    message(DEPRECATION "AUTEST_SANDBOX is deprecated; use E2ETEST_SANDBOX")
+    set(E2ETEST_SANDBOX "${AUTEST_SANDBOX}")
+  endif()
+  set(E2ETEST_OPTIONS
       ""
-      CACHE STRING "Additional options for autest (default \"\")"
+      CACHE STRING "Additional options for end-to-end tests (default \"\")"
   )
-  # Also create AUTEST_OPTIONS_LIST from the string for use in cmake targets.
-  # This prevents cmake from escaping spaces in the arguments, which confuses
-  # the autest command. The original AUTEST_OPTIONS string is used in the
-  # autest.sh script.
-  separate_arguments(AUTEST_OPTIONS_LIST UNIX_COMMAND "${AUTEST_OPTIONS}")
+  if(AUTEST_OPTIONS)
+    message(DEPRECATION "AUTEST_OPTIONS is deprecated; use E2ETEST_OPTIONS")
+    string(PREPEND E2ETEST_OPTIONS "${AUTEST_OPTIONS} ")
+  endif()
+  if(PYTEST_OPTIONS)
+    message(DEPRECATION "PYTEST_OPTIONS is deprecated; use E2ETEST_OPTIONS")
+    string(APPEND E2ETEST_OPTIONS " ${PYTEST_OPTIONS}")
+  endif()
   file(READ "${CMAKE_SOURCE_DIR}/tests/proxy-verifier-version.txt" PROXY_VERIFIER_VERSION)
   string(STRIP "${PROXY_VERIFIER_VERSION}" PROXY_VERIFIER_VERSION)
   file(READ "${CMAKE_SOURCE_DIR}/tests/proxy-verifier-checksum.txt" PROXY_VERIFIER_SHA1)
@@ -928,7 +945,7 @@ add_subdirectory(src/traffic_via)
 if(ENABLE_CRIPTS)
   add_subdirectory(src/cripts)
 endif()
-if(ENABLE_AUTEST)
+if(ENABLE_E2ETEST)
   add_subdirectory(tests)
 endif()
 if(ENABLE_FUZZING)

CMakePresets.json

@@ -52,29 +52,41 @@
       }
     },
     {
-      "name": "autest",
+      "name": "e2etest",
       "inherits": ["default"],
-      "binaryDir": "${sourceDir}/build-autest",
+      "binaryDir": "${sourceDir}/build-e2etest",
       "cacheVariables": {
-        "ENABLE_AUTEST": "ON",
-        "CMAKE_INSTALL_PREFIX": "/tmp/ts-autest",
+        "ENABLE_E2ETEST": "ON",
+        "CMAKE_INSTALL_PREFIX": "/tmp/ts-e2etest",
         "BUILD_EXPERIMENTAL_PLUGINS": "ON",
         "ENABLE_WASM_WAMR": "OFF",
         "ENABLE_EXAMPLE": "ON"
       }
     },
     {
-      "name": "autest-uds",
+      "name": "e2etest-uds",
       "inherits": ["default"],
-      "binaryDir": "${sourceDir}/build-autest",
+      "binaryDir": "${sourceDir}/build-e2etest",
       "cacheVariables": {
-        "ENABLE_AUTEST": "ON",
-        "ENABLE_AUTEST_UDS": "ON",
-        "CMAKE_INSTALL_PREFIX": "/tmp/ts-autest",
+        "ENABLE_E2ETEST": "ON",
+        "ENABLE_E2ETEST_UDS": "ON",
+        "CMAKE_INSTALL_PREFIX": "/tmp/ts-e2etest",
         "BUILD_EXPERIMENTAL_PLUGINS": "ON",
         "ENABLE_EXAMPLE": "ON"
       }
     },
+    {
+      "name": "autest",
+      "displayName": "Deprecated alias for e2etest",
+      "description": "Compatibility preset; use e2etest",
+      "inherits": ["e2etest"]
+    },
+    {
+      "name": "autest-uds",
+      "displayName": "Deprecated alias for e2etest-uds",
+      "description": "Compatibility preset; use e2etest-uds",
+      "inherits": ["e2etest-uds"]
+    },
     {
       "name": "dev",
       "displayName": "development",
@@ -224,11 +236,17 @@
         "ENABLE_QUICHE": "ON"
       }
     },
+    {
+      "name": "ci-fedora-e2etest",
+      "displayName": "CI Fedora E2E Test",
+      "description": "CI Pipeline config for Fedora Linux (e2etest build)",
+      "inherits": ["ci-fedora", "e2etest"]
+    },
     {
       "name": "ci-fedora-autest",
-      "displayName": "CI Fedora Autest",
-      "description": "CI Pipeline config for Fedora Linux (autest build)",
-      "inherits": ["ci-fedora", "autest"]
+      "displayName": "Deprecated CI Fedora e2etest alias",
+      "description": "Compatibility preset for the current external Jenkins pipeline",
+      "inherits": ["ci-fedora-e2etest"]
     },
     {
       "name": "ci-freebsd",
@@ -337,7 +355,7 @@
       "displayName": "CI branch Quiche",
       "inherits": ["branch"],
       "cacheVariables": {
-        "ENABLE_AUTEST": "ON",
+        "ENABLE_E2ETEST": "ON",
         "nuraft_ROOT": "/opt/nuraft-boringssl",
         "OPENSSL_ROOT_DIR": "/opt/h3-tools-boringssl/boringssl",
         "quiche_ROOT": "/opt/h3-tools-boringssl/quiche",
@@ -349,7 +367,7 @@
       "displayName": "CI branch Quiche",
       "inherits": ["branch"],
       "cacheVariables": {
-        "ENABLE_AUTEST": "ON",
+        "ENABLE_E2ETEST": "ON",
         "nuraft_ROOT": "/opt",
         "OPENSSL_ROOT_DIR": "/opt/openssl-quic/",
         "quiche_ROOT": "/opt/quiche",
@@ -365,26 +383,38 @@
       }
     },
     {
-      "name": "branch-autest",
-      "displayName": "CI branch autest",
+      "name": "branch-e2etest",
+      "displayName": "CI branch e2etest",
       "inherits": ["branch"],
       "cacheVariables": {
-        "ENABLE_AUTEST": "ON",
+        "ENABLE_E2ETEST": "ON",
         "ENABLE_EXAMPLE": "ON",
         "ENABLE_CRIPTS": "ON"
       }
     },
     {
-      "name": "branch-autest-uds",
-      "displayName": "CI branch autest",
+      "name": "branch-e2etest-uds",
+      "displayName": "CI branch e2etest",
       "inherits": ["branch"],
       "cacheVariables": {
-        "ENABLE_AUTEST": "ON",
-        "ENABLE_AUTEST_UDS": "ON",
+        "ENABLE_E2ETEST": "ON",
+        "ENABLE_E2ETEST_UDS": "ON",
         "ENABLE_EXAMPLE": "ON",
         "ENABLE_CRIPTS": "ON"
       }
     },
+    {
+      "name": "branch-autest",
+      "displayName": "Deprecated CI branch e2etest alias",
+      "description": "Compatibility preset; use branch-e2etest",
+      "inherits": ["branch-e2etest"]
+    },
+    {
+      "name": "branch-autest-uds",
+      "displayName": "Deprecated CI branch e2etest UDS alias",
+      "description": "Compatibility preset; use branch-e2etest-uds",
+      "inherits": ["branch-e2etest-uds"]
+    },
     {
       "name": "branch-freebsd",
       "displayName": "CI branch freebsd",
@@ -441,18 +471,23 @@
       "name": "branch-coverage",
       "displayName": "CI branch coverage",
       "description": "Defaults for branch coverage builds",
-      "inherits": ["branch-autest"],
+      "inherits": ["branch-e2etest"],
       "cacheVariables": {
         "CMAKE_CXX_FLAGS_DEBUG": "--coverage",
         "CMAKE_C_FLAGS_DEBUG": "--coverage"
       }
     }
   ],
   "buildPresets": [
+    {
+      "name": "e2etest",
+      "configurePreset": "e2etest",
+      "targets": ["e2etest"]
+    },
     {
       "name": "autest",
-      "configurePreset": "autest",
-      "targets": ["autest"]
+      "configurePreset": "e2etest",
+      "targets": ["e2etest"]
     }
   ]
 }