From 6a142f178659817f898af0249d3d38bae34a1355 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 03:30:58 +0200 Subject: [PATCH 01/12] Decode LatestConfigsRequest for direct RC polling on /api/v0.1/configurations Agentless remote config has no agent to relay client state via /v0.7/config: the native RC client polls the backend endpoint directly and reports its per-config apply state inline on that same request. Add the protobuf message definitions and wire them into the proxy's request deserializer so that state is visible to the test harness. --- utils/proxy/_decoders/protobuf_schemas.py | 1 + utils/proxy/_decoders/remoteconfig.descriptor | 22 ++++++++++++-- utils/proxy/_decoders/remoteconfig.proto | 29 +++++++++++++++++++ utils/proxy/_deserializer.py | 17 ++++++++++- 4 files changed, 66 insertions(+), 3 deletions(-) diff --git a/utils/proxy/_decoders/protobuf_schemas.py b/utils/proxy/_decoders/protobuf_schemas.py index 56af04c97f0..0ae795ca6ab 100644 --- a/utils/proxy/_decoders/protobuf_schemas.py +++ b/utils/proxy/_decoders/protobuf_schemas.py @@ -35,3 +35,4 @@ def _get_mesages(filename: str) -> dict[str, type[message.Message]]: File = _remoteconfig_messages["datadog.config.File"] OrgDataResponse = _remoteconfig_messages["datadog.config.OrgDataResponse"] OrgStatusResponse = _remoteconfig_messages["datadog.config.OrgStatusResponse"] +LatestConfigsRequest = _remoteconfig_messages["datadog.config.LatestConfigsRequest"] diff --git a/utils/proxy/_decoders/remoteconfig.descriptor b/utils/proxy/_decoders/remoteconfig.descriptor index 450e54c0a8b..d90a8ad202e 100644 --- a/utils/proxy/_decoders/remoteconfig.descriptor +++ b/utils/proxy/_decoders/remoteconfig.descriptor @@ -1,5 +1,5 @@ -Ö +Ñ (utils/proxy/_decoders/remoteconfig.protodatadog.config"¬ ConfigMetas- roots ( 2.datadog.config.TopMetaRroots5 @@ -34,4 +34,22 @@ topTargetsI enabled (Renabled authorized (R -authorizedbproto3 \ No newline at end of file +authorized"U +LatestConfigsRequest= +active_clients ( 2.datadog.config.ClientR activeClients"; +Client1 +state ( 2.datadog.config.ClientStateRstate"Î + ClientState! + root_version (R rootVersion' +targets_version (RtargetsVersion@ + config_states ( 2.datadog.config.ConfigStateR configStates + has_error (RhasError +error ( Rerror"“ + ConfigState +id ( Rid +version (Rversion +product ( Rproduct + apply_state (R +applyState + apply_error ( R +applyErrorbproto3 \ No newline at end of file diff --git a/utils/proxy/_decoders/remoteconfig.proto b/utils/proxy/_decoders/remoteconfig.proto index 97f50ac05dc..394a18e2be0 100644 --- a/utils/proxy/_decoders/remoteconfig.proto +++ b/utils/proxy/_decoders/remoteconfig.proto @@ -58,3 +58,32 @@ message OrgStatusResponse { bool enabled = 1; bool authorized = 2; } + +// Agent/native-client request sent when polling /api/v0.1/configurations. Needed to read back +// the client's reported config-apply state for agentless scenarios, where there is no agent +// relaying that state via /v0.7/config: the native RC client polls this endpoint directly, and +// reports the same per-config apply state (ClientState.config_states) inline on the request. + +message LatestConfigsRequest { + repeated Client active_clients = 6; +} + +message Client { + ClientState state = 1; +} + +message ClientState { + uint64 root_version = 1; + uint64 targets_version = 2; + repeated ConfigState config_states = 3; + bool has_error = 4; + string error = 5; +} + +message ConfigState { + string id = 1; + uint64 version = 2; + string product = 3; + uint64 apply_state = 4; + string apply_error = 5; +} diff --git a/utils/proxy/_deserializer.py b/utils/proxy/_deserializer.py index 1d61a37bae4..7a2a96a8221 100644 --- a/utils/proxy/_deserializer.py +++ b/utils/proxy/_deserializer.py @@ -26,7 +26,13 @@ ExportLogsServiceRequest, ExportLogsServiceResponse, ) -from ._decoders.protobuf_schemas import MetricPayload, TracePayload, SketchPayload, BackendResponsePayload +from ._decoders.protobuf_schemas import ( + MetricPayload, + TracePayload, + SketchPayload, + BackendResponsePayload, + LatestConfigsRequest, +) from ._decoders.metrics_v3 import decode_metrics_v3 from .trace_bytes_decoding import decode_trace_bytes_ascii, unpack_trace_bytes_msgpack from .traces.trace_v1 import deserialize_v1_trace, _uncompress_agent_v1_trace, decode_appsec_s_value @@ -239,6 +245,15 @@ def json_load(): return MessageToDict(BackendResponsePayload.FromString(content)) if path == "/api/beta/sketches": return MessageToDict(SketchPayload.FromString(content)) + if path == "/api/v0.1/configurations" and key == "request": + # The agentless native RC client polls this endpoint directly (no agent relay), and + # reports its per-config apply state inline on the request, the same information an + # agent-relayed client reports via a subsequent POST to /v0.7/config. + return MessageToDict( + LatestConfigsRequest.FromString(content), + preserving_proto_field_name=True, + use_integers_for_enums=True, + ) if content_type == "application/x-www-form-urlencoded" and content == b"[]" and path == "/v0.4/traces": return [] From 90f8b6d9f7fc6df737c793d087444979f302f792 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 03:31:20 +0200 Subject: [PATCH 02/12] Fix TUF test keyid derivation and support agentless RC polling interval TUF derives a key's keyid from the canonical JSON of the key object, not from the raw public key bytes. libdatadog's rust-tuf recomputes the keyid this way when verifying the root, and previously failed to match our test signature (metadata root signature threshold not met). Fix the derivation and regenerate the two pinned fixtures that embed the old (wrong) keyid. Also: TUF metadata version 0 is spec-invalid, but was used as an empty-state sentinel for the mocked backend; real clients (correctly) reject it. Move the sentinel to version 1, and add an opt-in agent_refresh_interval field to targets metadata (default None/omitted, a no-op for all existing callers) so agentless scenarios can override the native RC client's 60s default poll interval down to something tests can wait on. --- tests/test_the_test/test_remote_config.py | 4 +- utils/proxy/rc_response_builder.py | 13 +++-- utils/proxy/tuf.py | 58 +++++++++++++++++------ 3 files changed, 54 insertions(+), 21 deletions(-) diff --git a/tests/test_the_test/test_remote_config.py b/tests/test_the_test/test_remote_config.py index c0ecf49e5fc..8cd4120bd94 100644 --- a/tests/test_the_test/test_remote_config.py +++ b/tests/test_the_test/test_remote_config.py @@ -4,7 +4,7 @@ @scenarios.test_the_test def test_debugger_command_none(): expected = { - "targets": "ewogICJzaWduYXR1cmVzIjogWwogICAgewogICAgICAia2V5aWQiOiAiMTM5ZTM5NDBlNjRiNTQ5MTcyMjA4OGQ5YTBkNzQxNjI4ZmM4MjZlMDk0NzVkMzQxYTc4MGFjZGUzYzRiODA3MCIsCiAgICAgICJzaWciOiAiNWIyNDJlMDg5MjI0ZWExMzg5MjU0ZGE4MGQxMWQ3MWM4MDNkMGMyMGE1NDg1NzgwMGE2OTM4OWRhZjJlMjQwZTcyNTQ0Mjk0MjAzZWEyZWFmMDdmZjIzNjMxMzJjOGYxYWFmZTg4MTY0MTAwNWIwYzYwNjgwM2M4MWQzMzBiMGQiCiAgICB9CiAgXSwKICAic2lnbmVkIjogewogICAgIl90eXBlIjogInRhcmdldHMiLAogICAgImN1c3RvbSI6IHsKICAgICAgIm9wYXF1ZV9iYWNrZW5kX3N0YXRlIjogImV5Sm1iMjhpT2lBaVltRnlJbjA9IgogICAgfSwKICAgICJleHBpcmVzIjogIjMwMDAtMDEtMDFUMDA6MDA6MDBaIiwKICAgICJzcGVjX3ZlcnNpb24iOiAiMS4wIiwKICAgICJ0YXJnZXRzIjoge30sCiAgICAidmVyc2lvbiI6IDAKICB9Cn0=", + "targets": "ewogICJzaWduYXR1cmVzIjogWwogICAgewogICAgICAia2V5aWQiOiAiMmQ1Y2VkZmRiNGQyOWY4Mjk4NTA0YzFiYzUyMDM5NDQ5ODcyMmYwODM1MzI2MTE5NDBhNDk4ZjUxNjMxODQ3NSIsCiAgICAgICJzaWciOiAiNWIyNDJlMDg5MjI0ZWExMzg5MjU0ZGE4MGQxMWQ3MWM4MDNkMGMyMGE1NDg1NzgwMGE2OTM4OWRhZjJlMjQwZTcyNTQ0Mjk0MjAzZWEyZWFmMDdmZjIzNjMxMzJjOGYxYWFmZTg4MTY0MTAwNWIwYzYwNjgwM2M4MWQzMzBiMGQiCiAgICB9CiAgXSwKICAic2lnbmVkIjogewogICAgIl90eXBlIjogInRhcmdldHMiLAogICAgImN1c3RvbSI6IHsKICAgICAgIm9wYXF1ZV9iYWNrZW5kX3N0YXRlIjogImV5Sm1iMjhpT2lBaVltRnlJbjA9IgogICAgfSwKICAgICJleHBpcmVzIjogIjMwMDAtMDEtMDFUMDA6MDA6MDBaIiwKICAgICJzcGVjX3ZlcnNpb24iOiAiMS4wIiwKICAgICJ0YXJnZXRzIjoge30sCiAgICAidmVyc2lvbiI6IDAKICB9Cn0=", "target_files": [], "client_configs": [], } @@ -29,7 +29,7 @@ def test_debugger_command_one_probe(): ] expected = { - "targets": "ewogICJzaWduYXR1cmVzIjogWwogICAgewogICAgICAia2V5aWQiOiAiMTM5ZTM5NDBlNjRiNTQ5MTcyMjA4OGQ5YTBkNzQxNjI4ZmM4MjZlMDk0NzVkMzQxYTc4MGFjZGUzYzRiODA3MCIsCiAgICAgICJzaWciOiAiZjk0NzliYTAyNDRkYjBlMDAxNjdiZjczNTE0NTQxMWZmOTk3MmU2NWI0Njc5NzllODZjNDRiZmNhZmQ2OGEyNjQ4YzcyOGVkMDEwOTZhNDg4YmQ3ZWJjYTMyZTUzMWNjODdiYjBkYWYxMDA2YWQxODRjNTQ4OTQyN2Q5Nzc4MDMiCiAgICB9CiAgXSwKICAic2lnbmVkIjogewogICAgIl90eXBlIjogInRhcmdldHMiLAogICAgImN1c3RvbSI6IHsKICAgICAgIm9wYXF1ZV9iYWNrZW5kX3N0YXRlIjogImV5Sm1iMjhpT2lBaVltRnlJbjA9IgogICAgfSwKICAgICJleHBpcmVzIjogIjMwMDAtMDEtMDFUMDA6MDA6MDBaIiwKICAgICJzcGVjX3ZlcnNpb24iOiAiMS4wIiwKICAgICJ0YXJnZXRzIjogewogICAgICAiZGF0YWRvZy8yL0xJVkVfREVCVUdHSU5HL2xvZ1Byb2JlX2xvZzE3MGFhLWFjZGEtNDQ1My05MTExLTE0NzhhNm1ldGhvZC9jb25maWciOiB7CiAgICAgICAgImN1c3RvbSI6IHsKICAgICAgICAgICJ2IjogMQogICAgICAgIH0sCiAgICAgICAgImhhc2hlcyI6IHsKICAgICAgICAgICJzaGEyNTYiOiAiZWNmMzQ3ZmIwZWE0NjE2ZmU1NTc2YzIyODNhYWY1NmUyNjFmYmNjZDMxNjJiYTIxZjNjZmQwNDJjM2VjOWFjNSIKICAgICAgICB9LAogICAgICAgICJsZW5ndGgiOiAyODkKICAgICAgfQogICAgfSwKICAgICJ2ZXJzaW9uIjogMQogIH0KfQ==", + "targets": "ewogICJzaWduYXR1cmVzIjogWwogICAgewogICAgICAia2V5aWQiOiAiMmQ1Y2VkZmRiNGQyOWY4Mjk4NTA0YzFiYzUyMDM5NDQ5ODcyMmYwODM1MzI2MTE5NDBhNDk4ZjUxNjMxODQ3NSIsCiAgICAgICJzaWciOiAiZjk0NzliYTAyNDRkYjBlMDAxNjdiZjczNTE0NTQxMWZmOTk3MmU2NWI0Njc5NzllODZjNDRiZmNhZmQ2OGEyNjQ4YzcyOGVkMDEwOTZhNDg4YmQ3ZWJjYTMyZTUzMWNjODdiYjBkYWYxMDA2YWQxODRjNTQ4OTQyN2Q5Nzc4MDMiCiAgICB9CiAgXSwKICAic2lnbmVkIjogewogICAgIl90eXBlIjogInRhcmdldHMiLAogICAgImN1c3RvbSI6IHsKICAgICAgIm9wYXF1ZV9iYWNrZW5kX3N0YXRlIjogImV5Sm1iMjhpT2lBaVltRnlJbjA9IgogICAgfSwKICAgICJleHBpcmVzIjogIjMwMDAtMDEtMDFUMDA6MDA6MDBaIiwKICAgICJzcGVjX3ZlcnNpb24iOiAiMS4wIiwKICAgICJ0YXJnZXRzIjogewogICAgICAiZGF0YWRvZy8yL0xJVkVfREVCVUdHSU5HL2xvZ1Byb2JlX2xvZzE3MGFhLWFjZGEtNDQ1My05MTExLTE0NzhhNm1ldGhvZC9jb25maWciOiB7CiAgICAgICAgImN1c3RvbSI6IHsKICAgICAgICAgICJ2IjogMQogICAgICAgIH0sCiAgICAgICAgImhhc2hlcyI6IHsKICAgICAgICAgICJzaGEyNTYiOiAiZWNmMzQ3ZmIwZWE0NjE2ZmU1NTc2YzIyODNhYWY1NmUyNjFmYmNjZDMxNjJiYTIxZjNjZmQwNDJjM2VjOWFjNSIKICAgICAgICB9LAogICAgICAgICJsZW5ndGgiOiAyODkKICAgICAgfQogICAgfSwKICAgICJ2ZXJzaW9uIjogMQogIH0KfQ==", "target_files": [ { "path": "datadog/2/LIVE_DEBUGGING/logProbe_log170aa-acda-4453-9111-1478a6method/config", diff --git a/utils/proxy/rc_response_builder.py b/utils/proxy/rc_response_builder.py index 58657d20201..ab06ca5edc9 100644 --- a/utils/proxy/rc_response_builder.py +++ b/utils/proxy/rc_response_builder.py @@ -44,7 +44,7 @@ def build_rc_configurations_protobuf(rc_state: dict | None) -> bytes: Args: rc_state: RC state dict with 'targets' (base64 signed JSON) and 'target_files' (list of {path, raw} dicts). If None, - returns empty targets at version 0. + returns empty targets at version 1. Returns: Protobuf-encoded LatestConfigsResponse bytes @@ -58,9 +58,14 @@ def build_rc_configurations_protobuf(rc_state: dict | None) -> bytes: targets_parsed = json.loads(targets_json) version = targets_parsed.get("signed", {}).get("version", 1) else: - # Use version 0 for empty state so version 1 updates are accepted - version = 0 - targets_content = build_targets_content(version) + # TUF requires metadata versions to start at 1 (0 is spec-invalid and real TUF/uptane + # clients reject it), so the "nothing pushed yet" placeholder uses version 1. Real + # pushes are shifted up by one accordingly -- see send_state() in _remote_config.py. + version = 1 + # agent_refresh_interval=1: without it, the agentless native RC client defaults to a + # 60s poll interval (see libdd-remote-config's AgentlessFetcher), far exceeding the + # timeouts tests wait on. + targets_content = build_targets_content(version, agent_refresh_interval=1) targets_json = json.dumps(sign_tuf_document(targets_content), separators=(",", ":")).encode() # Build snapshot referencing targets diff --git a/utils/proxy/tuf.py b/utils/proxy/tuf.py index 62012b7c915..cde699b1cd9 100644 --- a/utils/proxy/tuf.py +++ b/utils/proxy/tuf.py @@ -25,17 +25,35 @@ # seed = 32 zero bytes (0x00 * 32) # signing_key = ed25519.SigningKey(seed) # public_key = signing_key.verify_key -# keyid = sha256(public_key_bytes).hex() +# keyid = sha256(canonical_json(key_object)).hex() # # Resulting values: # public_key_hex = "3b6a27bcceb6a42d62a3a8d02a6f0d73653215771de243a63ac048a18b59da29" -# keyid = "139e3940e64b5491722088d9a0d741628fc826e09475d341a780acde3c4b8070" +# keyid = "2d5cedfdb4d29f8298504c1bc520394498722f083532611940a498f516318475" # TEST_TUF_SIGNING_KEY_SEED = bytes.fromhex("0" * 64) # 32 zero bytes as seed TEST_TUF_SIGNING_KEY = SigningKey(TEST_TUF_SIGNING_KEY_SEED) TEST_TUF_PUBLIC_KEY = TEST_TUF_SIGNING_KEY.verify_key TEST_TUF_PUBLIC_KEY_HEX = TEST_TUF_PUBLIC_KEY.encode().hex() -TEST_TUF_KEYID = hashlib.sha256(TEST_TUF_PUBLIC_KEY.encode()).hexdigest() + +TEST_TUF_KEY = { + "keyid_hash_algorithms": ["sha256"], + "keytype": "ed25519", + "keyval": {"public": TEST_TUF_PUBLIC_KEY_HEX}, + "scheme": "ed25519", +} + + +def _canonical_json(content: dict) -> bytes: + """TUF canonical JSON: sorted keys, no whitespace.""" + return json.dumps(content, separators=(",", ":"), sort_keys=True).encode() + + +# TUF derives the keyid from the canonical JSON of the key object, not from the raw public key. +# The Datadog agent accepts whichever keyid the document declares, but tracers that verify the +# root themselves (libdatadog, via rust-tuf) recompute it and then match no signature, failing +# with "metadata root signature threshold not met". +TEST_TUF_KEYID = hashlib.sha256(_canonical_json(TEST_TUF_KEY)).hexdigest() def sign_tuf_document(signed_content: dict) -> dict: @@ -50,8 +68,7 @@ def sign_tuf_document(signed_content: dict) -> dict: Complete TUF document with {signatures: [...], signed: {...}} """ - canonical_json = json.dumps(signed_content, separators=(",", ":"), sort_keys=True).encode() - signature = TEST_TUF_SIGNING_KEY.sign(canonical_json).signature.hex() + signature = TEST_TUF_SIGNING_KEY.sign(_canonical_json(signed_content)).signature.hex() return { "signatures": [{"keyid": TEST_TUF_KEYID, "sig": signature}], @@ -69,14 +86,7 @@ def _build_tuf_root() -> dict: "_type": "root", "consistent_snapshot": True, "expires": "3000-01-01T00:00:00Z", # Far future expiry for testing - "keys": { - TEST_TUF_KEYID: { - "keyid_hash_algorithms": ["sha256"], - "keytype": "ed25519", - "keyval": {"public": TEST_TUF_PUBLIC_KEY_HEX}, - "scheme": "ed25519", - } - }, + "keys": {TEST_TUF_KEYID: TEST_TUF_KEY}, "roles": { "root": {"keyids": [TEST_TUF_KEYID], "threshold": 1}, "snapshot": {"keyids": [TEST_TUF_KEYID], "threshold": 1}, @@ -115,12 +125,21 @@ def get_tuf_root_json() -> str: DEFAULT_OPAQUE_BACKEND_STATE = "eyJmb28iOiAiYmFyIn0=" # base64('{"foo": "bar"}') DEFAULT_EXPIRES = "3000-01-01T00:00:00Z" +# The agentless native RC client (libdd-remote-config's AgentlessFetcher) defaults to a 60s +# poll interval unless the targets doc's custom.agent_refresh_interval says otherwise (clamped +# to [1, 60]s). Without this, tests that push config via the backend and then wait for the +# client's next poll would need to wait up to 60s. This field is ignored by the agent-mediated +# (target="tracer") flow, but it's still kept opt-in (None omits it) to avoid perturbing the +# exact pinned byte content of existing tracer-target callers/tests. +DEFAULT_AGENT_REFRESH_INTERVAL = None + def build_targets_content( version: int, targets: dict | None = None, opaque_backend_state: str = DEFAULT_OPAQUE_BACKEND_STATE, expires: str = DEFAULT_EXPIRES, + agent_refresh_interval: int | None = DEFAULT_AGENT_REFRESH_INTERVAL, ) -> dict: """Build the 'signed' content for TUF targets metadata. @@ -129,14 +148,20 @@ def build_targets_content( targets: Dict mapping target paths to their metadata (hashes, length, custom) opaque_backend_state: Base64-encoded backend state string expires: Expiration timestamp + agent_refresh_interval: Recommended poll interval in seconds for agentless clients. + Omitted from the 'custom' field entirely when None. Returns: The 'signed' portion of targets metadata (not yet wrapped with signatures) """ + custom = {"opaque_backend_state": opaque_backend_state} + if agent_refresh_interval is not None: + custom["agent_refresh_interval"] = agent_refresh_interval + return { "_type": "targets", - "custom": {"opaque_backend_state": opaque_backend_state}, + "custom": custom, "expires": expires, "spec_version": "1.0", "targets": targets if targets is not None else {}, @@ -149,6 +174,7 @@ def build_signed_targets( targets: dict | None = None, opaque_backend_state: str = DEFAULT_OPAQUE_BACKEND_STATE, expires: str = DEFAULT_EXPIRES, + agent_refresh_interval: int | None = DEFAULT_AGENT_REFRESH_INTERVAL, ) -> dict: """Build a complete signed TUF targets document. @@ -157,12 +183,14 @@ def build_signed_targets( targets: Dict mapping target paths to their metadata opaque_backend_state: Base64-encoded backend state string expires: Expiration timestamp + agent_refresh_interval: Recommended poll interval in seconds for agentless clients. + Omitted from the 'custom' field entirely when None. Returns: Complete signed TUF targets document with signatures """ - content = build_targets_content(version, targets, opaque_backend_state, expires) + content = build_targets_content(version, targets, opaque_backend_state, expires, agent_refresh_interval) return sign_tuf_document(content) From 113f91873080cb867f9db681d81889a8afe50adf Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 03:31:33 +0200 Subject: [PATCH 03/12] Make send_state()/RC test helpers agentless-aware When context.scenario.include_agent is False, there is no agent to relay client state via /v0.7/config: watch /api/v0.1/configurations directly (via interfaces.datadog_direct) and read the client's reported state out of the request instead of a follow-up response, normalizing protobuf's stringified uint64 fields back to int along the way. Also re-sign real backend config pushes at version + 1 and set agent_refresh_interval=1, to stay strictly greater than the version-1 empty-state sentinel and keep the agentless client's poll interval fast enough for tests to wait on (see previous commit). --- utils/_remote_config.py | 60 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 7 deletions(-) diff --git a/utils/_remote_config.py b/utils/_remote_config.py index 35a0597b10b..5388505d7f0 100644 --- a/utils/_remote_config.py +++ b/utils/_remote_config.py @@ -15,14 +15,14 @@ from utils._context.core import context from utils.dd_constants import RemoteConfigApplyState as ApplyState -from utils.interfaces import library +from utils.interfaces import library, datadog_direct from utils._logger import logger from utils.proxy.mocked_response import ( StaticJsonMockedTracerResponse, SequentialRemoteConfigJsonMockedTracerResponse, MockedBackendResponse, ) -from utils.proxy.tuf import build_signed_targets +from utils.proxy.tuf import build_signed_targets, sign_tuf_document from utils.proxy.rc_response_builder import build_rc_configurations_protobuf RemoteConfigTarget = Literal["tracer", "backend"] @@ -46,6 +46,25 @@ def from_json(d: dict) -> "RemoteConfigStateResults": return RemoteConfigStateResults(version=d["version"], state=d["state"], configs=d["configs"]) +def _normalize_protobuf_client_state(state: dict) -> dict: + """Coerce uint64 fields back to int. + + protobuf's JSON mapping (MessageToDict) renders uint64/int64 fields as strings (they don't + fit precisely in a JS number), unlike the plain-JSON client state reported over /v0.7/config + where these are already native ints. Comparing a str against the int constants/versions used + below (e.g. `state["apply_state"] == ApplyState.UNKNOWN`) would silently always be False. + """ + return { + **state, + "targets_version": int(state["targets_version"]) if "targets_version" in state else 0, + "root_version": int(state["root_version"]) if "root_version" in state else 0, + "config_states": [ + {**cs, "version": int(cs["version"]), "apply_state": int(cs.get("apply_state", 0))} + for cs in state.get("config_states", []) + ], + } + + def send_state( raw_payload: dict, *, @@ -85,6 +104,20 @@ def send_state( if target == "backend": assert backend_enabled, f"Remote config backend is not enabled on {context.scenario}" + # The backend's "no config pushed yet" placeholder occupies TUF version 1 (see + # rc_response_builder.build_rc_configurations_protobuf) -- 0 isn't spec-valid and + # real TUF/uptane clients reject it. So every real push is shifted up by one here to + # stay strictly greater than that placeholder. + targets = json.loads(base64.b64decode(raw_payload["targets"])) + targets["signed"]["version"] += 1 + # Also keep the agentless client polling fast (see rc_response_builder.py for why), + # since real config pushes go through this same TUF targets document. + targets["signed"]["custom"]["agent_refresh_interval"] = 1 + raw_payload = {**raw_payload, "targets": _json_to_base64(sign_tuf_document(targets["signed"]))} + if state_version != -1: + # Keep state_version (the caller's own global-state counter, e.g. _RemoteConfigState.version) + # shifted in lockstep, so the empty-client_configs comparison below still lines up. + state_version += 1 # Build protobuf on test runner side, send bytes to proxy rc_protobuf = build_rc_configurations_protobuf(raw_payload) MockedBackendResponse(path="/api/v0.1/configurations", content=rc_protobuf).send() @@ -110,12 +143,24 @@ def send_state( state = {} + # Agentless scenarios have no agent to relay client state via /v0.7/config: the native RC + # client polls /api/v0.1/configurations directly and reports the same per-config apply state + # inline on that request (LatestConfigsRequest.active_clients[0].state), instead of via a + # separate follow-up request. + agentless = not context.scenario.include_agent + watched_path = "/api/v0.1/configurations" if agentless else "/v0.7/config" + def remote_config_applied(data: dict) -> bool: nonlocal state - if data["path"] != "/v0.7/config": + if data["path"] != watched_path: return False - state = data.get("request", {}).get("content", {}).get("client", {}).get("state", {}) + if agentless: + active_clients = data.get("request", {}).get("content", {}).get("active_clients", []) + state = active_clients[0].get("state", {}) if active_clients else {} + state = _normalize_protobuf_client_state(state) + else: + state = data.get("request", {}).get("content", {}).get("client", {}).get("state", {}) targets_version = state.get("targets_version") config_states = state.get("config_states", []) logger.info( @@ -123,12 +168,12 @@ def remote_config_applied(data: dict) -> bool: ) if len(client_configs) == 0: - found = state["targets_version"] == state_version and state.get("config_states", []) == [] + found = state.get("targets_version", 0) == state_version and state.get("config_states", []) == [] if found: current_states.state = ApplyState.ACKNOWLEDGED return found - if state["targets_version"] != version: + if state.get("targets_version", 0) != version: return False for state in config_states: @@ -150,7 +195,8 @@ def remote_config_applied(data: dict) -> bool: return True logger.info(f"Waiting for RC version={version}, client_configs={client_configs}") - rv = library.wait_for(remote_config_applied, timeout=30) + watched_interface = datadog_direct if agentless else library + rv = watched_interface.wait_for(remote_config_applied, timeout=30) if not rv: logger.error( f"RC timed out. Last known state: targets_version={state.get('targets_version')}, " From 9e0d9ff1ed559ba32f0bb45716dcbd72f7059cad Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 03:31:46 +0200 Subject: [PATCH 04/12] Generalize BaseDebuggerTest to support an agentless backend interface Replace hardcoded interfaces.agent/_LOGS_PATH/_DEBUGGER_PATH/etc. references throughout BaseDebuggerTest with overridable class attributes (_backend_interface, _snapshot_paths, _traces_path, _telemetry_path, _symbols_interface, _symbols_path), defaulting to today's agent-mode values so no existing test changes behavior. Add AgentlessBaseDebuggerTest, which points these at interfaces.datadog_direct and the single unified agentless debugger-intake path (libdatadog's agentless debugger sender collapses logs/snapshots/diagnostics/symdb onto one path, unlike the agent-relayed protocol). Span-decoration collection (_collect_span_decoration) calls get_spans_list(), which only interfaces.agent implements (it parses /v0.4/traces msgpack; agentless traces use a different wire format). Gate that call behind hasattr() so it's skipped for agentless backends rather than raising, per decision: span-decoration assertions stay agent-only for now. --- tests/debugger/utils.py | 109 ++++++++++++++++++++++++++++------------ 1 file changed, 76 insertions(+), 33 deletions(-) diff --git a/tests/debugger/utils.py b/tests/debugger/utils.py index a96898910db..921beacf805 100644 --- a/tests/debugger/utils.py +++ b/tests/debugger/utils.py @@ -24,6 +24,11 @@ _SYMBOLS_PATH = "/symdb/v1/input" _TELEMETRY_PATH = "/api/v2/apmtelemetry" +# Agentless (direct-to-intake) path: libdatadog's agentless debugger sender collapses logs, +# snapshots, diagnostics AND symdb onto this single path on the debugger-intake host -- the +# intake demultiplexes by payload, not path (datadog-live-debugger/src/sender.rs derive_endpoint_path). +_AGENTLESS_DEBUGGER_PATH = "/api/v2/debugger" + # Library paths _DEBUGGER_V2_INPUT_PATH = "/debugger/v2/input" @@ -171,6 +176,16 @@ class BaseDebuggerTest: use_debugger_endpoint: bool = False + # Backend interface/paths, overridden by AgentlessBaseDebuggerTest for scenarios without a + # Datadog Agent (data captured over interfaces.datadog_direct instead of interfaces.agent, + # and logs/snapshots/diagnostics/symdb all collapse onto _AGENTLESS_DEBUGGER_PATH). + _backend_interface = interfaces.agent + _snapshot_paths: tuple[str, ...] = (_LOGS_PATH, _DEBUGGER_PATH) + _traces_path: str = _TRACES_PATH + _telemetry_path: str = _TELEMETRY_PATH + _symbols_interface = interfaces.library + _symbols_path: str = _SYMBOLS_PATH + def initialize_weblog_remote_config(self) -> None: self.setup_failures = [] if self.get_tracer()["language"] in ["ruby"]: @@ -478,7 +493,7 @@ def wait_for_all_probes(self, statuses: list[ProbeStatus], timeout: int = 30) -> logger.debug("Wating for all probes") self._wait_successful = False found_ids: set[str] = set() - interfaces.agent.wait_for( + self._backend_interface.wait_for( lambda data: self._wait_for_all_probes(data, statuses=statuses, found_ids=found_ids), timeout=timeout ) return self._wait_successful @@ -516,7 +531,7 @@ def _check_all_probes_status(probe_diagnostics: ProbeDiagnosticsCollection, stat if log_number >= BaseDebuggerTest._last_read: BaseDebuggerTest._last_read = log_number - if data["path"] in [_DEBUGGER_PATH, _LOGS_PATH]: + if data["path"] in self._snapshot_paths: probe_diagnostics = self._process_diagnostics_data([data]) logger.debug(probe_diagnostics) @@ -540,13 +555,13 @@ def wait_for_snapshot_received(self, exception_message: str = "", timeout: int = self._snapshot_found = False - interfaces.agent.wait_for( + self._backend_interface.wait_for( lambda data: self._wait_for_snapshot_received(data, exception_snapshot=exception_snapshot), timeout=timeout ) return self._snapshot_found def _wait_for_snapshot_received(self, data: dict, *, exception_snapshot: bool = False): - if data["path"] in [_LOGS_PATH, _DEBUGGER_PATH]: + if data["path"] in self._snapshot_paths: if exception_snapshot: logger.debug("Reading " + data["log_filename"] + ", looking for '" + self._exception_message + "'") @@ -598,7 +613,7 @@ def wait_for_all_snapshots(self, exception_message: str = "", timeout: int = 30) logger.debug(f"Waiting for snapshot with exception message: {exception_message}") self._exception_message = exception_message self._snapshot_found = False - interfaces.agent.wait_for( + self._backend_interface.wait_for( lambda data: self._wait_for_all_snapshots(data, exception_snapshot=True), timeout=timeout ) return self._snapshot_found @@ -606,13 +621,13 @@ def wait_for_all_snapshots(self, exception_message: str = "", timeout: int = 30) logger.debug(f"Waiting for snapshots from all probes: {self.probe_ids}") self._all_snapshots_found = False self._found_probe_ids: set[str] = set() - interfaces.agent.wait_for( + self._backend_interface.wait_for( lambda data: self._wait_for_all_snapshots(data, exception_snapshot=False), timeout=timeout ) return self._all_snapshots_found def _wait_for_all_snapshots(self, data: dict, *, exception_snapshot: bool = False) -> bool: - if data["path"] not in [_LOGS_PATH, _DEBUGGER_PATH]: + if data["path"] not in self._snapshot_paths: return False if exception_snapshot: @@ -670,10 +685,12 @@ def wait_for_snapshot_count(self, count: int, timeout: int = 5) -> bool: """ logger.debug(f"Waiting for {count} snapshots from probes: {self.probe_ids}") seen: set[tuple[str, int]] = set() - return interfaces.agent.wait_for(lambda data: self._count_snapshots(data, seen=seen) >= count, timeout=timeout) + return self._backend_interface.wait_for( + lambda data: self._count_snapshots(data, seen=seen) >= count, timeout=timeout + ) def _count_snapshots(self, data: dict, seen: set[tuple[str, int]]) -> int: - if data["path"] in [_LOGS_PATH, _DEBUGGER_PATH]: + if data["path"] in self._snapshot_paths: contents = data["request"].get("content", []) or [] for index, content in enumerate(_iter_snapshot_content_items(contents)): @@ -689,11 +706,11 @@ def wait_for_no_capture_reason_span(self, error_message: str, timeout: int) -> b self._error_message = error_message self._no_capture_reason_span_found = False - interfaces.agent.wait_for(self._wait_for_no_capture_reason_span, timeout=timeout) + self._backend_interface.wait_for(self._wait_for_no_capture_reason_span, timeout=timeout) return self._no_capture_reason_span_found def _wait_for_no_capture_reason_span(self, data: dict): - if data["path"] == _TRACES_PATH: + if data["path"] == self._traces_path: logger.debug( "Reading " + data["log_filename"] @@ -701,7 +718,7 @@ def _wait_for_no_capture_reason_span(self, data: dict): + self._error_message + "'" ) - spans = interfaces.agent.get_spans_list() + spans = self._backend_interface.get_spans_list() for span in spans: meta = span.meta if "_dd.debug.error.no_capture_reason" in meta: @@ -720,16 +737,16 @@ def wait_for_code_origin_span(self, timeout: int = 5) -> bool: self._span_found = False threshold = self._get_max_trace_file_number() - interfaces.agent.wait_for( + self._backend_interface.wait_for( lambda data: self._wait_for_code_origin_span(data, threshold=threshold), timeout=timeout, ) return self._span_found def _get_max_trace_file_number(self) -> int: - """Get the maximum trace file number currently in the agent interface.""" + """Get the maximum trace file number currently in the backend interface.""" max_number = 0 - for data in interfaces.agent.get_data(_TRACES_PATH): + for data in self._backend_interface.get_data(self._traces_path): log_filename_found = re.search(r"/(\d+)__", data["log_filename"]) if log_filename_found: file_number = int(log_filename_found.group(1)) @@ -737,7 +754,7 @@ def _get_max_trace_file_number(self) -> int: return max_number def _wait_for_code_origin_span(self, data: dict, *, threshold: int) -> bool: - if data["path"] == _TRACES_PATH: + if data["path"] == self._traces_path: log_filename_found = re.search(r"/(\d+)__", data["log_filename"]) if not log_filename_found: return False @@ -762,13 +779,13 @@ def _wait_for_code_origin_span(self, data: dict, *, threshold: int) -> bool: def wait_for_telemetry(self, telemetry_type: str, timeout: int = 5) -> dict | None: self._telemetry: dict | None = None - interfaces.agent.wait_for( + self._backend_interface.wait_for( lambda data: self._wait_for_telemetry(data, telemetry_type=telemetry_type), timeout=timeout ) return self._telemetry def _wait_for_telemetry(self, data: dict, telemetry_type: str) -> bool: - if data["path"] != _TELEMETRY_PATH: + if data["path"] != self._telemetry_path: return False content = data.get("request", {}).get("content", {}) @@ -799,7 +816,10 @@ def collect(self) -> None: def _collect_probe_diagnostics(self): def _read_data(): - if context.library == "java": + if len(self._snapshot_paths) == 1: + # Agentless: diagnostics, logs and snapshots all collapse onto one path. + path = self._snapshot_paths[0] + elif context.library == "java": if context.library.version > "1.27.0": path = _DEBUGGER_PATH else: @@ -815,7 +835,7 @@ def _read_data(): path = _LOGS_PATH # TODO: Should the default not be _DEBUGGER_PATH? logger.debug(f"Reading data from {path}") - return list(interfaces.agent.get_data(path)) + return list(self._backend_interface.get_data(path)) all_data = _read_data() self.probe_diagnostics = self._process_diagnostics_data(all_data) @@ -868,11 +888,11 @@ def _get_snapshot_hash(): # Collect snapshots from both the logs and debugger endpoints for compatibility for when we switched # snapshots to the debugger endpoint. - if not self.use_debugger_endpoint: - agent_logs_endpoint_requests += list(interfaces.agent.get_data(_LOGS_PATH)) - agent_logs_endpoint_requests += list(interfaces.agent.get_data(_DEBUGGER_PATH)) + if len(self._snapshot_paths) == 1 or self.use_debugger_endpoint: + agent_logs_endpoint_requests += list(self._backend_interface.get_data(self._snapshot_paths[-1])) else: - agent_logs_endpoint_requests += list(interfaces.agent.get_data(_DEBUGGER_PATH)) + for path in self._snapshot_paths: + agent_logs_endpoint_requests += list(self._backend_interface.get_data(path)) snapshot_hash: dict = {} @@ -895,8 +915,9 @@ def _get_snapshot_hash(): def get_snapshot_request_lengths(self, probe_id: str) -> list[int]: """Return decoded body lengths for backend requests containing a probe snapshot.""" - requests = list(interfaces.agent.get_data(_LOGS_PATH)) - requests += list(interfaces.agent.get_data(_DEBUGGER_PATH)) + requests = [] + for path in dict.fromkeys(self._snapshot_paths): + requests += list(self._backend_interface.get_data(path)) lengths: list[int] = [] for request in requests: @@ -914,12 +935,14 @@ def get_snapshot_request_lengths(self, probe_id: str) -> list[int]: def wait_for_additional_snapshots(self, timeout: int = 5) -> bool: """Wait for another backend request containing a snapshot for an expected probe.""" existing_files = { - data["log_filename"] for path in (_LOGS_PATH, _DEBUGGER_PATH) for data in interfaces.agent.get_data(path) + data["log_filename"] + for path in dict.fromkeys(self._snapshot_paths) + for data in self._backend_interface.get_data(path) } expected_probe_ids = set(self.probe_ids) def _contains_additional_snapshot(data: dict[str, Any]) -> bool: - if data["path"] not in (_LOGS_PATH, _DEBUGGER_PATH) or data["log_filename"] in existing_files: + if data["path"] not in self._snapshot_paths or data["log_filename"] in existing_files: return False content = data["request"].get("content", []) or [] @@ -930,7 +953,7 @@ def _contains_additional_snapshot(data: dict[str, Any]) -> bool: return False - return interfaces.agent.wait_for(_contains_additional_snapshot, timeout=timeout) + return self._backend_interface.wait_for(_contains_additional_snapshot, timeout=timeout) def _debugger_v2_input_snapshots_received(self): """Test that the library sends snapshots to the debugger/v2/input endpoint""" @@ -954,7 +977,13 @@ def _get_spans_hash(): else: span_decoration_line_key = "_dd.di.spandecorationargsandlocals.probe_id" - spans_list = interfaces.agent.get_spans_list() + if not hasattr(self._backend_interface, "get_spans_list"): + # Agentless mode's backend interface (interfaces.datadog_direct) captures direct-to-intake + # traffic in a different wire format than the agent-relayed /v0.4/traces payloads + # get_spans_list() parses, so span-decoration collection is agent-only. + return {} + + spans_list = self._backend_interface.get_spans_list() for span in spans_list: self.all_spans.append(span) @@ -1025,10 +1054,10 @@ def _get_spans_hash(): def _collect_symbols(self): def _get_symbols(): result: list[dict] = [] - raw_data = list(interfaces.library.get_data(_SYMBOLS_PATH)) + raw_data = list(self._symbols_interface.get_data(self._symbols_path)) if len(raw_data) == 0: - logger.info(f"No request has been sent to {_SYMBOLS_PATH}") + logger.info(f"No request has been sent to {self._symbols_path}") return result for data in raw_data: @@ -1054,7 +1083,7 @@ def _collect_symdb_upload_events(self): parameter equaling "event". """ events: list[dict[str, Any]] = [] - raw_data = list(interfaces.library.get_data(_SYMBOLS_PATH)) + raw_data = list(self._symbols_interface.get_data(self._symbols_path)) for data in raw_data: if not isinstance(data, dict) or "request" not in data: continue @@ -1155,3 +1184,17 @@ def write_approval(self, data: list, test_name: str, suffix: str) -> None: def read_approval(self, test_name: str, suffix: str) -> dict: with open(self._get_path(test_name, suffix), "r", encoding="utf-8") as f: return json.load(f) + + +class AgentlessBaseDebuggerTest(BaseDebuggerTest): + """Base for debugger/SymDB tests run against DEBUGGER_AGENTLESS (no Datadog Agent). + + libdatadog's agentless debugger sender collapses logs, snapshots, diagnostics and SymDB + onto a single path on the debugger-intake host (see _AGENTLESS_DEBUGGER_PATH), captured + over the datadog_direct proxy interface instead of the agent. + """ + + _backend_interface = interfaces.datadog_direct + _snapshot_paths: tuple[str, ...] = (_AGENTLESS_DEBUGGER_PATH,) + _symbols_interface = interfaces.datadog_direct + _symbols_path: str = _AGENTLESS_DEBUGGER_PATH From 88ee24d067c43c2ecd87ea5fdded334208330540 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 03:32:03 +0200 Subject: [PATCH 05/12] Generalize AgentlessEndToEndScenario beyond FFE's mock backend AgentlessEndToEndScenario hardcoded FFE's own MockFFEAgentlessBackendServer into its base __init__/configure(). Replace that with a _create_mock_backend() hook (default None, meaning no mock backend), structurally typed via a small AgentlessBackendServer protocol, so other agentless scenarios can reuse this base without carrying FFE-specific setup. FeatureFlaggingAgentlessEndToEndScenario now overrides the hook and keeps its own env/extra_hosts wiring. Move the generic 'capture direct egress' wiring (DD_SITE=mock-intake.invalid, HTTPS_PROXY through the proxy container, mounting the mitmproxy CA bundle so the weblog trusts the intercepted TLS, watchdog/teardown registration for interfaces.datadog_direct) up from FeatureFlaggingAgentlessEndToEndScenario into the shared base, gated on a new capture_direct_egress constructor param (default True). Add pass-through rc_api_enabled/rc_backend_enabled params: when rc_backend_enabled is set, wire the same TUF test root env vars already used to make the real Datadog Agent trust the mocked RC backend (utils/_context/containers.py::AgentContainer), so the native agentless RC client trusts it too. Override _wait_for_app_readiness() to skip the interfaces.library.ready wait when capturing direct egress: that readiness signal only ever fires on agent-facing traffic, which agentless mode never sends, so it would time out on every run otherwise. Also fix a resource leak in _start_mock_backend(): the mock backend was only assigned to self._mock_backend after backend.reset() succeeded, so a failing reset() skipped cleanup in configure()'s except handler and leaked the backend. --- .../_context/_scenarios/agentless_endtoend.py | 167 ++++++++++++------ 1 file changed, 117 insertions(+), 50 deletions(-) diff --git a/utils/_context/_scenarios/agentless_endtoend.py b/utils/_context/_scenarios/agentless_endtoend.py index d461c21de35..895db2d1e78 100644 --- a/utils/_context/_scenarios/agentless_endtoend.py +++ b/utils/_context/_scenarios/agentless_endtoend.py @@ -1,24 +1,29 @@ import json from pathlib import Path -from typing import TYPE_CHECKING, Literal, cast +from typing import Literal, Protocol import pytest from utils import interfaces from utils._context.containers import ServerlessInitContainer, TestedContainer from utils.docker_fixtures._core import extra_hosts_for_environment -from utils.mocked_backend.ffe import ( - EXPECTED_API_KEY, - MockFFEAgentlessBackendServer, - MockFFEAgentlessBackendStatus, -) +from utils.mocked_backend.ffe import EXPECTED_API_KEY, MockFFEAgentlessBackendServer from utils.proxy.ports import ProxyPorts +from utils.proxy.tuf import get_tuf_root_json from .core import ScenarioGroup, scenario_groups as all_scenario_groups from .endtoend import DdTraceEndToEndScenario -if TYPE_CHECKING: - from utils.interfaces._core import ProxyBasedInterfaceValidator +# Reused as the mock DD_API_KEY for agentless scenarios that don't need FFE's own mock backend. +AGENTLESS_MOCK_API_KEY = EXPECTED_API_KEY + + +class AgentlessBackendServer(Protocol): + """Structural type for the mock backend servers plugged into `AgentlessEndToEndScenario`.""" + + def reset(self) -> None: ... + def status(self) -> object: ... + def close(self) -> None: ... class AgentlessEndToEndScenario(DdTraceEndToEndScenario): @@ -27,8 +32,8 @@ class AgentlessEndToEndScenario(DdTraceEndToEndScenario): _default_scenario_groups: tuple[ScenarioGroup, ...] = () # exclude those scenario from tracer_release _mock_backend_status_filename = "mock_agentless_backend_status.json" - _mock_backend: MockFFEAgentlessBackendServer | None = None - _last_mock_backend_status: MockFFEAgentlessBackendStatus | None = None + _mock_backend: AgentlessBackendServer | None = None + _last_mock_backend_status: object | None = None def __init__( self, @@ -38,18 +43,60 @@ def __init__( weblog_env: dict[str, str | None] | None = None, other_weblog_containers: tuple[type[TestedContainer], ...] = (), scenario_groups: tuple[ScenarioGroup, ...] = (), - use_proxy_for_weblog: bool = False, + capture_direct_egress: bool = True, + rc_api_enabled: bool = False, + rc_backend_enabled: bool = False, + library_interface_timeout: int = 0, ) -> None: + self.capture_direct_egress = capture_direct_egress + + environment: dict[str, str | None] = dict(weblog_env or {}) + if capture_direct_egress: + environment.setdefault( + # The reserved .invalid domain fails closed if a request bypasses the proxy. + "DD_SITE", + "mock-intake.invalid", + ) + environment.setdefault("DD_PROXY_HTTPS", f"http://proxy:{ProxyPorts.datadog_direct}") + environment.setdefault("HTTPS_PROXY", f"http://proxy:{ProxyPorts.datadog_direct}") + + weblog_volumes: dict | None = None + if capture_direct_egress: + # The weblog talks HTTPS directly to the proxy (CONNECT tunnel), which + # terminates TLS with the mitmproxy CA -- same trust anchor already + # mounted into AgentContainer (utils/_context/containers.py) for its + # own HTTP_PROXY-based backend traffic. + weblog_volumes = { + "./utils/build/docker/agent/ca-certificates.crt": { + "bind": "/etc/ssl/certs/ca-certificates.crt", + "mode": "ro", + }, + } + + if rc_backend_enabled: + # The native agentless RC client only trusts Datadog's real embedded + # production/staging/gov TUF roots by default, so it would never trust the + # mocked backend's test-signed response without this override -- mirrors the + # same env vars/root JSON already used to make the real Agent trust the test + # TUF key (utils/_context/containers.py::AgentContainer.__init__). + tuf_root_json = get_tuf_root_json() + environment.setdefault("DD_REMOTE_CONFIGURATION_ENABLED", "true") + environment.setdefault("DD_REMOTE_CONFIGURATION_CONFIG_ROOT", tuf_root_json) + environment.setdefault("DD_REMOTE_CONFIGURATION_DIRECTOR_ROOT", tuf_root_json) + super().__init__( name, doc=doc, include_agent=False, - library_interface_timeout=0, + library_interface_timeout=library_interface_timeout, other_weblog_containers=other_weblog_containers, scenario_groups=[*scenario_groups, all_scenario_groups.agentless], use_proxy_for_agent=False, - use_proxy_for_weblog=use_proxy_for_weblog, - weblog_env=weblog_env, + use_proxy_for_weblog=capture_direct_egress, + rc_api_enabled=rc_api_enabled, + rc_backend_enabled=rc_backend_enabled, + weblog_env=environment, + weblog_volumes=weblog_volumes, ) def configure(self, config: pytest.Config) -> None: @@ -61,24 +108,36 @@ def configure(self, config: pytest.Config) -> None: self._start_mock_backend() super().configure(config) + + if self.capture_direct_egress: + interfaces.datadog_direct.configure(self.host_log_folder, replay=self.replay) except BaseException: self._stop_mock_backend(persist_status=False) raise + def _wait_for_app_readiness(self) -> None: + if self.capture_direct_egress: + # Agentless mode never sends agent-facing traffic (/v0.4/traces, etc.), so + # interfaces.library.ready (set only by such traffic) would never fire here. + # The weblog container's own healthcheck already confirms it's up. + return + super()._wait_for_app_readiness() + + def _create_mock_backend(self) -> AgentlessBackendServer | None: + """Override in subclasses that need a mock agentless config/data backend.""" + return None + def _start_mock_backend(self) -> None: - assert self._mock_backend is None, "mock FFE agentless backend is already running" + assert self._mock_backend is None, "mock agentless backend is already running" - self._mock_backend = MockFFEAgentlessBackendServer() - self._mock_backend.reset() + backend = self._create_mock_backend() + if backend is None: + return - environment = self.weblog_infra.library_container.environment - environment |= { - "DD_API_KEY": EXPECTED_API_KEY, - "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL": self._mock_backend.library_config_url, - } - self.weblog_infra.library_container.extra_hosts = extra_hosts_for_environment(environment) + self._mock_backend = backend + backend.reset() - def mock_backend_status(self) -> MockFFEAgentlessBackendStatus | None: + def mock_backend_status(self) -> object | None: if self._mock_backend is not None: return self._mock_backend.status() return self._last_mock_backend_status @@ -88,10 +147,7 @@ def _mock_backend_status_path(self) -> Path: return Path(self.host_log_folder) / self._mock_backend_status_filename def _load_mock_backend_status(self) -> None: - self._last_mock_backend_status = cast( - "MockFFEAgentlessBackendStatus", - json.loads(self._mock_backend_status_path.read_text(encoding="utf-8")), - ) + self._last_mock_backend_status = json.loads(self._mock_backend_status_path.read_text(encoding="utf-8")) def _stop_mock_backend(self, *, persist_status: bool = True) -> None: backend = self._mock_backend @@ -116,6 +172,21 @@ def close_targets(self) -> None: finally: self._stop_mock_backend() + def _start_interfaces_watchdog(self) -> None: + super()._start_interfaces_watchdog() + if self.capture_direct_egress: + self.start_interfaces_watchdog([interfaces.datadog_direct]) + + def _wait_and_stop_containers(self, *, is_empty_test_run: bool) -> None: + super()._wait_and_stop_containers(is_empty_test_run=is_empty_test_run) + if not self.capture_direct_egress: + return + + if self.replay: + interfaces.datadog_direct.load_data_from_logs() + + interfaces.datadog_direct.check_deserialization_errors() + class FeatureFlaggingAgentlessEndToEndScenario(AgentlessEndToEndScenario): """FFE end-to-end scenario with UFC available before the weblog starts.""" @@ -142,14 +213,6 @@ def __init__( environment.update(weblog_env or {}) other_weblog_containers: tuple[type[TestedContainer], ...] = () - if exposure_egress is not None: - environment |= { - # The reserved .invalid domain fails closed if a request bypasses the proxy. - "DD_SITE": "mock-intake.invalid", - "DD_PROXY_HTTPS": f"http://proxy:{ProxyPorts.datadog_direct}", - "HTTPS_PROXY": f"http://proxy:{ProxyPorts.datadog_direct}", - } - if exposure_egress == "sidecar": serverless_init_port = str(ServerlessInitContainer.apm_receiver_port) environment |= { @@ -164,7 +227,7 @@ def __init__( doc=doc, other_weblog_containers=other_weblog_containers, scenario_groups=(all_scenario_groups.ffe,), - use_proxy_for_weblog=exposure_egress is not None, + capture_direct_egress=exposure_egress is not None, weblog_env=environment, ) @@ -174,12 +237,26 @@ def __init__( for env_name in ("DD_AGENT_HOST", "DD_DOGSTATSD_HOST", "DD_TRACE_AGENT_PORT", "DD_TRACE_AGENT_URL"): self.weblog_infra.library_container.environment.pop(env_name, None) + def _create_mock_backend(self) -> MockFFEAgentlessBackendServer: + return MockFFEAgentlessBackendServer() + + def _start_mock_backend(self) -> None: + super()._start_mock_backend() + backend = self._mock_backend + assert isinstance(backend, MockFFEAgentlessBackendServer) + + environment = self.weblog_infra.library_container.environment + environment |= { + "DD_API_KEY": EXPECTED_API_KEY, + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL": backend.library_config_url, + } + self.weblog_infra.library_container.extra_hosts = extra_hosts_for_environment(environment) + def configure(self, config: pytest.Config) -> None: try: super().configure(config) if self.exposure_egress is not None: interfaces.datadog_sidecar.configure(self.host_log_folder, replay=self.replay) - interfaces.datadog_direct.configure(self.host_log_folder, replay=self.replay) except BaseException: self._stop_mock_backend(persist_status=False) raise @@ -199,7 +276,7 @@ def _set_containers_dependancies(self) -> None: def _start_interfaces_watchdog(self) -> None: super()._start_interfaces_watchdog() if self.exposure_egress is not None: - self.start_interfaces_watchdog([interfaces.datadog_sidecar, interfaces.datadog_direct]) + self.start_interfaces_watchdog([interfaces.datadog_sidecar]) def _wait_for_app_readiness(self) -> None: if self.exposure_egress is not None: @@ -217,18 +294,8 @@ def _wait_and_stop_containers(self, *, is_empty_test_run: bool) -> None: return if self.replay: - self._load_telemetry_interfaces() + interfaces.datadog_sidecar.load_data_from_logs() elif self.exposure_egress == "sidecar": self.serverless_init_container.stop() interfaces.datadog_sidecar.check_deserialization_errors() - interfaces.datadog_direct.check_deserialization_errors() - - @staticmethod - def _load_telemetry_interfaces() -> None: - telemetry_interfaces: tuple[ProxyBasedInterfaceValidator, ...] = ( - interfaces.datadog_sidecar, - interfaces.datadog_direct, - ) - for interface in telemetry_interfaces: - interface.load_data_from_logs() From 9e406142951ee2eda072b3b6269869639204debc Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 03:32:16 +0200 Subject: [PATCH 06/12] Add APM_TRACING_AGENTLESS and DEBUGGER_AGENTLESS scenarios APM_TRACING_AGENTLESS (AgentlessEndToEndScenario) covers direct-to-intake trace submission, client-side stats, and Remote Configuration under DD_AGENTLESS_ENABLED, with no Datadog Agent. DEBUGGER_AGENTLESS (new DebuggerAgentlessScenario, in its own module) covers agentless Dynamic Instrumentation (probe upload/logs/snapshots) and Symbol DB the same way, reusing the TUF-trust and mocked RC backend wiring generalized in the previous commit. Both are excluded from the tracer_release scenario group, same as the other agentless scenarios; add them to test_group_rules.py's exclusion list. --- tests/test_the_test/test_group_rules.py | 2 ++ utils/_context/_scenarios/__init__.py | 33 ++++++++++++++++++- .../_context/_scenarios/debugger_agentless.py | 31 +++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 utils/_context/_scenarios/debugger_agentless.py diff --git a/tests/test_the_test/test_group_rules.py b/tests/test_the_test/test_group_rules.py index 5f0b8e686c3..fea4327e41c 100644 --- a/tests/test_the_test/test_group_rules.py +++ b/tests/test_the_test/test_group_rules.py @@ -42,6 +42,8 @@ def test_tracer_release(): scenarios.fuzzer, dormant_agentless_scenario, *agentless_exposure_scenarios, + scenarios.apm_tracing_agentless, + scenarios.debugger_agentless, scenarios.mock_the_test, scenarios.mock_the_test_2, scenarios.test_the_test, diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 9d598ff2601..6e4bffb2be9 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -7,7 +7,12 @@ from .aws_lambda import LambdaScenario from .core import Scenario, scenario_groups from .default import DefaultScenario -from .agentless_endtoend import FeatureFlaggingAgentlessEndToEndScenario +from .agentless_endtoend import ( + AGENTLESS_MOCK_API_KEY, + AgentlessEndToEndScenario, + FeatureFlaggingAgentlessEndToEndScenario, +) +from .debugger_agentless import DebuggerAgentlessScenario from .endtoend import ( DockerScenario, DdTraceEndToEndScenario, @@ -860,6 +865,32 @@ class _Scenarios: exposure_egress="sidecar", ) + apm_tracing_agentless = AgentlessEndToEndScenario( + "APM_TRACING_AGENTLESS", + doc="Validate direct-to-intake trace submission, client-side stats, and Remote " + "Configuration when DD_AGENTLESS_ENABLED is set, without a Datadog Agent.", + rc_api_enabled=True, + rc_backend_enabled=True, + weblog_env={ + "DD_AGENTLESS_ENABLED": "true", + "DD_API_KEY": AGENTLESS_MOCK_API_KEY, + "DD_TRACE_STATS_COMPUTATION_ENABLED": "true", + "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "1", + }, + ) + + debugger_agentless = DebuggerAgentlessScenario( + "DEBUGGER_AGENTLESS", + doc="Validate agentless Dynamic Instrumentation (probe upload/logs/snapshots) and " + "Symbol DB, without a Datadog Agent.", + weblog_env={ + "DD_AGENTLESS_ENABLED": "true", + "DD_API_KEY": AGENTLESS_MOCK_API_KEY, + "DD_DYNAMIC_INSTRUMENTATION_ENABLED": "true", + "_DD_SYMBOL_DATABASE_FORCE_UPLOAD": "true", + }, + ) + remote_config_mocked_backend_asm_features_nocache = DdTraceEndToEndScenario( "REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE", rc_api_enabled=True, diff --git a/utils/_context/_scenarios/debugger_agentless.py b/utils/_context/_scenarios/debugger_agentless.py new file mode 100644 index 00000000000..d2375a4688f --- /dev/null +++ b/utils/_context/_scenarios/debugger_agentless.py @@ -0,0 +1,31 @@ +from .agentless_endtoend import AgentlessEndToEndScenario +from .core import scenario_groups + + +class DebuggerAgentlessScenario(AgentlessEndToEndScenario): + """Agentless Dynamic Instrumentation (probes) and Symbol DB, without a Datadog Agent. + + Reuses the same TUF test root/key already trusted by the real Agent + (`utils._context.containers.AgentContainer`) for the native agentless Remote + Configuration client, and the same mocked-backend protobuf responses + (`rc_api_enabled`/`rc_backend_enabled`) already served to the Agent's own RC poller -- + both wired generically in `AgentlessEndToEndScenario` whenever `rc_backend_enabled=True`. + """ + + def __init__(self, name: str, *, doc: str, weblog_env: dict[str, str | None] | None = None) -> None: + base_weblog_env: dict[str, str | None] = { + "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "0.2", + "DD_DYNAMIC_INSTRUMENTATION_UPLOAD_INTERVAL_SECONDS": "0.1", + "DD_DYNAMIC_INSTRUMENTATION_UPLOAD_FLUSH_INTERVAL": "0.1", + } + base_weblog_env.update(weblog_env or {}) + + super().__init__( + name, + doc=doc, + rc_api_enabled=True, + rc_backend_enabled=True, + library_interface_timeout=5, + scenario_groups=(scenario_groups.debugger,), + weblog_env=base_weblog_env, + ) From 8d16c54458af65a5ecda11050b8be3e91db9d1d7 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 03:32:29 +0200 Subject: [PATCH 07/12] Add test coverage for DD_AGENTLESS_ENABLED (traces, stats, RC, debugger, SymDB) tests/test_agentless.py (APM_TRACING_AGENTLESS): trace submission, client-side stats, and Remote Configuration all land directly on the intake with no Datadog Agent, each asserted on host/path/headers, and RC additionally drives a real config push through to ACKNOWLEDGED via send_apm_tracing_command(). tests/debugger/test_debugger_agentless.py (DEBUGGER_AGENTLESS): a log probe installed via the agentless native RC client emits a snapshot to the agentless debugger intake; Symbol DB (forced via _DD_SYMBOL_DATABASE_FORCE_UPLOAD) uploads to the same unified path. Both verified end-to-end against a live weblog build of the (unmerged) dd-trace-py bob/agentless-setting branch. Endpoints/shapes are derived from reading tracer/libdatadog source, since that branch isn't released; re-confirm against real captures once it merges. --- tests/debugger/test_debugger_agentless.py | 91 ++++++++++++++++++ tests/test_agentless.py | 112 ++++++++++++++++++++++ 2 files changed, 203 insertions(+) create mode 100644 tests/debugger/test_debugger_agentless.py create mode 100644 tests/test_agentless.py diff --git a/tests/debugger/test_debugger_agentless.py b/tests/debugger/test_debugger_agentless.py new file mode 100644 index 00000000000..0da0be611bc --- /dev/null +++ b/tests/debugger/test_debugger_agentless.py @@ -0,0 +1,91 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2021 Datadog, Inc. + +"""Dynamic Instrumentation (probe upload/logs/snapshots) and Symbol DB, without a Datadog Agent. + +Endpoints/shapes here are derived from reading the dd-trace-py/libdatadog source on the +(unmerged) `bob/agentless-setting` branch, not from live-captured traffic - see +tests/debugger/utils.py::AgentlessBaseDebuggerTest and +utils/_context/_scenarios/debugger_agentless.py for the scenario/interface wiring. +""" + +import tests.debugger.utils as debugger +from utils import features, scenarios + + +@features.debugger +@scenarios.debugger_agentless +class Test_Agentless_Debugger_Probe_Snapshot(debugger.AgentlessBaseDebuggerTest): + """A log probe installed via the agentless native RC client emits a snapshot directly to + the agentless debugger intake, with no Datadog Agent involved. + """ + + def setup_log_method_snapshot(self): + self.initialize_weblog_remote_config() + + probes = debugger.read_probes("probe_snapshot_log_method") + for probe in probes: + probe["id"] = debugger.generate_probe_id("log") + self.set_probes(probes) + + self.send_rc_probes() + if not self.wait_for_all_probes(statuses=["INSTALLED"], timeout=60): + self.setup_failures.append("Probes did not reach INSTALLED status") + return + + self.send_weblog_request("/debugger/log") + self.wait_for_all_probes(statuses=["EMITTING"]) + if not self.wait_for_all_snapshots(timeout=60): + self.setup_failures.append("Snapshot was not received") + + def test_log_method_snapshot(self): + self.collect() + + self.assert_setup_ok() + self.assert_rc_state_not_error() + self.assert_all_probes_are_emitting() + + for probe_id in self.probe_ids: + assert probe_id in self.probe_snapshots, f"No snapshot was captured for probe {probe_id}" + assert len(self.probe_snapshots[probe_id]) > 0, f"No snapshot was captured for probe {probe_id}" + + path = self._snapshot_paths[0] + requests = list(self._backend_interface.get_data(path)) + assert len(requests) > 0, f"No request captured on {path}" + + request = requests[-1] + assert request["host"] == "debugger-intake.mock-intake.invalid" + headers = {name.lower(): value for name, value in request["request"]["headers"]} + assert "dd-api-key" in headers + + +@features.debugger_symdb +@scenarios.debugger_agentless +class Test_Agentless_SymbolDB(debugger.AgentlessBaseDebuggerTest): + """Symbol DB, forced via _DD_SYMBOL_DATABASE_FORCE_UPLOAD, uploads directly to the agentless + debugger intake (the same unified path as logs/snapshots/diagnostics in agentless mode). + """ + + def setup_symdb_upload(self): + self.initialize_weblog_remote_config() + + def test_symdb_upload(self): + self.collect() + self.assert_setup_ok() + + assert len(self.symbols) > 0, "No symbol files were found" + + errors = [] + for symbol in self.symbols: + error = symbol.get("system-tests-error") + if error is not None: + errors.append( + f"Error is: {error}, exported to file: {symbol.get('system-tests-file-path', 'No file path')}" + ) + assert not errors, "Found system-tests-errors:\n" + "\n".join(f"- {err}" for err in errors) + + requests = list(self._symbols_interface.get_data(self._symbols_path)) + assert len(requests) > 0, f"No request captured on {self._symbols_path}" + headers = {name.lower(): value for name, value in requests[-1]["request"]["headers"]} + assert "dd-api-key" in headers diff --git a/tests/test_agentless.py b/tests/test_agentless.py new file mode 100644 index 00000000000..424b8b24c03 --- /dev/null +++ b/tests/test_agentless.py @@ -0,0 +1,112 @@ +"""Direct-to-intake delivery (DD_AGENTLESS_ENABLED), without a Datadog Agent. + +Endpoints below are derived from reading the tracer/libdatadog source on the (unmerged) +`bob/agentless-setting` branch of dd-trace-py, not from live-captured traffic. Re-confirm +against real proxy captures once the branch is buildable in this environment; see +utils/_context/_scenarios/agentless_endtoend.py and debugger_agentless.py for the scenario setup. +""" + +from utils import features, interfaces, scenarios, weblog +from utils._context._scenarios.agentless_endtoend import AGENTLESS_MOCK_API_KEY +from utils._remote_config import send_apm_tracing_command +from utils.dd_constants import RemoteConfigApplyState as ApplyState + +TRACE_SUBMISSION_PATH = "/v1/input" +TRACE_SUBMISSION_HOST = "browser-intake-mock-intake.invalid" + +STATS_PATH = "/api/v0.2/stats" +STATS_HOST = "trace.agent.mock-intake.invalid" + +RC_CONFIGURATIONS_PATH = "/api/v0.1/configurations" +RC_HOST = "config.mock-intake.invalid" + + +def _headers(request: dict) -> dict[str, str]: + return {name.lower(): value for name, value in request["request"]["headers"]} + + +def _requests_at(host: str, path: str) -> list[dict]: + return [data for data in interfaces.datadog_direct.get_data(path) if data["host"] == host] + + +@scenarios.apm_tracing_agentless +@features.not_reported +class Test_Agentless_Trace_Submission: + """Traces are sent directly to the intake, bypassing the Datadog Agent.""" + + def setup_trace_submission(self): + self.r = weblog.get("/") + + def test_trace_submission(self): + assert self.r.status_code == 200 + + requests = _requests_at(TRACE_SUBMISSION_HOST, TRACE_SUBMISSION_PATH) + assert len(requests) != 0, f"No request captured on {TRACE_SUBMISSION_HOST}{TRACE_SUBMISSION_PATH}" + + request = requests[-1] + assert request["response"]["status_code"] // 100 == 2 + + headers = _headers(request) + assert headers["dd-api-key"] in {AGENTLESS_MOCK_API_KEY, "--redacted--"} + + content = request["request"]["content"] + assert content, "Trace submission request body is empty" + + +@scenarios.apm_tracing_agentless +@features.client_side_stats_supported +class Test_Agentless_Stats: + """Client-side trace stats are sent directly to the intake, on their own endpoint.""" + + def setup_stats(self): + self.r = weblog.get("/") + + def test_stats(self): + assert self.r.status_code == 200 + + stats_requests = _requests_at(STATS_HOST, STATS_PATH) + assert len(stats_requests) != 0, f"No request captured on {STATS_HOST}{STATS_PATH}" + + request = stats_requests[-1] + assert request["response"]["status_code"] // 100 == 2 + + headers = _headers(request) + assert headers["dd-api-key"] in {AGENTLESS_MOCK_API_KEY, "--redacted--"} + + # Stats and traces are distinct payloads on distinct hosts/paths. + trace_requests = _requests_at(TRACE_SUBMISSION_HOST, TRACE_SUBMISSION_PATH) + assert request not in trace_requests + + +@scenarios.apm_tracing_agentless +@features.remote_config_object_supported +class Test_Agentless_Remote_Config: + """The native agentless Remote Configuration client polls the intake directly. + + There is no agent to relay client state via `/v0.7/config`: the native client polls + `/api/v0.1/configurations` directly and reports its per-config apply state inline on that + same request (LatestConfigsRequest.active_clients[0].state) instead of via a separate + follow-up request. send_apm_tracing_command()/send_state() (utils/_remote_config.py) now + detect this agentless shape (keyed off `context.scenario.include_agent`), so this drives a + real config push and confirms application the same way agent-mode RC tests do. + """ + + def setup_remote_config_poll(self): + self.rc_state = send_apm_tracing_command(dynamic_instrumentation_enabled=True) + + def test_remote_config_poll(self): + assert self.rc_state.state == ApplyState.ACKNOWLEDGED, ( + f"RC config was not acknowledged: state={self.rc_state.state}, configs={self.rc_state.configs}" + ) + for config in self.rc_state.configs.values(): + assert config.get("apply_state") != ApplyState.ERROR, f"RC config apply error: {config}" + + requests = _requests_at(RC_HOST, RC_CONFIGURATIONS_PATH) + assert len(requests) != 0, f"No request captured on {RC_HOST}{RC_CONFIGURATIONS_PATH}" + + request = requests[-1] + assert request["method"] == "POST" + + headers = _headers(request) + assert headers["dd-api-key"] in {AGENTLESS_MOCK_API_KEY, "--redacted--"} + assert headers.get("content-type") == "application/x-protobuf" From ce7d1c5f171f7e2a26b13afabb7964135782bde9 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 03:32:40 +0200 Subject: [PATCH 08/12] Mark agentless test coverage as missing_feature for non-python tracers Scenario and test code is generic (no python-only guards), scoped to python for now via manifest entries only, so dropping these entries is enough once another tracer implements DD_AGENTLESS_ENABLED. --- manifests/cpp_httpd.yml | 1 + manifests/cpp_nginx.yml | 1 + manifests/dotnet.yml | 2 ++ manifests/golang.yml | 2 ++ manifests/java.yml | 2 ++ manifests/nodejs.yml | 2 ++ manifests/php.yml | 2 ++ manifests/ruby.yml | 2 ++ manifests/rust.yml | 1 + 9 files changed, 15 insertions(+) diff --git a/manifests/cpp_httpd.yml b/manifests/cpp_httpd.yml index 945af4d29fc..b1ecd8e2392 100644 --- a/manifests/cpp_httpd.yml +++ b/manifests/cpp_httpd.yml @@ -136,6 +136,7 @@ manifest: tests/stats/test_stats.py::Test_Time_Bucketing::test_client_side_stats: missing_feature # Created by easy win activation script tests/stats/test_stats.py::Test_Time_Bucketing::test_client_side_stats_bucket_alignment: missing_feature # Created by easy win activation script tests/stats/test_stats.py::Test_Transport_Headers: missing_feature # Created by easy win activation script + tests/test_agentless.py: missing_feature (C++ is not in scope for agentless mode (DD_AGENTLESS_ENABLED); python-only for now) tests/test_baggage.py::Test_Baggage_Headers_Api_Datadog: incomplete_test_app (/otel_drop_in_baggage_api_datadog endpoint is not implemented) tests/test_baggage.py::Test_Baggage_Headers_Api_OTel: incomplete_test_app (/otel_drop_in_baggage_api_otel endpoint is not implemented) tests/test_baggage.py::Test_Baggage_Headers_Basic: incomplete_test_app (/make_distant_call endpoint is not implemented) diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml index a4ab8849ea7..48afcfd4170 100644 --- a/manifests/cpp_nginx.yml +++ b/manifests/cpp_nginx.yml @@ -396,6 +396,7 @@ manifest: tests/stats/test_stats.py::Test_Time_Bucketing::test_client_side_stats: missing_feature # Created by easy win activation script tests/stats/test_stats.py::Test_Time_Bucketing::test_client_side_stats_bucket_alignment: missing_feature # Created by easy win activation script tests/stats/test_stats.py::Test_Transport_Headers: missing_feature # Created by easy win activation script + tests/test_agentless.py: missing_feature (C++ is not in scope for agentless mode (DD_AGENTLESS_ENABLED); python-only for now) tests/test_baggage.py::Test_Baggage_Headers_Api_Datadog: incomplete_test_app (/otel_drop_in_baggage_api_datadog endpoint is not implemented) tests/test_baggage.py::Test_Baggage_Headers_Api_OTel: incomplete_test_app (/otel_drop_in_baggage_api_otel endpoint is not implemented) tests/test_baggage.py::Test_Baggage_Headers_Basic: incomplete_test_app (/make_distant_call endpoint is not implemented) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 774815390e7..c7926bd101a 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -608,6 +608,7 @@ manifest: tests/auto_inject/test_auto_inject_install.py::TestSimpleInstallerAutoInjectManualAppsec: v3.36.0 tests/auto_inject/test_auto_inject_install.py::TestSimpleInstallerAutoInjectManualProfiling: bug (PROF-12209) tests/cws/test_thread_context_sharing.py::Test_ThreadContextSharing: missing_feature (missing /security/thread_context_sharing endpoint on weblog) + tests/debugger/test_debugger_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for .NET; python-only for now) tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Line_Capture_Expressions: missing_feature (Not yet implemented) tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Method_Capture_Expressions: missing_feature (Not yet implemented) tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Method_Capture_Expressions::test_complex_capture_expressions: missing_feature (Not yet implemented) @@ -1212,6 +1213,7 @@ manifest: - weblog_declaration: uds: '>=3.43.0' poc: '>=3.43.0' + tests/test_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for .NET; python-only for now) tests/test_baggage.py::Test_Baggage_Headers_Api_Datadog: v3.6.0 tests/test_baggage.py::Test_Baggage_Headers_Api_OTel: bug (APMAPI-1849) tests/test_baggage.py::Test_Baggage_Headers_Basic: v3.6.0 diff --git a/manifests/golang.yml b/manifests/golang.yml index 1041600d993..484024ff747 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1080,6 +1080,7 @@ manifest: tests/auto_inject/test_auto_inject_install.py::TestHostAutoInjectInstallScriptAppsec: v2.0.0 tests/auto_inject/test_auto_inject_install.py::TestSimpleInstallerAutoInjectManualAppsec: v2.0.0 tests/cws/test_thread_context_sharing.py::Test_ThreadContextSharing: missing_feature (missing /security/thread_context_sharing endpoint on weblog) + tests/debugger/test_debugger_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for Go; python-only for now) tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Line_Capture_Expressions: v2.2.3 tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Method_Capture_Expressions: v2.2.3 tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Method_Capture_Expressions::test_complex_capture_expressions: missing_feature (index expression not yet supported in Go system-probe) @@ -1629,6 +1630,7 @@ manifest: tests/stats/test_stats.py::Test_Client_Stats_With_Client_Obfuscation: v2.9.1 tests/stats/test_stats.py::Test_Client_Stats_With_Client_Obfuscation_Disabled: missing_feature tests/stats/test_stats.py::Test_Stats_Service_Source: irrelevant (Only implemented for Java) + tests/test_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for Go; python-only for now) tests/test_baggage.py::Test_Baggage_Headers_Api_Datadog: incomplete_test_app (/otel_drop_in_baggage_api_datadog endpoint is not implemented) tests/test_baggage.py::Test_Baggage_Headers_Api_OTel: incomplete_test_app (/otel_drop_in_baggage_api_otel endpoint is not implemented) tests/test_baggage.py::Test_Baggage_Headers_Basic: incomplete_test_app (/make_distant_call endpoint is not correctly implemented) diff --git a/manifests/java.yml b/manifests/java.yml index f56d208458c..8b516ff8e34 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -2942,6 +2942,7 @@ manifest: - declaration: bug (SCP-962) component_version: '>=1.5.0' tests/cws/test_thread_context_sharing.py::Test_ThreadContextSharing: missing_feature (missing /security/thread_context_sharing endpoint on weblog) + tests/debugger/test_debugger_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for Java; python-only for now) tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Line_Capture_Expressions: - weblog_declaration: "*": missing_feature @@ -4270,6 +4271,7 @@ manifest: "*": v0.0.0 spring-boot-3-native: irrelevant (/rasp/sqli endpoint is not available) tests/stats/test_stats.py::Test_Time_Bucketing::test_client_side_stats_bucket_alignment: missing_feature + tests/test_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for Java; python-only for now) tests/test_baggage.py::Test_Baggage_Headers_Api_Datadog: irrelevant (no Datadog API for W3C Baggage) tests/test_baggage.py::Test_Baggage_Headers_Api_OTel: - weblog_declaration: diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 0a412f144a5..e26f5c5c7de 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1650,6 +1650,7 @@ manifest: tests/auto_inject/test_auto_inject_install.py::TestHostAutoInjectInstallScriptAppsec: *ref_5_43_0 tests/auto_inject/test_auto_inject_install.py::TestSimpleInstallerAutoInjectManualAppsec: *ref_5_43_0 tests/cws/test_thread_context_sharing.py::Test_ThreadContextSharing: missing_feature (missing /security/thread_context_sharing endpoint on weblog) + tests/debugger/test_debugger_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for Node.js; python-only for now) tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Line_Capture_Expressions: - weblog_declaration: "*": irrelevant @@ -2732,6 +2733,7 @@ manifest: tests/stats/test_stats.py::Test_Time_Bucketing::test_client_side_stats_bucket_alignment: - component_version: ">=6.10.0" # may be before declaration: flaky (APMLP-1755) + tests/test_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for Node.js; python-only for now) tests/test_baggage.py::Test_Baggage_Headers_Api_Datadog: - weblog_declaration: "*": incomplete_test_app (endpoint not implemented) diff --git a/manifests/php.yml b/manifests/php.yml index 9d3215755be..196fb757a44 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -1055,6 +1055,7 @@ manifest: tests/auto_inject/test_auto_inject_install.py::TestSimpleInstallerAutoInjectManualAppsec: v1.9.0 tests/auto_inject/test_auto_inject_install.py::TestSimpleInstallerAutoInjectManualProfiling: v1.9.0 tests/cws/test_thread_context_sharing.py::Test_ThreadContextSharing: missing_feature (missing /security/thread_context_sharing endpoint on weblog) + tests/debugger/test_debugger_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for PHP; python-only for now) tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Line_Capture_Expressions: missing_feature (Not yet implemented) tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Method_Capture_Expressions: - weblog_declaration: @@ -1761,6 +1762,7 @@ manifest: tests/stats/test_stats.py::Test_Client_Stats_With_Client_Obfuscation_Disabled: missing_feature tests/stats/test_stats.py::Test_Error_Sampler: missing_feature (error traces dropped client-side under sampling; errors counted in stats but traces not sent) tests/stats/test_stats.py::Test_Stats_Service_Source: irrelevant (Only implemented for Java) + tests/test_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for PHP; python-only for now) tests/test_baggage.py::Test_Baggage_Headers_Api_Datadog: - weblog_declaration: laravel11x: incomplete_test_app diff --git a/manifests/ruby.yml b/manifests/ruby.yml index e36a3206854..81851b2096d 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -1787,6 +1787,7 @@ manifest: tests/auto_inject/test_auto_inject_install.py::TestHostAutoInjectInstallScriptAppsec: v2.19.0 tests/auto_inject/test_auto_inject_install.py::TestSimpleInstallerAutoInjectManualAppsec: v2.19.0 tests/cws/test_thread_context_sharing.py::Test_ThreadContextSharing: missing_feature (missing /security/thread_context_sharing endpoint on weblog) + tests/debugger/test_debugger_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for Ruby; python-only for now) tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Line_Capture_Expressions: - weblog_declaration: "*": irrelevant @@ -2779,6 +2780,7 @@ manifest: sinatra14: missing_feature rails52: missing_feature sinatra22: missing_feature + tests/test_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for Ruby; python-only for now) tests/test_baggage.py::Test_Baggage_Headers_Api_Datadog: - weblog_declaration: "*": incomplete_test_app (endpoint not implemented) diff --git a/manifests/rust.yml b/manifests/rust.yml index 8743419d232..deadd279ba4 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -421,6 +421,7 @@ manifest: tests/stats/test_stats.py: '>=0.4.0' tests/stats/test_stats.py::Test_Client_Stats::test_grpc_status_code: missing_feature (weblog does not implement grpc endpoint) tests/stats/test_stats.py::Test_Stats_Service_Source::test_srv_src: irrelevant (Only implemented for Java) + tests/test_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented in dd-trace-rs; python-only for now) tests/test_baggage.py::Test_Baggage_Headers_Api_Datadog: incomplete_test_app (/otel_drop_in_baggage_api_datadog endpoint not implemented) tests/test_baggage.py::Test_Baggage_Headers_Api_OTel: incomplete_test_app (/otel_drop_in_baggage_api_otel endpoint not implemented) tests/test_baggage.py::Test_Baggage_Headers_Max_Bytes: missing_feature (baggage maximum size is not configurable) From 5a847bc996730c275e4432741c6d539d5b1356ad Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 04:02:41 +0200 Subject: [PATCH 09/12] Strengthen agentless trace/stats tests with real payload assertions Test_Agentless_Trace_Submission and Test_Agentless_Stats only checked headers and non-empty content, not the actual shape of the trace/stats payloads. Add real assertions on captured payloads (root span for GET /: service, type, error, http.method, http.status_code; matching stats bucket entry: AgentHostname, ClientComputed, Service, Type, Hits, TopLevelHits, Errors), derived from and verified against live-captured traffic against the branch. Verified live: APM_TRACING_AGENTLESS still 3/3 passing with these assertions. --- tests/test_agentless.py | 53 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/tests/test_agentless.py b/tests/test_agentless.py index 424b8b24c03..0a3dd6052b3 100644 --- a/tests/test_agentless.py +++ b/tests/test_agentless.py @@ -1,8 +1,11 @@ """Direct-to-intake delivery (DD_AGENTLESS_ENABLED), without a Datadog Agent. -Endpoints below are derived from reading the tracer/libdatadog source on the (unmerged) -`bob/agentless-setting` branch of dd-trace-py, not from live-captured traffic. Re-confirm -against real proxy captures once the branch is buildable in this environment; see +Endpoints and payload shapes below were confirmed against a live weblog build of the +(unmerged) `bob/agentless-setting` branch of dd-trace-py: trace submission is plain JSON +(not the msgpack v0.4 format `interfaces.agent.get_spans_list()` parses), keyed by +`traces[].spans[]`, each span using the same field names as the agent-relayed format; +stats submission reuses the msgpack `ClientStatsPayload` shape byte-for-byte. Re-confirm +against real proxy captures if the branch's wire format changes before it merges; see utils/_context/_scenarios/agentless_endtoend.py and debugger_agentless.py for the scenario setup. """ @@ -20,6 +23,8 @@ RC_CONFIGURATIONS_PATH = "/api/v0.1/configurations" RC_HOST = "config.mock-intake.invalid" +ROOT_SPAN_RESOURCE = "GET /" + def _headers(request: dict) -> dict[str, str]: return {name.lower(): value for name, value in request["request"]["headers"]} @@ -29,6 +34,28 @@ def _requests_at(host: str, path: str) -> list[dict]: return [data for data in interfaces.datadog_direct.get_data(path) if data["host"] == host] +def _find_root_span(resource: str) -> dict | None: + """Search every captured trace-submission request for a root span with this resource.""" + for request in _requests_at(TRACE_SUBMISSION_HOST, TRACE_SUBMISSION_PATH): + for trace in request["request"]["content"].get("traces", []): + for span in trace.get("spans", []): + if span.get("parent_id") == "0000000000000000" and span.get("resource") == resource: + return span + return None + + +def _find_stats_entry(resource: str) -> dict | None: + """Search every captured stats request for a bucket entry with this resource.""" + for request in _requests_at(STATS_HOST, STATS_PATH): + content = request["request"]["content"] + for payload in content.get("Stats", []): + for bucket in payload.get("Stats", []): + for entry in bucket.get("Stats", []): + if entry.get("Resource") == resource: + return entry + return None + + @scenarios.apm_tracing_agentless @features.not_reported class Test_Agentless_Trace_Submission: @@ -52,6 +79,14 @@ def test_trace_submission(self): content = request["request"]["content"] assert content, "Trace submission request body is empty" + span = _find_root_span(ROOT_SPAN_RESOURCE) + assert span is not None, f"No root span with resource {ROOT_SPAN_RESOURCE!r} was captured" + assert span["service"] == "weblog" + assert span["type"] == "web" + assert span["error"] == 0 + assert span["meta"]["http.method"] == "GET" + assert span["meta"]["http.status_code"] == "200" + @scenarios.apm_tracing_agentless @features.client_side_stats_supported @@ -73,10 +108,22 @@ def test_stats(self): headers = _headers(request) assert headers["dd-api-key"] in {AGENTLESS_MOCK_API_KEY, "--redacted--"} + content = request["request"]["content"] + assert content["AgentHostname"] == "weblog" + assert content["ClientComputed"] is True + # Stats and traces are distinct payloads on distinct hosts/paths. trace_requests = _requests_at(TRACE_SUBMISSION_HOST, TRACE_SUBMISSION_PATH) assert request not in trace_requests + entry = _find_stats_entry(ROOT_SPAN_RESOURCE) + assert entry is not None, f"No stats entry with resource {ROOT_SPAN_RESOURCE!r} was captured" + assert entry["Service"] == "weblog" + assert entry["Type"] == "web" + assert entry["Hits"] >= 1 + assert entry["TopLevelHits"] >= 1 + assert entry["Errors"] == 0 + @scenarios.apm_tracing_agentless @features.remote_config_object_supported From a4c2cd8ada436f4db0d7fb053ad475ed778c61de Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 04:58:08 +0200 Subject: [PATCH 10/12] Meticulously assert required headers on agentless payloads Trace submission, stats, RC, debugger snapshot, and SymDB requests each now assert their full required header set (dd-api-key, content-type, user-agent, datadog-meta-* tracer identity, dd-evp-origin, etc.), not just dd-api-key/content-type as before. Caught a real discrepancy along the way: SymDB uploads share dd-evp-origin: agent-debugger with logs/snapshots/diagnostics rather than a distinct agent-symdb value, and don't send x-datadog-additional-tags at all - fixed the assertion to match confirmed live behavior. --- tests/debugger/test_debugger_agentless.py | 23 ++++++-- tests/test_agentless.py | 64 +++++++++++++++++++++-- 2 files changed, 78 insertions(+), 9 deletions(-) diff --git a/tests/debugger/test_debugger_agentless.py b/tests/debugger/test_debugger_agentless.py index 0da0be611bc..169be93295d 100644 --- a/tests/debugger/test_debugger_agentless.py +++ b/tests/debugger/test_debugger_agentless.py @@ -4,14 +4,17 @@ """Dynamic Instrumentation (probe upload/logs/snapshots) and Symbol DB, without a Datadog Agent. -Endpoints/shapes here are derived from reading the dd-trace-py/libdatadog source on the -(unmerged) `bob/agentless-setting` branch, not from live-captured traffic - see -tests/debugger/utils.py::AgentlessBaseDebuggerTest and +Endpoints and header shapes below were confirmed against a live weblog build of the +(unmerged) `bob/agentless-setting` branch of dd-trace-py: logs/snapshots, diagnostics, and +Symbol DB all share the same agentless debugger intake path and `dd-evp-origin: agent-debugger` +header - the native client doesn't distinguish sub-features via evp-origin the way agent-mode +does. See tests/debugger/utils.py::AgentlessBaseDebuggerTest and utils/_context/_scenarios/debugger_agentless.py for the scenario/interface wiring. """ import tests.debugger.utils as debugger from utils import features, scenarios +from utils._context._scenarios.agentless_endtoend import AGENTLESS_MOCK_API_KEY @features.debugger @@ -57,7 +60,11 @@ def test_log_method_snapshot(self): request = requests[-1] assert request["host"] == "debugger-intake.mock-intake.invalid" headers = {name.lower(): value for name, value in request["request"]["headers"]} - assert "dd-api-key" in headers + assert headers["dd-api-key"] in {AGENTLESS_MOCK_API_KEY, "--redacted--"} + assert headers["dd-evp-origin"] == "agent-debugger" + assert headers["content-type"].startswith("multipart/form-data") + assert headers["user-agent"].startswith("Tracer/") + assert "datadog-entity-id" in headers @features.debugger_symdb @@ -88,4 +95,10 @@ def test_symdb_upload(self): requests = list(self._symbols_interface.get_data(self._symbols_path)) assert len(requests) > 0, f"No request captured on {self._symbols_path}" headers = {name.lower(): value for name, value in requests[-1]["request"]["headers"]} - assert "dd-api-key" in headers + assert headers["dd-api-key"] in {AGENTLESS_MOCK_API_KEY, "--redacted--"} + # Symbol DB shares the same debugger intake path/evp-origin as logs/snapshots/diagnostics - + # the native client doesn't distinguish sub-features via dd-evp-origin. + assert headers["dd-evp-origin"] == "agent-debugger" + assert headers["content-type"].startswith("multipart/form-data") + assert headers["user-agent"].startswith("Tracer/") + assert "datadog-entity-id" in headers diff --git a/tests/test_agentless.py b/tests/test_agentless.py index 0a3dd6052b3..65fc8f3cfe9 100644 --- a/tests/test_agentless.py +++ b/tests/test_agentless.py @@ -30,6 +30,18 @@ def _headers(request: dict) -> dict[str, str]: return {name.lower(): value for name, value in request["request"]["headers"]} +def _assert_headers(headers: dict[str, str], *, exact: dict[str, str], present: tuple[str, ...]) -> None: + """Assert exact values for deterministic headers and mere presence for value-varying ones.""" + for name, value in exact.items(): + assert headers.get(name) == value, f"header {name!r}: expected {value!r}, got {headers.get(name)!r}" + for name in present: + assert name in headers, f"missing required header {name!r}" + + +def _assert_api_key(headers: dict[str, str]) -> None: + assert headers["dd-api-key"] in {AGENTLESS_MOCK_API_KEY, "--redacted--"} + + def _requests_at(host: str, path: str) -> list[dict]: return [data for data in interfaces.datadog_direct.get_data(path) if data["host"] == host] @@ -74,7 +86,25 @@ def test_trace_submission(self): assert request["response"]["status_code"] // 100 == 2 headers = _headers(request) - assert headers["dd-api-key"] in {AGENTLESS_MOCK_API_KEY, "--redacted--"} + _assert_api_key(headers) + _assert_headers( + headers, + exact={ + "content-type": "application/json", + "datadog-meta-lang": "python", + "datadog-meta-lang-interpreter": "CPython", + "datadog-client-computed-top-level": "true", + }, + present=( + "user-agent", + "datadog-meta-lang-version", + "datadog-meta-tracer-version", + "datadog-entity-id", + "x-datadog-trace-count", + "content-length", + ), + ) + assert headers["user-agent"].startswith("Tracer/") content = request["request"]["content"] assert content, "Trace submission request body is empty" @@ -106,7 +136,26 @@ def test_stats(self): assert request["response"]["status_code"] // 100 == 2 headers = _headers(request) - assert headers["dd-api-key"] in {AGENTLESS_MOCK_API_KEY, "--redacted--"} + _assert_api_key(headers) + _assert_headers( + headers, + exact={ + "content-type": "application/msgpack", + "datadog-meta-lang": "python", + "datadog-meta-lang-interpreter": "CPython", + }, + present=( + "user-agent", + "datadog-meta-lang-version", + "datadog-meta-tracer-version", + "datadog-entity-id", + "content-length", + ), + ) + assert headers["user-agent"].startswith("Tracer/") + # Stats has no top-level-computed/trace-count headers: those are trace-submission-only. + assert "datadog-client-computed-top-level" not in headers + assert "x-datadog-trace-count" not in headers content = request["request"]["content"] assert content["AgentHostname"] == "weblog" @@ -155,5 +204,12 @@ def test_remote_config_poll(self): assert request["method"] == "POST" headers = _headers(request) - assert headers["dd-api-key"] in {AGENTLESS_MOCK_API_KEY, "--redacted--"} - assert headers.get("content-type") == "application/x-protobuf" + _assert_api_key(headers) + _assert_headers( + headers, + exact={"content-type": "application/x-protobuf"}, + present=("user-agent", "datadog-entity-id", "content-length"), + ) + # The native RC client is driven by libdatadog directly, not the Python-level tracer, + # so it identifies itself distinctly (no datadog-meta-lang-* headers, unlike traces/stats). + assert headers["user-agent"].startswith("Libdatadog/") From db8b0e11febec6a6f79fd51b697cb057d20375e6 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 13:39:30 +0200 Subject: [PATCH 11/12] Verify zstd compression and multi-flush stats Sequence monotonicity Trace submission now asserts content-encoding: zstd and that the wire size is genuinely smaller than the decoded body, following an audit of the agentless Rust encoder against the real Agent that found compression had been silently disabled (now fixed on the dd-trace-py branch). New Test_Agentless_Stats_Multi_Flush polls for two same-runtime stats flushes and asserts ClientStatsPayload.Sequence increments by exactly 1 per flush - a guarantee agent-mode never gives a real signal for, since the Agent's own re-aggregation always resets Sequence to 0 on relay. --- tests/test_agentless.py | 60 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/test_agentless.py b/tests/test_agentless.py index 65fc8f3cfe9..8c6ec8b0343 100644 --- a/tests/test_agentless.py +++ b/tests/test_agentless.py @@ -9,6 +9,8 @@ utils/_context/_scenarios/agentless_endtoend.py and debugger_agentless.py for the scenario setup. """ +import time + from utils import features, interfaces, scenarios, weblog from utils._context._scenarios.agentless_endtoend import AGENTLESS_MOCK_API_KEY from utils._remote_config import send_apm_tracing_command @@ -68,6 +70,15 @@ def _find_stats_entry(resource: str) -> dict | None: return None +def _stats_runtime_id(request: dict) -> str | None: + payloads = request["request"]["content"].get("Stats", []) + return payloads[0]["RuntimeID"] if payloads else None + + +def _stats_requests_by_runtime(runtime_id: str) -> list[dict]: + return [r for r in _requests_at(STATS_HOST, STATS_PATH) if _stats_runtime_id(r) == runtime_id] + + @scenarios.apm_tracing_agentless @features.not_reported class Test_Agentless_Trace_Submission: @@ -91,6 +102,7 @@ def test_trace_submission(self): headers, exact={ "content-type": "application/json", + "content-encoding": "zstd", "datadog-meta-lang": "python", "datadog-meta-lang-interpreter": "CPython", "datadog-client-computed-top-level": "true", @@ -106,6 +118,16 @@ def test_trace_submission(self): ) assert headers["user-agent"].startswith("Tracer/") + # The proxy transparently decompresses the body for capture (request["request"]["length"] + # is the decompressed size); compare it against the real over-the-wire content-length + # header to confirm the payload was actually compressed, not just labeled as such. + wire_length = int(headers["content-length"]) + decoded_length = request["request"]["length"] + assert wire_length < decoded_length, ( + f"Trace submission body doesn't look compressed: {wire_length} wire bytes vs " + f"{decoded_length} decoded bytes" + ) + content = request["request"]["content"] assert content, "Trace submission request body is empty" @@ -174,6 +196,44 @@ def test_stats(self): assert entry["Errors"] == 0 +@scenarios.apm_tracing_agentless +@features.client_side_stats_supported +class Test_Agentless_Stats_Multi_Flush: + """Sequence increments by exactly 1 across successive flushes of the same runtime. + + Stats buckets are 10s wide; the agent-mode writer's own re-aggregation always resets + Sequence to 0 on every relayed payload (see pkg/trace/stats/client_stats_aggregator.go), + so this monotonic-Sequence guarantee is agentless-specific - the Agent never gave a + real signal here to compare against, which is exactly why it's easy to get wrong. + """ + + def setup_multi_flush_stats(self): + runtime_id = None + deadline = time.time() + 60 + while time.time() < deadline: + weblog.get("/") + requests = _requests_at(STATS_HOST, STATS_PATH) + if requests: + runtime_id = _stats_runtime_id(requests[-1]) + if runtime_id and len(_stats_requests_by_runtime(runtime_id)) >= 2: + break + time.sleep(2) + self.runtime_id = runtime_id + + def test_multi_flush_stats(self): + assert self.runtime_id, "No stats request was ever captured" + + requests = _stats_requests_by_runtime(self.runtime_id) + assert len(requests) >= 2, ( + f"Expected at least 2 stats flushes for runtime {self.runtime_id!r}, got {len(requests)}" + ) + + sequences = [r["request"]["content"]["Stats"][0]["Sequence"] for r in requests] + assert len(set(sequences)) == len(sequences), f"Sequence numbers are not unique: {sequences}" + for prev, cur in zip(sequences, sequences[1:]): + assert cur == prev + 1, f"Sequence should increment by exactly 1 per flush, got: {sequences}" + + @scenarios.apm_tracing_agentless @features.remote_config_object_supported class Test_Agentless_Remote_Config: From c0a013a4636a7b8142ab7c8d9164fed5f275b31e Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 20:28:28 +0200 Subject: [PATCH 12/12] Contrain python to 4.15.0+ --- manifests/python.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/manifests/python.yml b/manifests/python.yml index b0f25ddfc80..ba1a2309cdb 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -2304,6 +2304,7 @@ manifest: python3.12: '>=4.3.1' uds-flask: '>=4.3.1' uwsgi-poc: '>=4.3.1' + tests/test_agentless.py: v4.15.0 tests/test_baggage.py::Test_Baggage_Headers_Api_Datadog: - weblog_declaration: "*": incomplete_test_app (endpoint not implemented)