From 6a142f178659817f898af0249d3d38bae34a1355 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 03:30:58 +0200 Subject: [PATCH 01/18] Decode LatestConfigsRequest for direct RC polling on /api/v0.1/configurations Agentless remote config has no agent to relay client state via /v0.7/config: the native RC client polls the backend endpoint directly and reports its per-config apply state inline on that same request. Add the protobuf message definitions and wire them into the proxy's request deserializer so that state is visible to the test harness. --- utils/proxy/_decoders/protobuf_schemas.py | 1 + utils/proxy/_decoders/remoteconfig.descriptor | 22 ++++++++++++-- utils/proxy/_decoders/remoteconfig.proto | 29 +++++++++++++++++++ utils/proxy/_deserializer.py | 17 ++++++++++- 4 files changed, 66 insertions(+), 3 deletions(-) diff --git a/utils/proxy/_decoders/protobuf_schemas.py b/utils/proxy/_decoders/protobuf_schemas.py index 56af04c97f0..0ae795ca6ab 100644 --- a/utils/proxy/_decoders/protobuf_schemas.py +++ b/utils/proxy/_decoders/protobuf_schemas.py @@ -35,3 +35,4 @@ def _get_mesages(filename: str) -> dict[str, type[message.Message]]: File = _remoteconfig_messages["datadog.config.File"] OrgDataResponse = _remoteconfig_messages["datadog.config.OrgDataResponse"] OrgStatusResponse = _remoteconfig_messages["datadog.config.OrgStatusResponse"] +LatestConfigsRequest = _remoteconfig_messages["datadog.config.LatestConfigsRequest"] diff --git a/utils/proxy/_decoders/remoteconfig.descriptor b/utils/proxy/_decoders/remoteconfig.descriptor index 450e54c0a8b..d90a8ad202e 100644 --- a/utils/proxy/_decoders/remoteconfig.descriptor +++ b/utils/proxy/_decoders/remoteconfig.descriptor @@ -1,5 +1,5 @@ -Ö +Ñ (utils/proxy/_decoders/remoteconfig.protodatadog.config"¬ ConfigMetas- roots ( 2.datadog.config.TopMetaRroots5 @@ -34,4 +34,22 @@ topTargetsI enabled (Renabled authorized (R -authorizedbproto3 \ No newline at end of file +authorized"U +LatestConfigsRequest= +active_clients ( 2.datadog.config.ClientR activeClients"; +Client1 +state ( 2.datadog.config.ClientStateRstate"Î + ClientState! + root_version (R rootVersion' +targets_version (RtargetsVersion@ + config_states ( 2.datadog.config.ConfigStateR configStates + has_error (RhasError +error ( Rerror"“ + ConfigState +id ( Rid +version (Rversion +product ( Rproduct + apply_state (R +applyState + apply_error ( R +applyErrorbproto3 \ No newline at end of file diff --git a/utils/proxy/_decoders/remoteconfig.proto b/utils/proxy/_decoders/remoteconfig.proto index 97f50ac05dc..394a18e2be0 100644 --- a/utils/proxy/_decoders/remoteconfig.proto +++ b/utils/proxy/_decoders/remoteconfig.proto @@ -58,3 +58,32 @@ message OrgStatusResponse { bool enabled = 1; bool authorized = 2; } + +// Agent/native-client request sent when polling /api/v0.1/configurations. Needed to read back +// the client's reported config-apply state for agentless scenarios, where there is no agent +// relaying that state via /v0.7/config: the native RC client polls this endpoint directly, and +// reports the same per-config apply state (ClientState.config_states) inline on the request. + +message LatestConfigsRequest { + repeated Client active_clients = 6; +} + +message Client { + ClientState state = 1; +} + +message ClientState { + uint64 root_version = 1; + uint64 targets_version = 2; + repeated ConfigState config_states = 3; + bool has_error = 4; + string error = 5; +} + +message ConfigState { + string id = 1; + uint64 version = 2; + string product = 3; + uint64 apply_state = 4; + string apply_error = 5; +} diff --git a/utils/proxy/_deserializer.py b/utils/proxy/_deserializer.py index 1d61a37bae4..7a2a96a8221 100644 --- a/utils/proxy/_deserializer.py +++ b/utils/proxy/_deserializer.py @@ -26,7 +26,13 @@ ExportLogsServiceRequest, ExportLogsServiceResponse, ) -from ._decoders.protobuf_schemas import MetricPayload, TracePayload, SketchPayload, BackendResponsePayload +from ._decoders.protobuf_schemas import ( + MetricPayload, + TracePayload, + SketchPayload, + BackendResponsePayload, + LatestConfigsRequest, +) from ._decoders.metrics_v3 import decode_metrics_v3 from .trace_bytes_decoding import decode_trace_bytes_ascii, unpack_trace_bytes_msgpack from .traces.trace_v1 import deserialize_v1_trace, _uncompress_agent_v1_trace, decode_appsec_s_value @@ -239,6 +245,15 @@ def json_load(): return MessageToDict(BackendResponsePayload.FromString(content)) if path == "/api/beta/sketches": return MessageToDict(SketchPayload.FromString(content)) + if path == "/api/v0.1/configurations" and key == "request": + # The agentless native RC client polls this endpoint directly (no agent relay), and + # reports its per-config apply state inline on the request, the same information an + # agent-relayed client reports via a subsequent POST to /v0.7/config. + return MessageToDict( + LatestConfigsRequest.FromString(content), + preserving_proto_field_name=True, + use_integers_for_enums=True, + ) if content_type == "application/x-www-form-urlencoded" and content == b"[]" and path == "/v0.4/traces": return [] From 90f8b6d9f7fc6df737c793d087444979f302f792 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 03:31:20 +0200 Subject: [PATCH 02/18] Fix TUF test keyid derivation and support agentless RC polling interval TUF derives a key's keyid from the canonical JSON of the key object, not from the raw public key bytes. libdatadog's rust-tuf recomputes the keyid this way when verifying the root, and previously failed to match our test signature (metadata root signature threshold not met). Fix the derivation and regenerate the two pinned fixtures that embed the old (wrong) keyid. Also: TUF metadata version 0 is spec-invalid, but was used as an empty-state sentinel for the mocked backend; real clients (correctly) reject it. Move the sentinel to version 1, and add an opt-in agent_refresh_interval field to targets metadata (default None/omitted, a no-op for all existing callers) so agentless scenarios can override the native RC client's 60s default poll interval down to something tests can wait on. --- tests/test_the_test/test_remote_config.py | 4 +- utils/proxy/rc_response_builder.py | 13 +++-- utils/proxy/tuf.py | 58 +++++++++++++++++------ 3 files changed, 54 insertions(+), 21 deletions(-) diff --git a/tests/test_the_test/test_remote_config.py b/tests/test_the_test/test_remote_config.py index c0ecf49e5fc..8cd4120bd94 100644 --- a/tests/test_the_test/test_remote_config.py +++ b/tests/test_the_test/test_remote_config.py @@ -4,7 +4,7 @@ @scenarios.test_the_test def test_debugger_command_none(): expected = { - "targets": "ewogICJzaWduYXR1cmVzIjogWwogICAgewogICAgICAia2V5aWQiOiAiMTM5ZTM5NDBlNjRiNTQ5MTcyMjA4OGQ5YTBkNzQxNjI4ZmM4MjZlMDk0NzVkMzQxYTc4MGFjZGUzYzRiODA3MCIsCiAgICAgICJzaWciOiAiNWIyNDJlMDg5MjI0ZWExMzg5MjU0ZGE4MGQxMWQ3MWM4MDNkMGMyMGE1NDg1NzgwMGE2OTM4OWRhZjJlMjQwZTcyNTQ0Mjk0MjAzZWEyZWFmMDdmZjIzNjMxMzJjOGYxYWFmZTg4MTY0MTAwNWIwYzYwNjgwM2M4MWQzMzBiMGQiCiAgICB9CiAgXSwKICAic2lnbmVkIjogewogICAgIl90eXBlIjogInRhcmdldHMiLAogICAgImN1c3RvbSI6IHsKICAgICAgIm9wYXF1ZV9iYWNrZW5kX3N0YXRlIjogImV5Sm1iMjhpT2lBaVltRnlJbjA9IgogICAgfSwKICAgICJleHBpcmVzIjogIjMwMDAtMDEtMDFUMDA6MDA6MDBaIiwKICAgICJzcGVjX3ZlcnNpb24iOiAiMS4wIiwKICAgICJ0YXJnZXRzIjoge30sCiAgICAidmVyc2lvbiI6IDAKICB9Cn0=", + "targets": "ewogICJzaWduYXR1cmVzIjogWwogICAgewogICAgICAia2V5aWQiOiAiMmQ1Y2VkZmRiNGQyOWY4Mjk4NTA0YzFiYzUyMDM5NDQ5ODcyMmYwODM1MzI2MTE5NDBhNDk4ZjUxNjMxODQ3NSIsCiAgICAgICJzaWciOiAiNWIyNDJlMDg5MjI0ZWExMzg5MjU0ZGE4MGQxMWQ3MWM4MDNkMGMyMGE1NDg1NzgwMGE2OTM4OWRhZjJlMjQwZTcyNTQ0Mjk0MjAzZWEyZWFmMDdmZjIzNjMxMzJjOGYxYWFmZTg4MTY0MTAwNWIwYzYwNjgwM2M4MWQzMzBiMGQiCiAgICB9CiAgXSwKICAic2lnbmVkIjogewogICAgIl90eXBlIjogInRhcmdldHMiLAogICAgImN1c3RvbSI6IHsKICAgICAgIm9wYXF1ZV9iYWNrZW5kX3N0YXRlIjogImV5Sm1iMjhpT2lBaVltRnlJbjA9IgogICAgfSwKICAgICJleHBpcmVzIjogIjMwMDAtMDEtMDFUMDA6MDA6MDBaIiwKICAgICJzcGVjX3ZlcnNpb24iOiAiMS4wIiwKICAgICJ0YXJnZXRzIjoge30sCiAgICAidmVyc2lvbiI6IDAKICB9Cn0=", "target_files": [], "client_configs": [], } @@ -29,7 +29,7 @@ def test_debugger_command_one_probe(): ] expected = { - "targets": "ewogICJzaWduYXR1cmVzIjogWwogICAgewogICAgICAia2V5aWQiOiAiMTM5ZTM5NDBlNjRiNTQ5MTcyMjA4OGQ5YTBkNzQxNjI4ZmM4MjZlMDk0NzVkMzQxYTc4MGFjZGUzYzRiODA3MCIsCiAgICAgICJzaWciOiAiZjk0NzliYTAyNDRkYjBlMDAxNjdiZjczNTE0NTQxMWZmOTk3MmU2NWI0Njc5NzllODZjNDRiZmNhZmQ2OGEyNjQ4YzcyOGVkMDEwOTZhNDg4YmQ3ZWJjYTMyZTUzMWNjODdiYjBkYWYxMDA2YWQxODRjNTQ4OTQyN2Q5Nzc4MDMiCiAgICB9CiAgXSwKICAic2lnbmVkIjogewogICAgIl90eXBlIjogInRhcmdldHMiLAogICAgImN1c3RvbSI6IHsKICAgICAgIm9wYXF1ZV9iYWNrZW5kX3N0YXRlIjogImV5Sm1iMjhpT2lBaVltRnlJbjA9IgogICAgfSwKICAgICJleHBpcmVzIjogIjMwMDAtMDEtMDFUMDA6MDA6MDBaIiwKICAgICJzcGVjX3ZlcnNpb24iOiAiMS4wIiwKICAgICJ0YXJnZXRzIjogewogICAgICAiZGF0YWRvZy8yL0xJVkVfREVCVUdHSU5HL2xvZ1Byb2JlX2xvZzE3MGFhLWFjZGEtNDQ1My05MTExLTE0NzhhNm1ldGhvZC9jb25maWciOiB7CiAgICAgICAgImN1c3RvbSI6IHsKICAgICAgICAgICJ2IjogMQogICAgICAgIH0sCiAgICAgICAgImhhc2hlcyI6IHsKICAgICAgICAgICJzaGEyNTYiOiAiZWNmMzQ3ZmIwZWE0NjE2ZmU1NTc2YzIyODNhYWY1NmUyNjFmYmNjZDMxNjJiYTIxZjNjZmQwNDJjM2VjOWFjNSIKICAgICAgICB9LAogICAgICAgICJsZW5ndGgiOiAyODkKICAgICAgfQogICAgfSwKICAgICJ2ZXJzaW9uIjogMQogIH0KfQ==", + "targets": "ewogICJzaWduYXR1cmVzIjogWwogICAgewogICAgICAia2V5aWQiOiAiMmQ1Y2VkZmRiNGQyOWY4Mjk4NTA0YzFiYzUyMDM5NDQ5ODcyMmYwODM1MzI2MTE5NDBhNDk4ZjUxNjMxODQ3NSIsCiAgICAgICJzaWciOiAiZjk0NzliYTAyNDRkYjBlMDAxNjdiZjczNTE0NTQxMWZmOTk3MmU2NWI0Njc5NzllODZjNDRiZmNhZmQ2OGEyNjQ4YzcyOGVkMDEwOTZhNDg4YmQ3ZWJjYTMyZTUzMWNjODdiYjBkYWYxMDA2YWQxODRjNTQ4OTQyN2Q5Nzc4MDMiCiAgICB9CiAgXSwKICAic2lnbmVkIjogewogICAgIl90eXBlIjogInRhcmdldHMiLAogICAgImN1c3RvbSI6IHsKICAgICAgIm9wYXF1ZV9iYWNrZW5kX3N0YXRlIjogImV5Sm1iMjhpT2lBaVltRnlJbjA9IgogICAgfSwKICAgICJleHBpcmVzIjogIjMwMDAtMDEtMDFUMDA6MDA6MDBaIiwKICAgICJzcGVjX3ZlcnNpb24iOiAiMS4wIiwKICAgICJ0YXJnZXRzIjogewogICAgICAiZGF0YWRvZy8yL0xJVkVfREVCVUdHSU5HL2xvZ1Byb2JlX2xvZzE3MGFhLWFjZGEtNDQ1My05MTExLTE0NzhhNm1ldGhvZC9jb25maWciOiB7CiAgICAgICAgImN1c3RvbSI6IHsKICAgICAgICAgICJ2IjogMQogICAgICAgIH0sCiAgICAgICAgImhhc2hlcyI6IHsKICAgICAgICAgICJzaGEyNTYiOiAiZWNmMzQ3ZmIwZWE0NjE2ZmU1NTc2YzIyODNhYWY1NmUyNjFmYmNjZDMxNjJiYTIxZjNjZmQwNDJjM2VjOWFjNSIKICAgICAgICB9LAogICAgICAgICJsZW5ndGgiOiAyODkKICAgICAgfQogICAgfSwKICAgICJ2ZXJzaW9uIjogMQogIH0KfQ==", "target_files": [ { "path": "datadog/2/LIVE_DEBUGGING/logProbe_log170aa-acda-4453-9111-1478a6method/config", diff --git a/utils/proxy/rc_response_builder.py b/utils/proxy/rc_response_builder.py index 58657d20201..ab06ca5edc9 100644 --- a/utils/proxy/rc_response_builder.py +++ b/utils/proxy/rc_response_builder.py @@ -44,7 +44,7 @@ def build_rc_configurations_protobuf(rc_state: dict | None) -> bytes: Args: rc_state: RC state dict with 'targets' (base64 signed JSON) and 'target_files' (list of {path, raw} dicts). If None, - returns empty targets at version 0. + returns empty targets at version 1. Returns: Protobuf-encoded LatestConfigsResponse bytes @@ -58,9 +58,14 @@ def build_rc_configurations_protobuf(rc_state: dict | None) -> bytes: targets_parsed = json.loads(targets_json) version = targets_parsed.get("signed", {}).get("version", 1) else: - # Use version 0 for empty state so version 1 updates are accepted - version = 0 - targets_content = build_targets_content(version) + # TUF requires metadata versions to start at 1 (0 is spec-invalid and real TUF/uptane + # clients reject it), so the "nothing pushed yet" placeholder uses version 1. Real + # pushes are shifted up by one accordingly -- see send_state() in _remote_config.py. + version = 1 + # agent_refresh_interval=1: without it, the agentless native RC client defaults to a + # 60s poll interval (see libdd-remote-config's AgentlessFetcher), far exceeding the + # timeouts tests wait on. + targets_content = build_targets_content(version, agent_refresh_interval=1) targets_json = json.dumps(sign_tuf_document(targets_content), separators=(",", ":")).encode() # Build snapshot referencing targets diff --git a/utils/proxy/tuf.py b/utils/proxy/tuf.py index 62012b7c915..cde699b1cd9 100644 --- a/utils/proxy/tuf.py +++ b/utils/proxy/tuf.py @@ -25,17 +25,35 @@ # seed = 32 zero bytes (0x00 * 32) # signing_key = ed25519.SigningKey(seed) # public_key = signing_key.verify_key -# keyid = sha256(public_key_bytes).hex() +# keyid = sha256(canonical_json(key_object)).hex() # # Resulting values: # public_key_hex = "3b6a27bcceb6a42d62a3a8d02a6f0d73653215771de243a63ac048a18b59da29" -# keyid = "139e3940e64b5491722088d9a0d741628fc826e09475d341a780acde3c4b8070" +# keyid = "2d5cedfdb4d29f8298504c1bc520394498722f083532611940a498f516318475" # TEST_TUF_SIGNING_KEY_SEED = bytes.fromhex("0" * 64) # 32 zero bytes as seed TEST_TUF_SIGNING_KEY = SigningKey(TEST_TUF_SIGNING_KEY_SEED) TEST_TUF_PUBLIC_KEY = TEST_TUF_SIGNING_KEY.verify_key TEST_TUF_PUBLIC_KEY_HEX = TEST_TUF_PUBLIC_KEY.encode().hex() -TEST_TUF_KEYID = hashlib.sha256(TEST_TUF_PUBLIC_KEY.encode()).hexdigest() + +TEST_TUF_KEY = { + "keyid_hash_algorithms": ["sha256"], + "keytype": "ed25519", + "keyval": {"public": TEST_TUF_PUBLIC_KEY_HEX}, + "scheme": "ed25519", +} + + +def _canonical_json(content: dict) -> bytes: + """TUF canonical JSON: sorted keys, no whitespace.""" + return json.dumps(content, separators=(",", ":"), sort_keys=True).encode() + + +# TUF derives the keyid from the canonical JSON of the key object, not from the raw public key. +# The Datadog agent accepts whichever keyid the document declares, but tracers that verify the +# root themselves (libdatadog, via rust-tuf) recompute it and then match no signature, failing +# with "metadata root signature threshold not met". +TEST_TUF_KEYID = hashlib.sha256(_canonical_json(TEST_TUF_KEY)).hexdigest() def sign_tuf_document(signed_content: dict) -> dict: @@ -50,8 +68,7 @@ def sign_tuf_document(signed_content: dict) -> dict: Complete TUF document with {signatures: [...], signed: {...}} """ - canonical_json = json.dumps(signed_content, separators=(",", ":"), sort_keys=True).encode() - signature = TEST_TUF_SIGNING_KEY.sign(canonical_json).signature.hex() + signature = TEST_TUF_SIGNING_KEY.sign(_canonical_json(signed_content)).signature.hex() return { "signatures": [{"keyid": TEST_TUF_KEYID, "sig": signature}], @@ -69,14 +86,7 @@ def _build_tuf_root() -> dict: "_type": "root", "consistent_snapshot": True, "expires": "3000-01-01T00:00:00Z", # Far future expiry for testing - "keys": { - TEST_TUF_KEYID: { - "keyid_hash_algorithms": ["sha256"], - "keytype": "ed25519", - "keyval": {"public": TEST_TUF_PUBLIC_KEY_HEX}, - "scheme": "ed25519", - } - }, + "keys": {TEST_TUF_KEYID: TEST_TUF_KEY}, "roles": { "root": {"keyids": [TEST_TUF_KEYID], "threshold": 1}, "snapshot": {"keyids": [TEST_TUF_KEYID], "threshold": 1}, @@ -115,12 +125,21 @@ def get_tuf_root_json() -> str: DEFAULT_OPAQUE_BACKEND_STATE = "eyJmb28iOiAiYmFyIn0=" # base64('{"foo": "bar"}') DEFAULT_EXPIRES = "3000-01-01T00:00:00Z" +# The agentless native RC client (libdd-remote-config's AgentlessFetcher) defaults to a 60s +# poll interval unless the targets doc's custom.agent_refresh_interval says otherwise (clamped +# to [1, 60]s). Without this, tests that push config via the backend and then wait for the +# client's next poll would need to wait up to 60s. This field is ignored by the agent-mediated +# (target="tracer") flow, but it's still kept opt-in (None omits it) to avoid perturbing the +# exact pinned byte content of existing tracer-target callers/tests. +DEFAULT_AGENT_REFRESH_INTERVAL = None + def build_targets_content( version: int, targets: dict | None = None, opaque_backend_state: str = DEFAULT_OPAQUE_BACKEND_STATE, expires: str = DEFAULT_EXPIRES, + agent_refresh_interval: int | None = DEFAULT_AGENT_REFRESH_INTERVAL, ) -> dict: """Build the 'signed' content for TUF targets metadata. @@ -129,14 +148,20 @@ def build_targets_content( targets: Dict mapping target paths to their metadata (hashes, length, custom) opaque_backend_state: Base64-encoded backend state string expires: Expiration timestamp + agent_refresh_interval: Recommended poll interval in seconds for agentless clients. + Omitted from the 'custom' field entirely when None. Returns: The 'signed' portion of targets metadata (not yet wrapped with signatures) """ + custom = {"opaque_backend_state": opaque_backend_state} + if agent_refresh_interval is not None: + custom["agent_refresh_interval"] = agent_refresh_interval + return { "_type": "targets", - "custom": {"opaque_backend_state": opaque_backend_state}, + "custom": custom, "expires": expires, "spec_version": "1.0", "targets": targets if targets is not None else {}, @@ -149,6 +174,7 @@ def build_signed_targets( targets: dict | None = None, opaque_backend_state: str = DEFAULT_OPAQUE_BACKEND_STATE, expires: str = DEFAULT_EXPIRES, + agent_refresh_interval: int | None = DEFAULT_AGENT_REFRESH_INTERVAL, ) -> dict: """Build a complete signed TUF targets document. @@ -157,12 +183,14 @@ def build_signed_targets( targets: Dict mapping target paths to their metadata opaque_backend_state: Base64-encoded backend state string expires: Expiration timestamp + agent_refresh_interval: Recommended poll interval in seconds for agentless clients. + Omitted from the 'custom' field entirely when None. Returns: Complete signed TUF targets document with signatures """ - content = build_targets_content(version, targets, opaque_backend_state, expires) + content = build_targets_content(version, targets, opaque_backend_state, expires, agent_refresh_interval) return sign_tuf_document(content) From 113f91873080cb867f9db681d81889a8afe50adf Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 03:31:33 +0200 Subject: [PATCH 03/18] Make send_state()/RC test helpers agentless-aware When context.scenario.include_agent is False, there is no agent to relay client state via /v0.7/config: watch /api/v0.1/configurations directly (via interfaces.datadog_direct) and read the client's reported state out of the request instead of a follow-up response, normalizing protobuf's stringified uint64 fields back to int along the way. Also re-sign real backend config pushes at version + 1 and set agent_refresh_interval=1, to stay strictly greater than the version-1 empty-state sentinel and keep the agentless client's poll interval fast enough for tests to wait on (see previous commit). --- utils/_remote_config.py | 60 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 7 deletions(-) diff --git a/utils/_remote_config.py b/utils/_remote_config.py index 35a0597b10b..5388505d7f0 100644 --- a/utils/_remote_config.py +++ b/utils/_remote_config.py @@ -15,14 +15,14 @@ from utils._context.core import context from utils.dd_constants import RemoteConfigApplyState as ApplyState -from utils.interfaces import library +from utils.interfaces import library, datadog_direct from utils._logger import logger from utils.proxy.mocked_response import ( StaticJsonMockedTracerResponse, SequentialRemoteConfigJsonMockedTracerResponse, MockedBackendResponse, ) -from utils.proxy.tuf import build_signed_targets +from utils.proxy.tuf import build_signed_targets, sign_tuf_document from utils.proxy.rc_response_builder import build_rc_configurations_protobuf RemoteConfigTarget = Literal["tracer", "backend"] @@ -46,6 +46,25 @@ def from_json(d: dict) -> "RemoteConfigStateResults": return RemoteConfigStateResults(version=d["version"], state=d["state"], configs=d["configs"]) +def _normalize_protobuf_client_state(state: dict) -> dict: + """Coerce uint64 fields back to int. + + protobuf's JSON mapping (MessageToDict) renders uint64/int64 fields as strings (they don't + fit precisely in a JS number), unlike the plain-JSON client state reported over /v0.7/config + where these are already native ints. Comparing a str against the int constants/versions used + below (e.g. `state["apply_state"] == ApplyState.UNKNOWN`) would silently always be False. + """ + return { + **state, + "targets_version": int(state["targets_version"]) if "targets_version" in state else 0, + "root_version": int(state["root_version"]) if "root_version" in state else 0, + "config_states": [ + {**cs, "version": int(cs["version"]), "apply_state": int(cs.get("apply_state", 0))} + for cs in state.get("config_states", []) + ], + } + + def send_state( raw_payload: dict, *, @@ -85,6 +104,20 @@ def send_state( if target == "backend": assert backend_enabled, f"Remote config backend is not enabled on {context.scenario}" + # The backend's "no config pushed yet" placeholder occupies TUF version 1 (see + # rc_response_builder.build_rc_configurations_protobuf) -- 0 isn't spec-valid and + # real TUF/uptane clients reject it. So every real push is shifted up by one here to + # stay strictly greater than that placeholder. + targets = json.loads(base64.b64decode(raw_payload["targets"])) + targets["signed"]["version"] += 1 + # Also keep the agentless client polling fast (see rc_response_builder.py for why), + # since real config pushes go through this same TUF targets document. + targets["signed"]["custom"]["agent_refresh_interval"] = 1 + raw_payload = {**raw_payload, "targets": _json_to_base64(sign_tuf_document(targets["signed"]))} + if state_version != -1: + # Keep state_version (the caller's own global-state counter, e.g. _RemoteConfigState.version) + # shifted in lockstep, so the empty-client_configs comparison below still lines up. + state_version += 1 # Build protobuf on test runner side, send bytes to proxy rc_protobuf = build_rc_configurations_protobuf(raw_payload) MockedBackendResponse(path="/api/v0.1/configurations", content=rc_protobuf).send() @@ -110,12 +143,24 @@ def send_state( state = {} + # Agentless scenarios have no agent to relay client state via /v0.7/config: the native RC + # client polls /api/v0.1/configurations directly and reports the same per-config apply state + # inline on that request (LatestConfigsRequest.active_clients[0].state), instead of via a + # separate follow-up request. + agentless = not context.scenario.include_agent + watched_path = "/api/v0.1/configurations" if agentless else "/v0.7/config" + def remote_config_applied(data: dict) -> bool: nonlocal state - if data["path"] != "/v0.7/config": + if data["path"] != watched_path: return False - state = data.get("request", {}).get("content", {}).get("client", {}).get("state", {}) + if agentless: + active_clients = data.get("request", {}).get("content", {}).get("active_clients", []) + state = active_clients[0].get("state", {}) if active_clients else {} + state = _normalize_protobuf_client_state(state) + else: + state = data.get("request", {}).get("content", {}).get("client", {}).get("state", {}) targets_version = state.get("targets_version") config_states = state.get("config_states", []) logger.info( @@ -123,12 +168,12 @@ def remote_config_applied(data: dict) -> bool: ) if len(client_configs) == 0: - found = state["targets_version"] == state_version and state.get("config_states", []) == [] + found = state.get("targets_version", 0) == state_version and state.get("config_states", []) == [] if found: current_states.state = ApplyState.ACKNOWLEDGED return found - if state["targets_version"] != version: + if state.get("targets_version", 0) != version: return False for state in config_states: @@ -150,7 +195,8 @@ def remote_config_applied(data: dict) -> bool: return True logger.info(f"Waiting for RC version={version}, client_configs={client_configs}") - rv = library.wait_for(remote_config_applied, timeout=30) + watched_interface = datadog_direct if agentless else library + rv = watched_interface.wait_for(remote_config_applied, timeout=30) if not rv: logger.error( f"RC timed out. Last known state: targets_version={state.get('targets_version')}, " From 9e0d9ff1ed559ba32f0bb45716dcbd72f7059cad Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 03:31:46 +0200 Subject: [PATCH 04/18] Generalize BaseDebuggerTest to support an agentless backend interface Replace hardcoded interfaces.agent/_LOGS_PATH/_DEBUGGER_PATH/etc. references throughout BaseDebuggerTest with overridable class attributes (_backend_interface, _snapshot_paths, _traces_path, _telemetry_path, _symbols_interface, _symbols_path), defaulting to today's agent-mode values so no existing test changes behavior. Add AgentlessBaseDebuggerTest, which points these at interfaces.datadog_direct and the single unified agentless debugger-intake path (libdatadog's agentless debugger sender collapses logs/snapshots/diagnostics/symdb onto one path, unlike the agent-relayed protocol). Span-decoration collection (_collect_span_decoration) calls get_spans_list(), which only interfaces.agent implements (it parses /v0.4/traces msgpack; agentless traces use a different wire format). Gate that call behind hasattr() so it's skipped for agentless backends rather than raising, per decision: span-decoration assertions stay agent-only for now. --- tests/debugger/utils.py | 109 ++++++++++++++++++++++++++++------------ 1 file changed, 76 insertions(+), 33 deletions(-) diff --git a/tests/debugger/utils.py b/tests/debugger/utils.py index a96898910db..921beacf805 100644 --- a/tests/debugger/utils.py +++ b/tests/debugger/utils.py @@ -24,6 +24,11 @@ _SYMBOLS_PATH = "/symdb/v1/input" _TELEMETRY_PATH = "/api/v2/apmtelemetry" +# Agentless (direct-to-intake) path: libdatadog's agentless debugger sender collapses logs, +# snapshots, diagnostics AND symdb onto this single path on the debugger-intake host -- the +# intake demultiplexes by payload, not path (datadog-live-debugger/src/sender.rs derive_endpoint_path). +_AGENTLESS_DEBUGGER_PATH = "/api/v2/debugger" + # Library paths _DEBUGGER_V2_INPUT_PATH = "/debugger/v2/input" @@ -171,6 +176,16 @@ class BaseDebuggerTest: use_debugger_endpoint: bool = False + # Backend interface/paths, overridden by AgentlessBaseDebuggerTest for scenarios without a + # Datadog Agent (data captured over interfaces.datadog_direct instead of interfaces.agent, + # and logs/snapshots/diagnostics/symdb all collapse onto _AGENTLESS_DEBUGGER_PATH). + _backend_interface = interfaces.agent + _snapshot_paths: tuple[str, ...] = (_LOGS_PATH, _DEBUGGER_PATH) + _traces_path: str = _TRACES_PATH + _telemetry_path: str = _TELEMETRY_PATH + _symbols_interface = interfaces.library + _symbols_path: str = _SYMBOLS_PATH + def initialize_weblog_remote_config(self) -> None: self.setup_failures = [] if self.get_tracer()["language"] in ["ruby"]: @@ -478,7 +493,7 @@ def wait_for_all_probes(self, statuses: list[ProbeStatus], timeout: int = 30) -> logger.debug("Wating for all probes") self._wait_successful = False found_ids: set[str] = set() - interfaces.agent.wait_for( + self._backend_interface.wait_for( lambda data: self._wait_for_all_probes(data, statuses=statuses, found_ids=found_ids), timeout=timeout ) return self._wait_successful @@ -516,7 +531,7 @@ def _check_all_probes_status(probe_diagnostics: ProbeDiagnosticsCollection, stat if log_number >= BaseDebuggerTest._last_read: BaseDebuggerTest._last_read = log_number - if data["path"] in [_DEBUGGER_PATH, _LOGS_PATH]: + if data["path"] in self._snapshot_paths: probe_diagnostics = self._process_diagnostics_data([data]) logger.debug(probe_diagnostics) @@ -540,13 +555,13 @@ def wait_for_snapshot_received(self, exception_message: str = "", timeout: int = self._snapshot_found = False - interfaces.agent.wait_for( + self._backend_interface.wait_for( lambda data: self._wait_for_snapshot_received(data, exception_snapshot=exception_snapshot), timeout=timeout ) return self._snapshot_found def _wait_for_snapshot_received(self, data: dict, *, exception_snapshot: bool = False): - if data["path"] in [_LOGS_PATH, _DEBUGGER_PATH]: + if data["path"] in self._snapshot_paths: if exception_snapshot: logger.debug("Reading " + data["log_filename"] + ", looking for '" + self._exception_message + "'") @@ -598,7 +613,7 @@ def wait_for_all_snapshots(self, exception_message: str = "", timeout: int = 30) logger.debug(f"Waiting for snapshot with exception message: {exception_message}") self._exception_message = exception_message self._snapshot_found = False - interfaces.agent.wait_for( + self._backend_interface.wait_for( lambda data: self._wait_for_all_snapshots(data, exception_snapshot=True), timeout=timeout ) return self._snapshot_found @@ -606,13 +621,13 @@ def wait_for_all_snapshots(self, exception_message: str = "", timeout: int = 30) logger.debug(f"Waiting for snapshots from all probes: {self.probe_ids}") self._all_snapshots_found = False self._found_probe_ids: set[str] = set() - interfaces.agent.wait_for( + self._backend_interface.wait_for( lambda data: self._wait_for_all_snapshots(data, exception_snapshot=False), timeout=timeout ) return self._all_snapshots_found def _wait_for_all_snapshots(self, data: dict, *, exception_snapshot: bool = False) -> bool: - if data["path"] not in [_LOGS_PATH, _DEBUGGER_PATH]: + if data["path"] not in self._snapshot_paths: return False if exception_snapshot: @@ -670,10 +685,12 @@ def wait_for_snapshot_count(self, count: int, timeout: int = 5) -> bool: """ logger.debug(f"Waiting for {count} snapshots from probes: {self.probe_ids}") seen: set[tuple[str, int]] = set() - return interfaces.agent.wait_for(lambda data: self._count_snapshots(data, seen=seen) >= count, timeout=timeout) + return self._backend_interface.wait_for( + lambda data: self._count_snapshots(data, seen=seen) >= count, timeout=timeout + ) def _count_snapshots(self, data: dict, seen: set[tuple[str, int]]) -> int: - if data["path"] in [_LOGS_PATH, _DEBUGGER_PATH]: + if data["path"] in self._snapshot_paths: contents = data["request"].get("content", []) or [] for index, content in enumerate(_iter_snapshot_content_items(contents)): @@ -689,11 +706,11 @@ def wait_for_no_capture_reason_span(self, error_message: str, timeout: int) -> b self._error_message = error_message self._no_capture_reason_span_found = False - interfaces.agent.wait_for(self._wait_for_no_capture_reason_span, timeout=timeout) + self._backend_interface.wait_for(self._wait_for_no_capture_reason_span, timeout=timeout) return self._no_capture_reason_span_found def _wait_for_no_capture_reason_span(self, data: dict): - if data["path"] == _TRACES_PATH: + if data["path"] == self._traces_path: logger.debug( "Reading " + data["log_filename"] @@ -701,7 +718,7 @@ def _wait_for_no_capture_reason_span(self, data: dict): + self._error_message + "'" ) - spans = interfaces.agent.get_spans_list() + spans = self._backend_interface.get_spans_list() for span in spans: meta = span.meta if "_dd.debug.error.no_capture_reason" in meta: @@ -720,16 +737,16 @@ def wait_for_code_origin_span(self, timeout: int = 5) -> bool: self._span_found = False threshold = self._get_max_trace_file_number() - interfaces.agent.wait_for( + self._backend_interface.wait_for( lambda data: self._wait_for_code_origin_span(data, threshold=threshold), timeout=timeout, ) return self._span_found def _get_max_trace_file_number(self) -> int: - """Get the maximum trace file number currently in the agent interface.""" + """Get the maximum trace file number currently in the backend interface.""" max_number = 0 - for data in interfaces.agent.get_data(_TRACES_PATH): + for data in self._backend_interface.get_data(self._traces_path): log_filename_found = re.search(r"/(\d+)__", data["log_filename"]) if log_filename_found: file_number = int(log_filename_found.group(1)) @@ -737,7 +754,7 @@ def _get_max_trace_file_number(self) -> int: return max_number def _wait_for_code_origin_span(self, data: dict, *, threshold: int) -> bool: - if data["path"] == _TRACES_PATH: + if data["path"] == self._traces_path: log_filename_found = re.search(r"/(\d+)__", data["log_filename"]) if not log_filename_found: return False @@ -762,13 +779,13 @@ def _wait_for_code_origin_span(self, data: dict, *, threshold: int) -> bool: def wait_for_telemetry(self, telemetry_type: str, timeout: int = 5) -> dict | None: self._telemetry: dict | None = None - interfaces.agent.wait_for( + self._backend_interface.wait_for( lambda data: self._wait_for_telemetry(data, telemetry_type=telemetry_type), timeout=timeout ) return self._telemetry def _wait_for_telemetry(self, data: dict, telemetry_type: str) -> bool: - if data["path"] != _TELEMETRY_PATH: + if data["path"] != self._telemetry_path: return False content = data.get("request", {}).get("content", {}) @@ -799,7 +816,10 @@ def collect(self) -> None: def _collect_probe_diagnostics(self): def _read_data(): - if context.library == "java": + if len(self._snapshot_paths) == 1: + # Agentless: diagnostics, logs and snapshots all collapse onto one path. + path = self._snapshot_paths[0] + elif context.library == "java": if context.library.version > "1.27.0": path = _DEBUGGER_PATH else: @@ -815,7 +835,7 @@ def _read_data(): path = _LOGS_PATH # TODO: Should the default not be _DEBUGGER_PATH? logger.debug(f"Reading data from {path}") - return list(interfaces.agent.get_data(path)) + return list(self._backend_interface.get_data(path)) all_data = _read_data() self.probe_diagnostics = self._process_diagnostics_data(all_data) @@ -868,11 +888,11 @@ def _get_snapshot_hash(): # Collect snapshots from both the logs and debugger endpoints for compatibility for when we switched # snapshots to the debugger endpoint. - if not self.use_debugger_endpoint: - agent_logs_endpoint_requests += list(interfaces.agent.get_data(_LOGS_PATH)) - agent_logs_endpoint_requests += list(interfaces.agent.get_data(_DEBUGGER_PATH)) + if len(self._snapshot_paths) == 1 or self.use_debugger_endpoint: + agent_logs_endpoint_requests += list(self._backend_interface.get_data(self._snapshot_paths[-1])) else: - agent_logs_endpoint_requests += list(interfaces.agent.get_data(_DEBUGGER_PATH)) + for path in self._snapshot_paths: + agent_logs_endpoint_requests += list(self._backend_interface.get_data(path)) snapshot_hash: dict = {} @@ -895,8 +915,9 @@ def _get_snapshot_hash(): def get_snapshot_request_lengths(self, probe_id: str) -> list[int]: """Return decoded body lengths for backend requests containing a probe snapshot.""" - requests = list(interfaces.agent.get_data(_LOGS_PATH)) - requests += list(interfaces.agent.get_data(_DEBUGGER_PATH)) + requests = [] + for path in dict.fromkeys(self._snapshot_paths): + requests += list(self._backend_interface.get_data(path)) lengths: list[int] = [] for request in requests: @@ -914,12 +935,14 @@ def get_snapshot_request_lengths(self, probe_id: str) -> list[int]: def wait_for_additional_snapshots(self, timeout: int = 5) -> bool: """Wait for another backend request containing a snapshot for an expected probe.""" existing_files = { - data["log_filename"] for path in (_LOGS_PATH, _DEBUGGER_PATH) for data in interfaces.agent.get_data(path) + data["log_filename"] + for path in dict.fromkeys(self._snapshot_paths) + for data in self._backend_interface.get_data(path) } expected_probe_ids = set(self.probe_ids) def _contains_additional_snapshot(data: dict[str, Any]) -> bool: - if data["path"] not in (_LOGS_PATH, _DEBUGGER_PATH) or data["log_filename"] in existing_files: + if data["path"] not in self._snapshot_paths or data["log_filename"] in existing_files: return False content = data["request"].get("content", []) or [] @@ -930,7 +953,7 @@ def _contains_additional_snapshot(data: dict[str, Any]) -> bool: return False - return interfaces.agent.wait_for(_contains_additional_snapshot, timeout=timeout) + return self._backend_interface.wait_for(_contains_additional_snapshot, timeout=timeout) def _debugger_v2_input_snapshots_received(self): """Test that the library sends snapshots to the debugger/v2/input endpoint""" @@ -954,7 +977,13 @@ def _get_spans_hash(): else: span_decoration_line_key = "_dd.di.spandecorationargsandlocals.probe_id" - spans_list = interfaces.agent.get_spans_list() + if not hasattr(self._backend_interface, "get_spans_list"): + # Agentless mode's backend interface (interfaces.datadog_direct) captures direct-to-intake + # traffic in a different wire format than the agent-relayed /v0.4/traces payloads + # get_spans_list() parses, so span-decoration collection is agent-only. + return {} + + spans_list = self._backend_interface.get_spans_list() for span in spans_list: self.all_spans.append(span) @@ -1025,10 +1054,10 @@ def _get_spans_hash(): def _collect_symbols(self): def _get_symbols(): result: list[dict] = [] - raw_data = list(interfaces.library.get_data(_SYMBOLS_PATH)) + raw_data = list(self._symbols_interface.get_data(self._symbols_path)) if len(raw_data) == 0: - logger.info(f"No request has been sent to {_SYMBOLS_PATH}") + logger.info(f"No request has been sent to {self._symbols_path}") return result for data in raw_data: @@ -1054,7 +1083,7 @@ def _collect_symdb_upload_events(self): parameter equaling "event". """ events: list[dict[str, Any]] = [] - raw_data = list(interfaces.library.get_data(_SYMBOLS_PATH)) + raw_data = list(self._symbols_interface.get_data(self._symbols_path)) for data in raw_data: if not isinstance(data, dict) or "request" not in data: continue @@ -1155,3 +1184,17 @@ def write_approval(self, data: list, test_name: str, suffix: str) -> None: def read_approval(self, test_name: str, suffix: str) -> dict: with open(self._get_path(test_name, suffix), "r", encoding="utf-8") as f: return json.load(f) + + +class AgentlessBaseDebuggerTest(BaseDebuggerTest): + """Base for debugger/SymDB tests run against DEBUGGER_AGENTLESS (no Datadog Agent). + + libdatadog's agentless debugger sender collapses logs, snapshots, diagnostics and SymDB + onto a single path on the debugger-intake host (see _AGENTLESS_DEBUGGER_PATH), captured + over the datadog_direct proxy interface instead of the agent. + """ + + _backend_interface = interfaces.datadog_direct + _snapshot_paths: tuple[str, ...] = (_AGENTLESS_DEBUGGER_PATH,) + _symbols_interface = interfaces.datadog_direct + _symbols_path: str = _AGENTLESS_DEBUGGER_PATH From 88ee24d067c43c2ecd87ea5fdded334208330540 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 03:32:03 +0200 Subject: [PATCH 05/18] Generalize AgentlessEndToEndScenario beyond FFE's mock backend AgentlessEndToEndScenario hardcoded FFE's own MockFFEAgentlessBackendServer into its base __init__/configure(). Replace that with a _create_mock_backend() hook (default None, meaning no mock backend), structurally typed via a small AgentlessBackendServer protocol, so other agentless scenarios can reuse this base without carrying FFE-specific setup. FeatureFlaggingAgentlessEndToEndScenario now overrides the hook and keeps its own env/extra_hosts wiring. Move the generic 'capture direct egress' wiring (DD_SITE=mock-intake.invalid, HTTPS_PROXY through the proxy container, mounting the mitmproxy CA bundle so the weblog trusts the intercepted TLS, watchdog/teardown registration for interfaces.datadog_direct) up from FeatureFlaggingAgentlessEndToEndScenario into the shared base, gated on a new capture_direct_egress constructor param (default True). Add pass-through rc_api_enabled/rc_backend_enabled params: when rc_backend_enabled is set, wire the same TUF test root env vars already used to make the real Datadog Agent trust the mocked RC backend (utils/_context/containers.py::AgentContainer), so the native agentless RC client trusts it too. Override _wait_for_app_readiness() to skip the interfaces.library.ready wait when capturing direct egress: that readiness signal only ever fires on agent-facing traffic, which agentless mode never sends, so it would time out on every run otherwise. Also fix a resource leak in _start_mock_backend(): the mock backend was only assigned to self._mock_backend after backend.reset() succeeded, so a failing reset() skipped cleanup in configure()'s except handler and leaked the backend. --- .../_context/_scenarios/agentless_endtoend.py | 167 ++++++++++++------ 1 file changed, 117 insertions(+), 50 deletions(-) diff --git a/utils/_context/_scenarios/agentless_endtoend.py b/utils/_context/_scenarios/agentless_endtoend.py index d461c21de35..895db2d1e78 100644 --- a/utils/_context/_scenarios/agentless_endtoend.py +++ b/utils/_context/_scenarios/agentless_endtoend.py @@ -1,24 +1,29 @@ import json from pathlib import Path -from typing import TYPE_CHECKING, Literal, cast +from typing import Literal, Protocol import pytest from utils import interfaces from utils._context.containers import ServerlessInitContainer, TestedContainer from utils.docker_fixtures._core import extra_hosts_for_environment -from utils.mocked_backend.ffe import ( - EXPECTED_API_KEY, - MockFFEAgentlessBackendServer, - MockFFEAgentlessBackendStatus, -) +from utils.mocked_backend.ffe import EXPECTED_API_KEY, MockFFEAgentlessBackendServer from utils.proxy.ports import ProxyPorts +from utils.proxy.tuf import get_tuf_root_json from .core import ScenarioGroup, scenario_groups as all_scenario_groups from .endtoend import DdTraceEndToEndScenario -if TYPE_CHECKING: - from utils.interfaces._core import ProxyBasedInterfaceValidator +# Reused as the mock DD_API_KEY for agentless scenarios that don't need FFE's own mock backend. +AGENTLESS_MOCK_API_KEY = EXPECTED_API_KEY + + +class AgentlessBackendServer(Protocol): + """Structural type for the mock backend servers plugged into `AgentlessEndToEndScenario`.""" + + def reset(self) -> None: ... + def status(self) -> object: ... + def close(self) -> None: ... class AgentlessEndToEndScenario(DdTraceEndToEndScenario): @@ -27,8 +32,8 @@ class AgentlessEndToEndScenario(DdTraceEndToEndScenario): _default_scenario_groups: tuple[ScenarioGroup, ...] = () # exclude those scenario from tracer_release _mock_backend_status_filename = "mock_agentless_backend_status.json" - _mock_backend: MockFFEAgentlessBackendServer | None = None - _last_mock_backend_status: MockFFEAgentlessBackendStatus | None = None + _mock_backend: AgentlessBackendServer | None = None + _last_mock_backend_status: object | None = None def __init__( self, @@ -38,18 +43,60 @@ def __init__( weblog_env: dict[str, str | None] | None = None, other_weblog_containers: tuple[type[TestedContainer], ...] = (), scenario_groups: tuple[ScenarioGroup, ...] = (), - use_proxy_for_weblog: bool = False, + capture_direct_egress: bool = True, + rc_api_enabled: bool = False, + rc_backend_enabled: bool = False, + library_interface_timeout: int = 0, ) -> None: + self.capture_direct_egress = capture_direct_egress + + environment: dict[str, str | None] = dict(weblog_env or {}) + if capture_direct_egress: + environment.setdefault( + # The reserved .invalid domain fails closed if a request bypasses the proxy. + "DD_SITE", + "mock-intake.invalid", + ) + environment.setdefault("DD_PROXY_HTTPS", f"http://proxy:{ProxyPorts.datadog_direct}") + environment.setdefault("HTTPS_PROXY", f"http://proxy:{ProxyPorts.datadog_direct}") + + weblog_volumes: dict | None = None + if capture_direct_egress: + # The weblog talks HTTPS directly to the proxy (CONNECT tunnel), which + # terminates TLS with the mitmproxy CA -- same trust anchor already + # mounted into AgentContainer (utils/_context/containers.py) for its + # own HTTP_PROXY-based backend traffic. + weblog_volumes = { + "./utils/build/docker/agent/ca-certificates.crt": { + "bind": "/etc/ssl/certs/ca-certificates.crt", + "mode": "ro", + }, + } + + if rc_backend_enabled: + # The native agentless RC client only trusts Datadog's real embedded + # production/staging/gov TUF roots by default, so it would never trust the + # mocked backend's test-signed response without this override -- mirrors the + # same env vars/root JSON already used to make the real Agent trust the test + # TUF key (utils/_context/containers.py::AgentContainer.__init__). + tuf_root_json = get_tuf_root_json() + environment.setdefault("DD_REMOTE_CONFIGURATION_ENABLED", "true") + environment.setdefault("DD_REMOTE_CONFIGURATION_CONFIG_ROOT", tuf_root_json) + environment.setdefault("DD_REMOTE_CONFIGURATION_DIRECTOR_ROOT", tuf_root_json) + super().__init__( name, doc=doc, include_agent=False, - library_interface_timeout=0, + library_interface_timeout=library_interface_timeout, other_weblog_containers=other_weblog_containers, scenario_groups=[*scenario_groups, all_scenario_groups.agentless], use_proxy_for_agent=False, - use_proxy_for_weblog=use_proxy_for_weblog, - weblog_env=weblog_env, + use_proxy_for_weblog=capture_direct_egress, + rc_api_enabled=rc_api_enabled, + rc_backend_enabled=rc_backend_enabled, + weblog_env=environment, + weblog_volumes=weblog_volumes, ) def configure(self, config: pytest.Config) -> None: @@ -61,24 +108,36 @@ def configure(self, config: pytest.Config) -> None: self._start_mock_backend() super().configure(config) + + if self.capture_direct_egress: + interfaces.datadog_direct.configure(self.host_log_folder, replay=self.replay) except BaseException: self._stop_mock_backend(persist_status=False) raise + def _wait_for_app_readiness(self) -> None: + if self.capture_direct_egress: + # Agentless mode never sends agent-facing traffic (/v0.4/traces, etc.), so + # interfaces.library.ready (set only by such traffic) would never fire here. + # The weblog container's own healthcheck already confirms it's up. + return + super()._wait_for_app_readiness() + + def _create_mock_backend(self) -> AgentlessBackendServer | None: + """Override in subclasses that need a mock agentless config/data backend.""" + return None + def _start_mock_backend(self) -> None: - assert self._mock_backend is None, "mock FFE agentless backend is already running" + assert self._mock_backend is None, "mock agentless backend is already running" - self._mock_backend = MockFFEAgentlessBackendServer() - self._mock_backend.reset() + backend = self._create_mock_backend() + if backend is None: + return - environment = self.weblog_infra.library_container.environment - environment |= { - "DD_API_KEY": EXPECTED_API_KEY, - "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL": self._mock_backend.library_config_url, - } - self.weblog_infra.library_container.extra_hosts = extra_hosts_for_environment(environment) + self._mock_backend = backend + backend.reset() - def mock_backend_status(self) -> MockFFEAgentlessBackendStatus | None: + def mock_backend_status(self) -> object | None: if self._mock_backend is not None: return self._mock_backend.status() return self._last_mock_backend_status @@ -88,10 +147,7 @@ def _mock_backend_status_path(self) -> Path: return Path(self.host_log_folder) / self._mock_backend_status_filename def _load_mock_backend_status(self) -> None: - self._last_mock_backend_status = cast( - "MockFFEAgentlessBackendStatus", - json.loads(self._mock_backend_status_path.read_text(encoding="utf-8")), - ) + self._last_mock_backend_status = json.loads(self._mock_backend_status_path.read_text(encoding="utf-8")) def _stop_mock_backend(self, *, persist_status: bool = True) -> None: backend = self._mock_backend @@ -116,6 +172,21 @@ def close_targets(self) -> None: finally: self._stop_mock_backend() + def _start_interfaces_watchdog(self) -> None: + super()._start_interfaces_watchdog() + if self.capture_direct_egress: + self.start_interfaces_watchdog([interfaces.datadog_direct]) + + def _wait_and_stop_containers(self, *, is_empty_test_run: bool) -> None: + super()._wait_and_stop_containers(is_empty_test_run=is_empty_test_run) + if not self.capture_direct_egress: + return + + if self.replay: + interfaces.datadog_direct.load_data_from_logs() + + interfaces.datadog_direct.check_deserialization_errors() + class FeatureFlaggingAgentlessEndToEndScenario(AgentlessEndToEndScenario): """FFE end-to-end scenario with UFC available before the weblog starts.""" @@ -142,14 +213,6 @@ def __init__( environment.update(weblog_env or {}) other_weblog_containers: tuple[type[TestedContainer], ...] = () - if exposure_egress is not None: - environment |= { - # The reserved .invalid domain fails closed if a request bypasses the proxy. - "DD_SITE": "mock-intake.invalid", - "DD_PROXY_HTTPS": f"http://proxy:{ProxyPorts.datadog_direct}", - "HTTPS_PROXY": f"http://proxy:{ProxyPorts.datadog_direct}", - } - if exposure_egress == "sidecar": serverless_init_port = str(ServerlessInitContainer.apm_receiver_port) environment |= { @@ -164,7 +227,7 @@ def __init__( doc=doc, other_weblog_containers=other_weblog_containers, scenario_groups=(all_scenario_groups.ffe,), - use_proxy_for_weblog=exposure_egress is not None, + capture_direct_egress=exposure_egress is not None, weblog_env=environment, ) @@ -174,12 +237,26 @@ def __init__( for env_name in ("DD_AGENT_HOST", "DD_DOGSTATSD_HOST", "DD_TRACE_AGENT_PORT", "DD_TRACE_AGENT_URL"): self.weblog_infra.library_container.environment.pop(env_name, None) + def _create_mock_backend(self) -> MockFFEAgentlessBackendServer: + return MockFFEAgentlessBackendServer() + + def _start_mock_backend(self) -> None: + super()._start_mock_backend() + backend = self._mock_backend + assert isinstance(backend, MockFFEAgentlessBackendServer) + + environment = self.weblog_infra.library_container.environment + environment |= { + "DD_API_KEY": EXPECTED_API_KEY, + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL": backend.library_config_url, + } + self.weblog_infra.library_container.extra_hosts = extra_hosts_for_environment(environment) + def configure(self, config: pytest.Config) -> None: try: super().configure(config) if self.exposure_egress is not None: interfaces.datadog_sidecar.configure(self.host_log_folder, replay=self.replay) - interfaces.datadog_direct.configure(self.host_log_folder, replay=self.replay) except BaseException: self._stop_mock_backend(persist_status=False) raise @@ -199,7 +276,7 @@ def _set_containers_dependancies(self) -> None: def _start_interfaces_watchdog(self) -> None: super()._start_interfaces_watchdog() if self.exposure_egress is not None: - self.start_interfaces_watchdog([interfaces.datadog_sidecar, interfaces.datadog_direct]) + self.start_interfaces_watchdog([interfaces.datadog_sidecar]) def _wait_for_app_readiness(self) -> None: if self.exposure_egress is not None: @@ -217,18 +294,8 @@ def _wait_and_stop_containers(self, *, is_empty_test_run: bool) -> None: return if self.replay: - self._load_telemetry_interfaces() + interfaces.datadog_sidecar.load_data_from_logs() elif self.exposure_egress == "sidecar": self.serverless_init_container.stop() interfaces.datadog_sidecar.check_deserialization_errors() - interfaces.datadog_direct.check_deserialization_errors() - - @staticmethod - def _load_telemetry_interfaces() -> None: - telemetry_interfaces: tuple[ProxyBasedInterfaceValidator, ...] = ( - interfaces.datadog_sidecar, - interfaces.datadog_direct, - ) - for interface in telemetry_interfaces: - interface.load_data_from_logs() From 9e406142951ee2eda072b3b6269869639204debc Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 03:32:16 +0200 Subject: [PATCH 06/18] Add APM_TRACING_AGENTLESS and DEBUGGER_AGENTLESS scenarios APM_TRACING_AGENTLESS (AgentlessEndToEndScenario) covers direct-to-intake trace submission, client-side stats, and Remote Configuration under DD_AGENTLESS_ENABLED, with no Datadog Agent. DEBUGGER_AGENTLESS (new DebuggerAgentlessScenario, in its own module) covers agentless Dynamic Instrumentation (probe upload/logs/snapshots) and Symbol DB the same way, reusing the TUF-trust and mocked RC backend wiring generalized in the previous commit. Both are excluded from the tracer_release scenario group, same as the other agentless scenarios; add them to test_group_rules.py's exclusion list. --- tests/test_the_test/test_group_rules.py | 2 ++ utils/_context/_scenarios/__init__.py | 33 ++++++++++++++++++- .../_context/_scenarios/debugger_agentless.py | 31 +++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 utils/_context/_scenarios/debugger_agentless.py diff --git a/tests/test_the_test/test_group_rules.py b/tests/test_the_test/test_group_rules.py index 5f0b8e686c3..fea4327e41c 100644 --- a/tests/test_the_test/test_group_rules.py +++ b/tests/test_the_test/test_group_rules.py @@ -42,6 +42,8 @@ def test_tracer_release(): scenarios.fuzzer, dormant_agentless_scenario, *agentless_exposure_scenarios, + scenarios.apm_tracing_agentless, + scenarios.debugger_agentless, scenarios.mock_the_test, scenarios.mock_the_test_2, scenarios.test_the_test, diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 9d598ff2601..6e4bffb2be9 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -7,7 +7,12 @@ from .aws_lambda import LambdaScenario from .core import Scenario, scenario_groups from .default import DefaultScenario -from .agentless_endtoend import FeatureFlaggingAgentlessEndToEndScenario +from .agentless_endtoend import ( + AGENTLESS_MOCK_API_KEY, + AgentlessEndToEndScenario, + FeatureFlaggingAgentlessEndToEndScenario, +) +from .debugger_agentless import DebuggerAgentlessScenario from .endtoend import ( DockerScenario, DdTraceEndToEndScenario, @@ -860,6 +865,32 @@ class _Scenarios: exposure_egress="sidecar", ) + apm_tracing_agentless = AgentlessEndToEndScenario( + "APM_TRACING_AGENTLESS", + doc="Validate direct-to-intake trace submission, client-side stats, and Remote " + "Configuration when DD_AGENTLESS_ENABLED is set, without a Datadog Agent.", + rc_api_enabled=True, + rc_backend_enabled=True, + weblog_env={ + "DD_AGENTLESS_ENABLED": "true", + "DD_API_KEY": AGENTLESS_MOCK_API_KEY, + "DD_TRACE_STATS_COMPUTATION_ENABLED": "true", + "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "1", + }, + ) + + debugger_agentless = DebuggerAgentlessScenario( + "DEBUGGER_AGENTLESS", + doc="Validate agentless Dynamic Instrumentation (probe upload/logs/snapshots) and " + "Symbol DB, without a Datadog Agent.", + weblog_env={ + "DD_AGENTLESS_ENABLED": "true", + "DD_API_KEY": AGENTLESS_MOCK_API_KEY, + "DD_DYNAMIC_INSTRUMENTATION_ENABLED": "true", + "_DD_SYMBOL_DATABASE_FORCE_UPLOAD": "true", + }, + ) + remote_config_mocked_backend_asm_features_nocache = DdTraceEndToEndScenario( "REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE", rc_api_enabled=True, diff --git a/utils/_context/_scenarios/debugger_agentless.py b/utils/_context/_scenarios/debugger_agentless.py new file mode 100644 index 00000000000..d2375a4688f --- /dev/null +++ b/utils/_context/_scenarios/debugger_agentless.py @@ -0,0 +1,31 @@ +from .agentless_endtoend import AgentlessEndToEndScenario +from .core import scenario_groups + + +class DebuggerAgentlessScenario(AgentlessEndToEndScenario): + """Agentless Dynamic Instrumentation (probes) and Symbol DB, without a Datadog Agent. + + Reuses the same TUF test root/key already trusted by the real Agent + (`utils._context.containers.AgentContainer`) for the native agentless Remote + Configuration client, and the same mocked-backend protobuf responses + (`rc_api_enabled`/`rc_backend_enabled`) already served to the Agent's own RC poller -- + both wired generically in `AgentlessEndToEndScenario` whenever `rc_backend_enabled=True`. + """ + + def __init__(self, name: str, *, doc: str, weblog_env: dict[str, str | None] | None = None) -> None: + base_weblog_env: dict[str, str | None] = { + "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "0.2", + "DD_DYNAMIC_INSTRUMENTATION_UPLOAD_INTERVAL_SECONDS": "0.1", + "DD_DYNAMIC_INSTRUMENTATION_UPLOAD_FLUSH_INTERVAL": "0.1", + } + base_weblog_env.update(weblog_env or {}) + + super().__init__( + name, + doc=doc, + rc_api_enabled=True, + rc_backend_enabled=True, + library_interface_timeout=5, + scenario_groups=(scenario_groups.debugger,), + weblog_env=base_weblog_env, + ) From 8d16c54458af65a5ecda11050b8be3e91db9d1d7 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 03:32:29 +0200 Subject: [PATCH 07/18] Add test coverage for DD_AGENTLESS_ENABLED (traces, stats, RC, debugger, SymDB) tests/test_agentless.py (APM_TRACING_AGENTLESS): trace submission, client-side stats, and Remote Configuration all land directly on the intake with no Datadog Agent, each asserted on host/path/headers, and RC additionally drives a real config push through to ACKNOWLEDGED via send_apm_tracing_command(). tests/debugger/test_debugger_agentless.py (DEBUGGER_AGENTLESS): a log probe installed via the agentless native RC client emits a snapshot to the agentless debugger intake; Symbol DB (forced via _DD_SYMBOL_DATABASE_FORCE_UPLOAD) uploads to the same unified path. Both verified end-to-end against a live weblog build of the (unmerged) dd-trace-py bob/agentless-setting branch. Endpoints/shapes are derived from reading tracer/libdatadog source, since that branch isn't released; re-confirm against real captures once it merges. --- tests/debugger/test_debugger_agentless.py | 91 ++++++++++++++++++ tests/test_agentless.py | 112 ++++++++++++++++++++++ 2 files changed, 203 insertions(+) create mode 100644 tests/debugger/test_debugger_agentless.py create mode 100644 tests/test_agentless.py diff --git a/tests/debugger/test_debugger_agentless.py b/tests/debugger/test_debugger_agentless.py new file mode 100644 index 00000000000..0da0be611bc --- /dev/null +++ b/tests/debugger/test_debugger_agentless.py @@ -0,0 +1,91 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2021 Datadog, Inc. + +"""Dynamic Instrumentation (probe upload/logs/snapshots) and Symbol DB, without a Datadog Agent. + +Endpoints/shapes here are derived from reading the dd-trace-py/libdatadog source on the +(unmerged) `bob/agentless-setting` branch, not from live-captured traffic - see +tests/debugger/utils.py::AgentlessBaseDebuggerTest and +utils/_context/_scenarios/debugger_agentless.py for the scenario/interface wiring. +""" + +import tests.debugger.utils as debugger +from utils import features, scenarios + + +@features.debugger +@scenarios.debugger_agentless +class Test_Agentless_Debugger_Probe_Snapshot(debugger.AgentlessBaseDebuggerTest): + """A log probe installed via the agentless native RC client emits a snapshot directly to + the agentless debugger intake, with no Datadog Agent involved. + """ + + def setup_log_method_snapshot(self): + self.initialize_weblog_remote_config() + + probes = debugger.read_probes("probe_snapshot_log_method") + for probe in probes: + probe["id"] = debugger.generate_probe_id("log") + self.set_probes(probes) + + self.send_rc_probes() + if not self.wait_for_all_probes(statuses=["INSTALLED"], timeout=60): + self.setup_failures.append("Probes did not reach INSTALLED status") + return + + self.send_weblog_request("/debugger/log") + self.wait_for_all_probes(statuses=["EMITTING"]) + if not self.wait_for_all_snapshots(timeout=60): + self.setup_failures.append("Snapshot was not received") + + def test_log_method_snapshot(self): + self.collect() + + self.assert_setup_ok() + self.assert_rc_state_not_error() + self.assert_all_probes_are_emitting() + + for probe_id in self.probe_ids: + assert probe_id in self.probe_snapshots, f"No snapshot was captured for probe {probe_id}" + assert len(self.probe_snapshots[probe_id]) > 0, f"No snapshot was captured for probe {probe_id}" + + path = self._snapshot_paths[0] + requests = list(self._backend_interface.get_data(path)) + assert len(requests) > 0, f"No request captured on {path}" + + request = requests[-1] + assert request["host"] == "debugger-intake.mock-intake.invalid" + headers = {name.lower(): value for name, value in request["request"]["headers"]} + assert "dd-api-key" in headers + + +@features.debugger_symdb +@scenarios.debugger_agentless +class Test_Agentless_SymbolDB(debugger.AgentlessBaseDebuggerTest): + """Symbol DB, forced via _DD_SYMBOL_DATABASE_FORCE_UPLOAD, uploads directly to the agentless + debugger intake (the same unified path as logs/snapshots/diagnostics in agentless mode). + """ + + def setup_symdb_upload(self): + self.initialize_weblog_remote_config() + + def test_symdb_upload(self): + self.collect() + self.assert_setup_ok() + + assert len(self.symbols) > 0, "No symbol files were found" + + errors = [] + for symbol in self.symbols: + error = symbol.get("system-tests-error") + if error is not None: + errors.append( + f"Error is: {error}, exported to file: {symbol.get('system-tests-file-path', 'No file path')}" + ) + assert not errors, "Found system-tests-errors:\n" + "\n".join(f"- {err}" for err in errors) + + requests = list(self._symbols_interface.get_data(self._symbols_path)) + assert len(requests) > 0, f"No request captured on {self._symbols_path}" + headers = {name.lower(): value for name, value in requests[-1]["request"]["headers"]} + assert "dd-api-key" in headers diff --git a/tests/test_agentless.py b/tests/test_agentless.py new file mode 100644 index 00000000000..424b8b24c03 --- /dev/null +++ b/tests/test_agentless.py @@ -0,0 +1,112 @@ +"""Direct-to-intake delivery (DD_AGENTLESS_ENABLED), without a Datadog Agent. + +Endpoints below are derived from reading the tracer/libdatadog source on the (unmerged) +`bob/agentless-setting` branch of dd-trace-py, not from live-captured traffic. Re-confirm +against real proxy captures once the branch is buildable in this environment; see +utils/_context/_scenarios/agentless_endtoend.py and debugger_agentless.py for the scenario setup. +""" + +from utils import features, interfaces, scenarios, weblog +from utils._context._scenarios.agentless_endtoend import AGENTLESS_MOCK_API_KEY +from utils._remote_config import send_apm_tracing_command +from utils.dd_constants import RemoteConfigApplyState as ApplyState + +TRACE_SUBMISSION_PATH = "/v1/input" +TRACE_SUBMISSION_HOST = "browser-intake-mock-intake.invalid" + +STATS_PATH = "/api/v0.2/stats" +STATS_HOST = "trace.agent.mock-intake.invalid" + +RC_CONFIGURATIONS_PATH = "/api/v0.1/configurations" +RC_HOST = "config.mock-intake.invalid" + + +def _headers(request: dict) -> dict[str, str]: + return {name.lower(): value for name, value in request["request"]["headers"]} + + +def _requests_at(host: str, path: str) -> list[dict]: + return [data for data in interfaces.datadog_direct.get_data(path) if data["host"] == host] + + +@scenarios.apm_tracing_agentless +@features.not_reported +class Test_Agentless_Trace_Submission: + """Traces are sent directly to the intake, bypassing the Datadog Agent.""" + + def setup_trace_submission(self): + self.r = weblog.get("/") + + def test_trace_submission(self): + assert self.r.status_code == 200 + + requests = _requests_at(TRACE_SUBMISSION_HOST, TRACE_SUBMISSION_PATH) + assert len(requests) != 0, f"No request captured on {TRACE_SUBMISSION_HOST}{TRACE_SUBMISSION_PATH}" + + request = requests[-1] + assert request["response"]["status_code"] // 100 == 2 + + headers = _headers(request) + assert headers["dd-api-key"] in {AGENTLESS_MOCK_API_KEY, "--redacted--"} + + content = request["request"]["content"] + assert content, "Trace submission request body is empty" + + +@scenarios.apm_tracing_agentless +@features.client_side_stats_supported +class Test_Agentless_Stats: + """Client-side trace stats are sent directly to the intake, on their own endpoint.""" + + def setup_stats(self): + self.r = weblog.get("/") + + def test_stats(self): + assert self.r.status_code == 200 + + stats_requests = _requests_at(STATS_HOST, STATS_PATH) + assert len(stats_requests) != 0, f"No request captured on {STATS_HOST}{STATS_PATH}" + + request = stats_requests[-1] + assert request["response"]["status_code"] // 100 == 2 + + headers = _headers(request) + assert headers["dd-api-key"] in {AGENTLESS_MOCK_API_KEY, "--redacted--"} + + # Stats and traces are distinct payloads on distinct hosts/paths. + trace_requests = _requests_at(TRACE_SUBMISSION_HOST, TRACE_SUBMISSION_PATH) + assert request not in trace_requests + + +@scenarios.apm_tracing_agentless +@features.remote_config_object_supported +class Test_Agentless_Remote_Config: + """The native agentless Remote Configuration client polls the intake directly. + + There is no agent to relay client state via `/v0.7/config`: the native client polls + `/api/v0.1/configurations` directly and reports its per-config apply state inline on that + same request (LatestConfigsRequest.active_clients[0].state) instead of via a separate + follow-up request. send_apm_tracing_command()/send_state() (utils/_remote_config.py) now + detect this agentless shape (keyed off `context.scenario.include_agent`), so this drives a + real config push and confirms application the same way agent-mode RC tests do. + """ + + def setup_remote_config_poll(self): + self.rc_state = send_apm_tracing_command(dynamic_instrumentation_enabled=True) + + def test_remote_config_poll(self): + assert self.rc_state.state == ApplyState.ACKNOWLEDGED, ( + f"RC config was not acknowledged: state={self.rc_state.state}, configs={self.rc_state.configs}" + ) + for config in self.rc_state.configs.values(): + assert config.get("apply_state") != ApplyState.ERROR, f"RC config apply error: {config}" + + requests = _requests_at(RC_HOST, RC_CONFIGURATIONS_PATH) + assert len(requests) != 0, f"No request captured on {RC_HOST}{RC_CONFIGURATIONS_PATH}" + + request = requests[-1] + assert request["method"] == "POST" + + headers = _headers(request) + assert headers["dd-api-key"] in {AGENTLESS_MOCK_API_KEY, "--redacted--"} + assert headers.get("content-type") == "application/x-protobuf" From ce7d1c5f171f7e2a26b13afabb7964135782bde9 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 03:32:40 +0200 Subject: [PATCH 08/18] Mark agentless test coverage as missing_feature for non-python tracers Scenario and test code is generic (no python-only guards), scoped to python for now via manifest entries only, so dropping these entries is enough once another tracer implements DD_AGENTLESS_ENABLED. --- manifests/cpp_httpd.yml | 1 + manifests/cpp_nginx.yml | 1 + manifests/dotnet.yml | 2 ++ manifests/golang.yml | 2 ++ manifests/java.yml | 2 ++ manifests/nodejs.yml | 2 ++ manifests/php.yml | 2 ++ manifests/ruby.yml | 2 ++ manifests/rust.yml | 1 + 9 files changed, 15 insertions(+) diff --git a/manifests/cpp_httpd.yml b/manifests/cpp_httpd.yml index 945af4d29fc..b1ecd8e2392 100644 --- a/manifests/cpp_httpd.yml +++ b/manifests/cpp_httpd.yml @@ -136,6 +136,7 @@ manifest: tests/stats/test_stats.py::Test_Time_Bucketing::test_client_side_stats: missing_feature # Created by easy win activation script tests/stats/test_stats.py::Test_Time_Bucketing::test_client_side_stats_bucket_alignment: missing_feature # Created by easy win activation script tests/stats/test_stats.py::Test_Transport_Headers: missing_feature # Created by easy win activation script + tests/test_agentless.py: missing_feature (C++ is not in scope for agentless mode (DD_AGENTLESS_ENABLED); python-only for now) tests/test_baggage.py::Test_Baggage_Headers_Api_Datadog: incomplete_test_app (/otel_drop_in_baggage_api_datadog endpoint is not implemented) tests/test_baggage.py::Test_Baggage_Headers_Api_OTel: incomplete_test_app (/otel_drop_in_baggage_api_otel endpoint is not implemented) tests/test_baggage.py::Test_Baggage_Headers_Basic: incomplete_test_app (/make_distant_call endpoint is not implemented) diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml index a4ab8849ea7..48afcfd4170 100644 --- a/manifests/cpp_nginx.yml +++ b/manifests/cpp_nginx.yml @@ -396,6 +396,7 @@ manifest: tests/stats/test_stats.py::Test_Time_Bucketing::test_client_side_stats: missing_feature # Created by easy win activation script tests/stats/test_stats.py::Test_Time_Bucketing::test_client_side_stats_bucket_alignment: missing_feature # Created by easy win activation script tests/stats/test_stats.py::Test_Transport_Headers: missing_feature # Created by easy win activation script + tests/test_agentless.py: missing_feature (C++ is not in scope for agentless mode (DD_AGENTLESS_ENABLED); python-only for now) tests/test_baggage.py::Test_Baggage_Headers_Api_Datadog: incomplete_test_app (/otel_drop_in_baggage_api_datadog endpoint is not implemented) tests/test_baggage.py::Test_Baggage_Headers_Api_OTel: incomplete_test_app (/otel_drop_in_baggage_api_otel endpoint is not implemented) tests/test_baggage.py::Test_Baggage_Headers_Basic: incomplete_test_app (/make_distant_call endpoint is not implemented) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 774815390e7..c7926bd101a 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -608,6 +608,7 @@ manifest: tests/auto_inject/test_auto_inject_install.py::TestSimpleInstallerAutoInjectManualAppsec: v3.36.0 tests/auto_inject/test_auto_inject_install.py::TestSimpleInstallerAutoInjectManualProfiling: bug (PROF-12209) tests/cws/test_thread_context_sharing.py::Test_ThreadContextSharing: missing_feature (missing /security/thread_context_sharing endpoint on weblog) + tests/debugger/test_debugger_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for .NET; python-only for now) tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Line_Capture_Expressions: missing_feature (Not yet implemented) tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Method_Capture_Expressions: missing_feature (Not yet implemented) tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Method_Capture_Expressions::test_complex_capture_expressions: missing_feature (Not yet implemented) @@ -1212,6 +1213,7 @@ manifest: - weblog_declaration: uds: '>=3.43.0' poc: '>=3.43.0' + tests/test_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for .NET; python-only for now) tests/test_baggage.py::Test_Baggage_Headers_Api_Datadog: v3.6.0 tests/test_baggage.py::Test_Baggage_Headers_Api_OTel: bug (APMAPI-1849) tests/test_baggage.py::Test_Baggage_Headers_Basic: v3.6.0 diff --git a/manifests/golang.yml b/manifests/golang.yml index 1041600d993..484024ff747 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1080,6 +1080,7 @@ manifest: tests/auto_inject/test_auto_inject_install.py::TestHostAutoInjectInstallScriptAppsec: v2.0.0 tests/auto_inject/test_auto_inject_install.py::TestSimpleInstallerAutoInjectManualAppsec: v2.0.0 tests/cws/test_thread_context_sharing.py::Test_ThreadContextSharing: missing_feature (missing /security/thread_context_sharing endpoint on weblog) + tests/debugger/test_debugger_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for Go; python-only for now) tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Line_Capture_Expressions: v2.2.3 tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Method_Capture_Expressions: v2.2.3 tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Method_Capture_Expressions::test_complex_capture_expressions: missing_feature (index expression not yet supported in Go system-probe) @@ -1629,6 +1630,7 @@ manifest: tests/stats/test_stats.py::Test_Client_Stats_With_Client_Obfuscation: v2.9.1 tests/stats/test_stats.py::Test_Client_Stats_With_Client_Obfuscation_Disabled: missing_feature tests/stats/test_stats.py::Test_Stats_Service_Source: irrelevant (Only implemented for Java) + tests/test_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for Go; python-only for now) tests/test_baggage.py::Test_Baggage_Headers_Api_Datadog: incomplete_test_app (/otel_drop_in_baggage_api_datadog endpoint is not implemented) tests/test_baggage.py::Test_Baggage_Headers_Api_OTel: incomplete_test_app (/otel_drop_in_baggage_api_otel endpoint is not implemented) tests/test_baggage.py::Test_Baggage_Headers_Basic: incomplete_test_app (/make_distant_call endpoint is not correctly implemented) diff --git a/manifests/java.yml b/manifests/java.yml index f56d208458c..8b516ff8e34 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -2942,6 +2942,7 @@ manifest: - declaration: bug (SCP-962) component_version: '>=1.5.0' tests/cws/test_thread_context_sharing.py::Test_ThreadContextSharing: missing_feature (missing /security/thread_context_sharing endpoint on weblog) + tests/debugger/test_debugger_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for Java; python-only for now) tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Line_Capture_Expressions: - weblog_declaration: "*": missing_feature @@ -4270,6 +4271,7 @@ manifest: "*": v0.0.0 spring-boot-3-native: irrelevant (/rasp/sqli endpoint is not available) tests/stats/test_stats.py::Test_Time_Bucketing::test_client_side_stats_bucket_alignment: missing_feature + tests/test_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for Java; python-only for now) tests/test_baggage.py::Test_Baggage_Headers_Api_Datadog: irrelevant (no Datadog API for W3C Baggage) tests/test_baggage.py::Test_Baggage_Headers_Api_OTel: - weblog_declaration: diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 0a412f144a5..e26f5c5c7de 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1650,6 +1650,7 @@ manifest: tests/auto_inject/test_auto_inject_install.py::TestHostAutoInjectInstallScriptAppsec: *ref_5_43_0 tests/auto_inject/test_auto_inject_install.py::TestSimpleInstallerAutoInjectManualAppsec: *ref_5_43_0 tests/cws/test_thread_context_sharing.py::Test_ThreadContextSharing: missing_feature (missing /security/thread_context_sharing endpoint on weblog) + tests/debugger/test_debugger_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for Node.js; python-only for now) tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Line_Capture_Expressions: - weblog_declaration: "*": irrelevant @@ -2732,6 +2733,7 @@ manifest: tests/stats/test_stats.py::Test_Time_Bucketing::test_client_side_stats_bucket_alignment: - component_version: ">=6.10.0" # may be before declaration: flaky (APMLP-1755) + tests/test_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for Node.js; python-only for now) tests/test_baggage.py::Test_Baggage_Headers_Api_Datadog: - weblog_declaration: "*": incomplete_test_app (endpoint not implemented) diff --git a/manifests/php.yml b/manifests/php.yml index 9d3215755be..196fb757a44 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -1055,6 +1055,7 @@ manifest: tests/auto_inject/test_auto_inject_install.py::TestSimpleInstallerAutoInjectManualAppsec: v1.9.0 tests/auto_inject/test_auto_inject_install.py::TestSimpleInstallerAutoInjectManualProfiling: v1.9.0 tests/cws/test_thread_context_sharing.py::Test_ThreadContextSharing: missing_feature (missing /security/thread_context_sharing endpoint on weblog) + tests/debugger/test_debugger_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for PHP; python-only for now) tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Line_Capture_Expressions: missing_feature (Not yet implemented) tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Method_Capture_Expressions: - weblog_declaration: @@ -1761,6 +1762,7 @@ manifest: tests/stats/test_stats.py::Test_Client_Stats_With_Client_Obfuscation_Disabled: missing_feature tests/stats/test_stats.py::Test_Error_Sampler: missing_feature (error traces dropped client-side under sampling; errors counted in stats but traces not sent) tests/stats/test_stats.py::Test_Stats_Service_Source: irrelevant (Only implemented for Java) + tests/test_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for PHP; python-only for now) tests/test_baggage.py::Test_Baggage_Headers_Api_Datadog: - weblog_declaration: laravel11x: incomplete_test_app diff --git a/manifests/ruby.yml b/manifests/ruby.yml index e36a3206854..81851b2096d 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -1787,6 +1787,7 @@ manifest: tests/auto_inject/test_auto_inject_install.py::TestHostAutoInjectInstallScriptAppsec: v2.19.0 tests/auto_inject/test_auto_inject_install.py::TestSimpleInstallerAutoInjectManualAppsec: v2.19.0 tests/cws/test_thread_context_sharing.py::Test_ThreadContextSharing: missing_feature (missing /security/thread_context_sharing endpoint on weblog) + tests/debugger/test_debugger_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for Ruby; python-only for now) tests/debugger/test_debugger_capture_expressions.py::Test_Debugger_Line_Capture_Expressions: - weblog_declaration: "*": irrelevant @@ -2779,6 +2780,7 @@ manifest: sinatra14: missing_feature rails52: missing_feature sinatra22: missing_feature + tests/test_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented for Ruby; python-only for now) tests/test_baggage.py::Test_Baggage_Headers_Api_Datadog: - weblog_declaration: "*": incomplete_test_app (endpoint not implemented) diff --git a/manifests/rust.yml b/manifests/rust.yml index 8743419d232..deadd279ba4 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -421,6 +421,7 @@ manifest: tests/stats/test_stats.py: '>=0.4.0' tests/stats/test_stats.py::Test_Client_Stats::test_grpc_status_code: missing_feature (weblog does not implement grpc endpoint) tests/stats/test_stats.py::Test_Stats_Service_Source::test_srv_src: irrelevant (Only implemented for Java) + tests/test_agentless.py: missing_feature (Agentless mode (DD_AGENTLESS_ENABLED) is not implemented in dd-trace-rs; python-only for now) tests/test_baggage.py::Test_Baggage_Headers_Api_Datadog: incomplete_test_app (/otel_drop_in_baggage_api_datadog endpoint not implemented) tests/test_baggage.py::Test_Baggage_Headers_Api_OTel: incomplete_test_app (/otel_drop_in_baggage_api_otel endpoint not implemented) tests/test_baggage.py::Test_Baggage_Headers_Max_Bytes: missing_feature (baggage maximum size is not configurable) From 5a847bc996730c275e4432741c6d539d5b1356ad Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 04:02:41 +0200 Subject: [PATCH 09/18] Strengthen agentless trace/stats tests with real payload assertions Test_Agentless_Trace_Submission and Test_Agentless_Stats only checked headers and non-empty content, not the actual shape of the trace/stats payloads. Add real assertions on captured payloads (root span for GET /: service, type, error, http.method, http.status_code; matching stats bucket entry: AgentHostname, ClientComputed, Service, Type, Hits, TopLevelHits, Errors), derived from and verified against live-captured traffic against the branch. Verified live: APM_TRACING_AGENTLESS still 3/3 passing with these assertions. --- tests/test_agentless.py | 53 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/tests/test_agentless.py b/tests/test_agentless.py index 424b8b24c03..0a3dd6052b3 100644 --- a/tests/test_agentless.py +++ b/tests/test_agentless.py @@ -1,8 +1,11 @@ """Direct-to-intake delivery (DD_AGENTLESS_ENABLED), without a Datadog Agent. -Endpoints below are derived from reading the tracer/libdatadog source on the (unmerged) -`bob/agentless-setting` branch of dd-trace-py, not from live-captured traffic. Re-confirm -against real proxy captures once the branch is buildable in this environment; see +Endpoints and payload shapes below were confirmed against a live weblog build of the +(unmerged) `bob/agentless-setting` branch of dd-trace-py: trace submission is plain JSON +(not the msgpack v0.4 format `interfaces.agent.get_spans_list()` parses), keyed by +`traces[].spans[]`, each span using the same field names as the agent-relayed format; +stats submission reuses the msgpack `ClientStatsPayload` shape byte-for-byte. Re-confirm +against real proxy captures if the branch's wire format changes before it merges; see utils/_context/_scenarios/agentless_endtoend.py and debugger_agentless.py for the scenario setup. """ @@ -20,6 +23,8 @@ RC_CONFIGURATIONS_PATH = "/api/v0.1/configurations" RC_HOST = "config.mock-intake.invalid" +ROOT_SPAN_RESOURCE = "GET /" + def _headers(request: dict) -> dict[str, str]: return {name.lower(): value for name, value in request["request"]["headers"]} @@ -29,6 +34,28 @@ def _requests_at(host: str, path: str) -> list[dict]: return [data for data in interfaces.datadog_direct.get_data(path) if data["host"] == host] +def _find_root_span(resource: str) -> dict | None: + """Search every captured trace-submission request for a root span with this resource.""" + for request in _requests_at(TRACE_SUBMISSION_HOST, TRACE_SUBMISSION_PATH): + for trace in request["request"]["content"].get("traces", []): + for span in trace.get("spans", []): + if span.get("parent_id") == "0000000000000000" and span.get("resource") == resource: + return span + return None + + +def _find_stats_entry(resource: str) -> dict | None: + """Search every captured stats request for a bucket entry with this resource.""" + for request in _requests_at(STATS_HOST, STATS_PATH): + content = request["request"]["content"] + for payload in content.get("Stats", []): + for bucket in payload.get("Stats", []): + for entry in bucket.get("Stats", []): + if entry.get("Resource") == resource: + return entry + return None + + @scenarios.apm_tracing_agentless @features.not_reported class Test_Agentless_Trace_Submission: @@ -52,6 +79,14 @@ def test_trace_submission(self): content = request["request"]["content"] assert content, "Trace submission request body is empty" + span = _find_root_span(ROOT_SPAN_RESOURCE) + assert span is not None, f"No root span with resource {ROOT_SPAN_RESOURCE!r} was captured" + assert span["service"] == "weblog" + assert span["type"] == "web" + assert span["error"] == 0 + assert span["meta"]["http.method"] == "GET" + assert span["meta"]["http.status_code"] == "200" + @scenarios.apm_tracing_agentless @features.client_side_stats_supported @@ -73,10 +108,22 @@ def test_stats(self): headers = _headers(request) assert headers["dd-api-key"] in {AGENTLESS_MOCK_API_KEY, "--redacted--"} + content = request["request"]["content"] + assert content["AgentHostname"] == "weblog" + assert content["ClientComputed"] is True + # Stats and traces are distinct payloads on distinct hosts/paths. trace_requests = _requests_at(TRACE_SUBMISSION_HOST, TRACE_SUBMISSION_PATH) assert request not in trace_requests + entry = _find_stats_entry(ROOT_SPAN_RESOURCE) + assert entry is not None, f"No stats entry with resource {ROOT_SPAN_RESOURCE!r} was captured" + assert entry["Service"] == "weblog" + assert entry["Type"] == "web" + assert entry["Hits"] >= 1 + assert entry["TopLevelHits"] >= 1 + assert entry["Errors"] == 0 + @scenarios.apm_tracing_agentless @features.remote_config_object_supported From a4c2cd8ada436f4db0d7fb053ad475ed778c61de Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 04:58:08 +0200 Subject: [PATCH 10/18] Meticulously assert required headers on agentless payloads Trace submission, stats, RC, debugger snapshot, and SymDB requests each now assert their full required header set (dd-api-key, content-type, user-agent, datadog-meta-* tracer identity, dd-evp-origin, etc.), not just dd-api-key/content-type as before. Caught a real discrepancy along the way: SymDB uploads share dd-evp-origin: agent-debugger with logs/snapshots/diagnostics rather than a distinct agent-symdb value, and don't send x-datadog-additional-tags at all - fixed the assertion to match confirmed live behavior. --- tests/debugger/test_debugger_agentless.py | 23 ++++++-- tests/test_agentless.py | 64 +++++++++++++++++++++-- 2 files changed, 78 insertions(+), 9 deletions(-) diff --git a/tests/debugger/test_debugger_agentless.py b/tests/debugger/test_debugger_agentless.py index 0da0be611bc..169be93295d 100644 --- a/tests/debugger/test_debugger_agentless.py +++ b/tests/debugger/test_debugger_agentless.py @@ -4,14 +4,17 @@ """Dynamic Instrumentation (probe upload/logs/snapshots) and Symbol DB, without a Datadog Agent. -Endpoints/shapes here are derived from reading the dd-trace-py/libdatadog source on the -(unmerged) `bob/agentless-setting` branch, not from live-captured traffic - see -tests/debugger/utils.py::AgentlessBaseDebuggerTest and +Endpoints and header shapes below were confirmed against a live weblog build of the +(unmerged) `bob/agentless-setting` branch of dd-trace-py: logs/snapshots, diagnostics, and +Symbol DB all share the same agentless debugger intake path and `dd-evp-origin: agent-debugger` +header - the native client doesn't distinguish sub-features via evp-origin the way agent-mode +does. See tests/debugger/utils.py::AgentlessBaseDebuggerTest and utils/_context/_scenarios/debugger_agentless.py for the scenario/interface wiring. """ import tests.debugger.utils as debugger from utils import features, scenarios +from utils._context._scenarios.agentless_endtoend import AGENTLESS_MOCK_API_KEY @features.debugger @@ -57,7 +60,11 @@ def test_log_method_snapshot(self): request = requests[-1] assert request["host"] == "debugger-intake.mock-intake.invalid" headers = {name.lower(): value for name, value in request["request"]["headers"]} - assert "dd-api-key" in headers + assert headers["dd-api-key"] in {AGENTLESS_MOCK_API_KEY, "--redacted--"} + assert headers["dd-evp-origin"] == "agent-debugger" + assert headers["content-type"].startswith("multipart/form-data") + assert headers["user-agent"].startswith("Tracer/") + assert "datadog-entity-id" in headers @features.debugger_symdb @@ -88,4 +95,10 @@ def test_symdb_upload(self): requests = list(self._symbols_interface.get_data(self._symbols_path)) assert len(requests) > 0, f"No request captured on {self._symbols_path}" headers = {name.lower(): value for name, value in requests[-1]["request"]["headers"]} - assert "dd-api-key" in headers + assert headers["dd-api-key"] in {AGENTLESS_MOCK_API_KEY, "--redacted--"} + # Symbol DB shares the same debugger intake path/evp-origin as logs/snapshots/diagnostics - + # the native client doesn't distinguish sub-features via dd-evp-origin. + assert headers["dd-evp-origin"] == "agent-debugger" + assert headers["content-type"].startswith("multipart/form-data") + assert headers["user-agent"].startswith("Tracer/") + assert "datadog-entity-id" in headers diff --git a/tests/test_agentless.py b/tests/test_agentless.py index 0a3dd6052b3..65fc8f3cfe9 100644 --- a/tests/test_agentless.py +++ b/tests/test_agentless.py @@ -30,6 +30,18 @@ def _headers(request: dict) -> dict[str, str]: return {name.lower(): value for name, value in request["request"]["headers"]} +def _assert_headers(headers: dict[str, str], *, exact: dict[str, str], present: tuple[str, ...]) -> None: + """Assert exact values for deterministic headers and mere presence for value-varying ones.""" + for name, value in exact.items(): + assert headers.get(name) == value, f"header {name!r}: expected {value!r}, got {headers.get(name)!r}" + for name in present: + assert name in headers, f"missing required header {name!r}" + + +def _assert_api_key(headers: dict[str, str]) -> None: + assert headers["dd-api-key"] in {AGENTLESS_MOCK_API_KEY, "--redacted--"} + + def _requests_at(host: str, path: str) -> list[dict]: return [data for data in interfaces.datadog_direct.get_data(path) if data["host"] == host] @@ -74,7 +86,25 @@ def test_trace_submission(self): assert request["response"]["status_code"] // 100 == 2 headers = _headers(request) - assert headers["dd-api-key"] in {AGENTLESS_MOCK_API_KEY, "--redacted--"} + _assert_api_key(headers) + _assert_headers( + headers, + exact={ + "content-type": "application/json", + "datadog-meta-lang": "python", + "datadog-meta-lang-interpreter": "CPython", + "datadog-client-computed-top-level": "true", + }, + present=( + "user-agent", + "datadog-meta-lang-version", + "datadog-meta-tracer-version", + "datadog-entity-id", + "x-datadog-trace-count", + "content-length", + ), + ) + assert headers["user-agent"].startswith("Tracer/") content = request["request"]["content"] assert content, "Trace submission request body is empty" @@ -106,7 +136,26 @@ def test_stats(self): assert request["response"]["status_code"] // 100 == 2 headers = _headers(request) - assert headers["dd-api-key"] in {AGENTLESS_MOCK_API_KEY, "--redacted--"} + _assert_api_key(headers) + _assert_headers( + headers, + exact={ + "content-type": "application/msgpack", + "datadog-meta-lang": "python", + "datadog-meta-lang-interpreter": "CPython", + }, + present=( + "user-agent", + "datadog-meta-lang-version", + "datadog-meta-tracer-version", + "datadog-entity-id", + "content-length", + ), + ) + assert headers["user-agent"].startswith("Tracer/") + # Stats has no top-level-computed/trace-count headers: those are trace-submission-only. + assert "datadog-client-computed-top-level" not in headers + assert "x-datadog-trace-count" not in headers content = request["request"]["content"] assert content["AgentHostname"] == "weblog" @@ -155,5 +204,12 @@ def test_remote_config_poll(self): assert request["method"] == "POST" headers = _headers(request) - assert headers["dd-api-key"] in {AGENTLESS_MOCK_API_KEY, "--redacted--"} - assert headers.get("content-type") == "application/x-protobuf" + _assert_api_key(headers) + _assert_headers( + headers, + exact={"content-type": "application/x-protobuf"}, + present=("user-agent", "datadog-entity-id", "content-length"), + ) + # The native RC client is driven by libdatadog directly, not the Python-level tracer, + # so it identifies itself distinctly (no datadog-meta-lang-* headers, unlike traces/stats). + assert headers["user-agent"].startswith("Libdatadog/") From db8b0e11febec6a6f79fd51b697cb057d20375e6 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 13:39:30 +0200 Subject: [PATCH 11/18] Verify zstd compression and multi-flush stats Sequence monotonicity Trace submission now asserts content-encoding: zstd and that the wire size is genuinely smaller than the decoded body, following an audit of the agentless Rust encoder against the real Agent that found compression had been silently disabled (now fixed on the dd-trace-py branch). New Test_Agentless_Stats_Multi_Flush polls for two same-runtime stats flushes and asserts ClientStatsPayload.Sequence increments by exactly 1 per flush - a guarantee agent-mode never gives a real signal for, since the Agent's own re-aggregation always resets Sequence to 0 on relay. --- tests/test_agentless.py | 60 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/test_agentless.py b/tests/test_agentless.py index 65fc8f3cfe9..8c6ec8b0343 100644 --- a/tests/test_agentless.py +++ b/tests/test_agentless.py @@ -9,6 +9,8 @@ utils/_context/_scenarios/agentless_endtoend.py and debugger_agentless.py for the scenario setup. """ +import time + from utils import features, interfaces, scenarios, weblog from utils._context._scenarios.agentless_endtoend import AGENTLESS_MOCK_API_KEY from utils._remote_config import send_apm_tracing_command @@ -68,6 +70,15 @@ def _find_stats_entry(resource: str) -> dict | None: return None +def _stats_runtime_id(request: dict) -> str | None: + payloads = request["request"]["content"].get("Stats", []) + return payloads[0]["RuntimeID"] if payloads else None + + +def _stats_requests_by_runtime(runtime_id: str) -> list[dict]: + return [r for r in _requests_at(STATS_HOST, STATS_PATH) if _stats_runtime_id(r) == runtime_id] + + @scenarios.apm_tracing_agentless @features.not_reported class Test_Agentless_Trace_Submission: @@ -91,6 +102,7 @@ def test_trace_submission(self): headers, exact={ "content-type": "application/json", + "content-encoding": "zstd", "datadog-meta-lang": "python", "datadog-meta-lang-interpreter": "CPython", "datadog-client-computed-top-level": "true", @@ -106,6 +118,16 @@ def test_trace_submission(self): ) assert headers["user-agent"].startswith("Tracer/") + # The proxy transparently decompresses the body for capture (request["request"]["length"] + # is the decompressed size); compare it against the real over-the-wire content-length + # header to confirm the payload was actually compressed, not just labeled as such. + wire_length = int(headers["content-length"]) + decoded_length = request["request"]["length"] + assert wire_length < decoded_length, ( + f"Trace submission body doesn't look compressed: {wire_length} wire bytes vs " + f"{decoded_length} decoded bytes" + ) + content = request["request"]["content"] assert content, "Trace submission request body is empty" @@ -174,6 +196,44 @@ def test_stats(self): assert entry["Errors"] == 0 +@scenarios.apm_tracing_agentless +@features.client_side_stats_supported +class Test_Agentless_Stats_Multi_Flush: + """Sequence increments by exactly 1 across successive flushes of the same runtime. + + Stats buckets are 10s wide; the agent-mode writer's own re-aggregation always resets + Sequence to 0 on every relayed payload (see pkg/trace/stats/client_stats_aggregator.go), + so this monotonic-Sequence guarantee is agentless-specific - the Agent never gave a + real signal here to compare against, which is exactly why it's easy to get wrong. + """ + + def setup_multi_flush_stats(self): + runtime_id = None + deadline = time.time() + 60 + while time.time() < deadline: + weblog.get("/") + requests = _requests_at(STATS_HOST, STATS_PATH) + if requests: + runtime_id = _stats_runtime_id(requests[-1]) + if runtime_id and len(_stats_requests_by_runtime(runtime_id)) >= 2: + break + time.sleep(2) + self.runtime_id = runtime_id + + def test_multi_flush_stats(self): + assert self.runtime_id, "No stats request was ever captured" + + requests = _stats_requests_by_runtime(self.runtime_id) + assert len(requests) >= 2, ( + f"Expected at least 2 stats flushes for runtime {self.runtime_id!r}, got {len(requests)}" + ) + + sequences = [r["request"]["content"]["Stats"][0]["Sequence"] for r in requests] + assert len(set(sequences)) == len(sequences), f"Sequence numbers are not unique: {sequences}" + for prev, cur in zip(sequences, sequences[1:]): + assert cur == prev + 1, f"Sequence should increment by exactly 1 per flush, got: {sequences}" + + @scenarios.apm_tracing_agentless @features.remote_config_object_supported class Test_Agentless_Remote_Config: From c0a013a4636a7b8142ab7c8d9164fed5f275b31e Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 27 Aug 2026 20:28:28 +0200 Subject: [PATCH 12/18] Contrain python to 4.15.0+ --- manifests/python.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/manifests/python.yml b/manifests/python.yml index b0f25ddfc80..ba1a2309cdb 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -2304,6 +2304,7 @@ manifest: python3.12: '>=4.3.1' uds-flask: '>=4.3.1' uwsgi-poc: '>=4.3.1' + tests/test_agentless.py: v4.15.0 tests/test_baggage.py::Test_Baggage_Headers_Api_Datadog: - weblog_declaration: "*": incomplete_test_app (endpoint not implemented) From c3c5e188a671aa63eb3dc47a8ce3c43b878705a8 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Fri, 28 Aug 2026 03:56:34 +0200 Subject: [PATCH 13/18] Add coverage for agentless instrumentation telemetry Telemetry (POST /api/v2/apmtelemetry) has no per-product transport setting of its own and simply follows the global DD_AGENTLESS_ENABLED switch - it was already flowing agentlessly and captured in every prior run, but had zero test coverage. Notably, this is the transport for generate-metrics events (internal tracer metrics like spans_created), which is distinct from DogStatsD-based runtime metrics (CPU/memory/GC) - those have no agentless transport at all on this branch and are silently dropped without an agent; out of scope for this commit. Caught and fixed a real setup-dispatch bug along the way: system-tests matches setup_ to test_ by exact name, so a single shared setup_telemetry method silently never ran for either test method - confirmed live via the pytest debug log that each test now independently triggers and waits for its own telemetry traffic. --- tests/test_agentless.py | 87 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/tests/test_agentless.py b/tests/test_agentless.py index 8c6ec8b0343..2b39e3e415f 100644 --- a/tests/test_agentless.py +++ b/tests/test_agentless.py @@ -25,6 +25,9 @@ RC_CONFIGURATIONS_PATH = "/api/v0.1/configurations" RC_HOST = "config.mock-intake.invalid" +TELEMETRY_PATH = "/api/v2/apmtelemetry" +TELEMETRY_HOST = "instrumentation-telemetry-intake.mock-intake.invalid" + ROOT_SPAN_RESOURCE = "GET /" @@ -79,6 +82,27 @@ def _stats_requests_by_runtime(runtime_id: str) -> list[dict]: return [r for r in _requests_at(STATS_HOST, STATS_PATH) if _stats_runtime_id(r) == runtime_id] +def _telemetry_events(request_type: str) -> list[dict]: + """Flatten every captured telemetry request into individual events, unwrapping message-batch.""" + events = [] + for request in _requests_at(TELEMETRY_HOST, TELEMETRY_PATH): + content = request["request"]["content"] + if content.get("request_type") == request_type: + events.append(content) + elif content.get("request_type") == "message-batch": + events.extend(p for p in content.get("payload", []) if p.get("request_type") == request_type) + return events + + +def _find_metric_series(metric: str, namespace: str) -> dict | None: + """Search every captured generate-metrics event for a series with this metric/namespace.""" + for event in _telemetry_events("generate-metrics"): + for series in event["payload"].get("series", []): + if series.get("metric") == metric and series.get("namespace") == namespace: + return series + return None + + @scenarios.apm_tracing_agentless @features.not_reported class Test_Agentless_Trace_Submission: @@ -234,6 +258,69 @@ def test_multi_flush_stats(self): assert cur == prev + 1, f"Sequence should increment by exactly 1 per flush, got: {sequences}" +@scenarios.apm_tracing_agentless +@features.not_reported +class Test_Agentless_Telemetry: + """Instrumentation telemetry - including generate-metrics, the actual transport for internal + tracer metrics like spans_created/spans_finished - has no transport setting of its own and + simply follows the global DD_AGENTLESS_ENABLED switch, going straight to its own dedicated + telemetry intake host instead of through an Agent. This is distinct from (and, on this + branch, NOT a substitute for) DogStatsD-based runtime metrics (CPU/memory/GC), which have no + agentless transport at all and are silently dropped without an Agent - out of scope here. + """ + + def _trigger_and_wait_for_metrics(self): + self.r = weblog.get("/") + interfaces.datadog_direct.wait_for( + lambda d: d["host"] == TELEMETRY_HOST + and d["path"] == TELEMETRY_PATH + and _find_metric_series("spans_created", "tracers") is not None, + timeout=30, + ) + + def setup_telemetry_endpoint(self): + self._trigger_and_wait_for_metrics() + + def test_telemetry_endpoint(self): + assert self.r.status_code == 200 + + requests = _requests_at(TELEMETRY_HOST, TELEMETRY_PATH) + assert len(requests) != 0, f"No request captured on {TELEMETRY_HOST}{TELEMETRY_PATH}" + + request = requests[-1] + assert request["response"]["status_code"] // 100 == 2 + + content = request["request"]["content"] + headers = _headers(request) + _assert_api_key(headers) + _assert_headers( + headers, + exact={"content-type": "application/json", "dd-client-library-language": "python"}, + present=( + "user-agent", + "dd-telemetry-request-type", + "dd-telemetry-api-version", + "dd-client-library-version", + "dd-session-id", + "datadog-entity-id", + "content-length", + ), + ) + assert headers["user-agent"].startswith("telemetry/") + assert headers["dd-telemetry-request-type"] == content["request_type"] + + def setup_telemetry_generate_metrics(self): + self._trigger_and_wait_for_metrics() + + def test_telemetry_generate_metrics(self): + series = _find_metric_series("spans_created", "tracers") + assert series is not None, "No tracers.spans_created generate-metrics event was captured" + assert series["type"] == "count" + assert series["common"] is True + assert series["points"], "Metric series has no data points" + assert series["points"][0][1] > 0 + + @scenarios.apm_tracing_agentless @features.remote_config_object_supported class Test_Agentless_Remote_Config: From dfacce7e1ba80c134ed968d98e53b2362946f210 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Fri, 28 Aug 2026 15:57:05 +0200 Subject: [PATCH 14/18] Add coverage for agentless crashtracker submission Crashtracker submission was completely broken agentlessly: a real triggered crash (SIGSEGV) produced zero network activity toward either agentless mechanism for 90+ seconds. Root-caused on the dd-trace-py branch (ddtrace/internal/core/crashtracking.py, ddtrace/internal/settings/crashtracker.py, uncommitted there) to two compounding bugs - a too-short collection timeout that got the collector SIGKILLed mid-symbolication, and the crash receiver subprocess's environment never forwarding HTTP(S)_PROXY, so it silently couldn't reach the mock intake through the proxy this harness requires. Test_Agentless_Crashtracking covers both agentless delivery mechanisms confirmed by a live capture after the fix: the dedicated errors-intake endpoint (POST .../api/v2/errorsintake) and a telemetry 'logs' event - both share the telemetry-intake host, just on different paths. Distinguishes the full crash report from crashtracker's lightweight 'ping' log by tags, matching the convention tests/parametric/test_crashtracking.py already uses for the agent-mode equivalent. The crash-triggering request needs a much longer client timeout than this project's 5s default: DD_CRASHTRACKING_WAIT_FOR_RECEIVER defaults to true, so the crashing process blocks in its own signal handler until the report actually finishes uploading (up to the now-correct 30s collector timeout) before it can exit - the parent's response, and therefore this HTTP call, legitimately takes that long. --- tests/test_agentless.py | 92 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/tests/test_agentless.py b/tests/test_agentless.py index 2b39e3e415f..beef5f7f0b1 100644 --- a/tests/test_agentless.py +++ b/tests/test_agentless.py @@ -9,6 +9,7 @@ utils/_context/_scenarios/agentless_endtoend.py and debugger_agentless.py for the scenario setup. """ +import json import time from utils import features, interfaces, scenarios, weblog @@ -28,6 +29,10 @@ TELEMETRY_PATH = "/api/v2/apmtelemetry" TELEMETRY_HOST = "instrumentation-telemetry-intake.mock-intake.invalid" +# Crash reports ride the same telemetry-intake host as everything else in this file - the +# errors-intake endpoint is just a different path on it, not a separate host, confirmed live. +CRASH_ERRORS_INTAKE_PATH = "/api/v2/errorsintake" + ROOT_SPAN_RESOURCE = "GET /" @@ -103,6 +108,22 @@ def _find_metric_series(metric: str, namespace: str) -> dict | None: return None +def _find_crash_report_log() -> dict | None: + """Search captured telemetry "logs" events for the full crash report. + + Crashtracker sends a lightweight "ping" log as soon as config/metadata are available, + then the full symbolicated report once collection finishes - distinguish them by tags + (only the ping carries is_crash_ping:true), matching the convention already used by + tests/parametric/test_crashtracking.py for the agent-mode equivalent. + """ + for event in _telemetry_events("logs"): + for log in event["payload"].get("logs", []): + tags = log.get("tags", "") + if "is_crash:true" in tags and "is_crash_ping:true" not in tags: + return log + return None + + @scenarios.apm_tracing_agentless @features.not_reported class Test_Agentless_Trace_Submission: @@ -321,6 +342,77 @@ def test_telemetry_generate_metrics(self): assert series["points"][0][1] > 0 +@scenarios.apm_tracing_agentless +@features.not_reported +class Test_Agentless_Crashtracking: + """A real crash report reaches the intake through two parallel agentless mechanisms: a + dedicated errors-intake endpoint, and a "logs" telemetry event - both ride the same + telemetry-intake host used by Test_Agentless_Telemetry, just on different paths. + + Confirmed against a fix for a real bug on the dd-trace-py branch: crash reports were + silently dropped agentlessly (too-short collection timeout, and the crash receiver + subprocess's environment not forwarding HTTP(S)_PROXY, so it could never reach the mock + intake through the proxy this test harness requires). If this test hangs/times out again, + that regression is the first thing to check. + """ + + def _trigger_crash_and_wait(self): + # wait_for_receiver defaults to true: the crashing child blocks in its own signal + # handler until the receiver finishes collecting/uploading (up to the collector + # timeout), and the parent's os.waitpid() - and thus this HTTP response - doesn't + # return until the child actually exits. Give it real headroom past the client's + # normal 5s default, or this legitimately times out on a slow/busy host. + self.r = weblog.get( + "/spawn_child", params={"sleep": 0, "crash": "true", "fork": "true"}, timeout=45 + ) + interfaces.datadog_direct.wait_for( + lambda d: d["host"] == TELEMETRY_HOST + and d["path"] == CRASH_ERRORS_INTAKE_PATH + and _find_crash_report_log() is not None, + timeout=90, + ) + + def setup_crash_report_errors_intake(self): + self._trigger_crash_and_wait() + + def test_crash_report_errors_intake(self): + assert self.r.status_code == 200 + + requests = _requests_at(TELEMETRY_HOST, CRASH_ERRORS_INTAKE_PATH) + assert len(requests) != 0, f"No request captured on {TELEMETRY_HOST}{CRASH_ERRORS_INTAKE_PATH}" + + request = requests[-1] + assert request["response"]["status_code"] // 100 == 2 + + headers = _headers(request) + _assert_api_key(headers) + _assert_headers( + headers, + exact={"content-type": "application/json"}, + present=("user-agent", "datadog-entity-id", "content-length"), + ) + assert headers["user-agent"].startswith("crashtracker/") + + content = request["request"]["content"] + assert content["ddsource"] == "crashtracker" + assert content["error"]["is_crash"] is True + assert content["error"]["type"] == "SIGSEGV" + assert content["sig_info"]["si_signo_human_readable"] == "SIGSEGV" + assert content["proc_info"]["pid"] > 0 + + def setup_crash_report_telemetry_logs(self): + self._trigger_crash_and_wait() + + def test_crash_report_telemetry_logs(self): + log = _find_crash_report_log() + assert log is not None, "No full crash-report telemetry logs event was captured" + + message = json.loads(log["message"]) + assert message["error"]["is_crash"] is True + assert message["error"]["kind"] == "UnixSignal" + assert "SIGSEGV" in message["error"]["message"] + + @scenarios.apm_tracing_agentless @features.remote_config_object_supported class Test_Agentless_Remote_Config: From e2fc1718fd3a040f2a3f2f54ff0fbd9f276d4ed6 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Wed, 2 Sep 2026 21:36:34 +0200 Subject: [PATCH 15/18] Fix mypy type errors flagged by CI on PR #7607 8 real type errors: an int assigned into a str-typed dict literal (tuf.py), a missing dict annotation and an attribute access on the base Scenario class that only EndToEndScenario actually has (_remote_config.py, defaulted via getattr), two subclass interface-attribute overrides incompatible with the base class's narrower inferred type (debugger/utils.py, fixed by annotating both as their shared ProxyBasedInterfaceValidator base), and mock_backend_status()'s object|None return type being indexed directly in two FFE tests (narrowed with isinstance checks). Also guards a third, newly-surfaced get_spans_list() call the same way the existing hasattr-gated one already is, now that widening _backend_interface's annotation makes mypy correctly see it's not always present. --- tests/debugger/utils.py | 11 +++++++++-- tests/ffe/test_agentless_configuration.py | 2 +- .../test_the_test/test_mock_ffe_agentless_backend.py | 2 +- utils/_remote_config.py | 4 ++-- utils/proxy/tuf.py | 2 +- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/debugger/utils.py b/tests/debugger/utils.py index 921beacf805..8fee09e5931 100644 --- a/tests/debugger/utils.py +++ b/tests/debugger/utils.py @@ -13,6 +13,7 @@ from collections.abc import Iterator from utils import interfaces, remote_config, weblog, context, logger +from utils.interfaces import ProxyBasedInterfaceValidator from utils.dd_constants import RemoteConfigApplyState as ApplyState from utils.dd_types import DataDogAgentSpan @@ -179,11 +180,11 @@ class BaseDebuggerTest: # Backend interface/paths, overridden by AgentlessBaseDebuggerTest for scenarios without a # Datadog Agent (data captured over interfaces.datadog_direct instead of interfaces.agent, # and logs/snapshots/diagnostics/symdb all collapse onto _AGENTLESS_DEBUGGER_PATH). - _backend_interface = interfaces.agent + _backend_interface: ProxyBasedInterfaceValidator = interfaces.agent _snapshot_paths: tuple[str, ...] = (_LOGS_PATH, _DEBUGGER_PATH) _traces_path: str = _TRACES_PATH _telemetry_path: str = _TELEMETRY_PATH - _symbols_interface = interfaces.library + _symbols_interface: ProxyBasedInterfaceValidator = interfaces.library _symbols_path: str = _SYMBOLS_PATH def initialize_weblog_remote_config(self) -> None: @@ -718,6 +719,12 @@ def _wait_for_no_capture_reason_span(self, data: dict): + self._error_message + "'" ) + if not hasattr(self._backend_interface, "get_spans_list"): + # Agentless mode's backend interface (interfaces.datadog_direct) captures direct-to-intake + # traffic in a different wire format than the agent-relayed /v0.4/traces payloads + # get_spans_list() parses, so this check is agent-only. + return None + spans = self._backend_interface.get_spans_list() for span in spans: meta = span.meta diff --git a/tests/ffe/test_agentless_configuration.py b/tests/ffe/test_agentless_configuration.py index fbc8b964fd3..40f845080ac 100644 --- a/tests/ffe/test_agentless_configuration.py +++ b/tests/ffe/test_agentless_configuration.py @@ -26,6 +26,6 @@ def test_default_agentless_source(self) -> None: assert json.loads(self.response.text)["value"] == "on-value" backend_status = scenarios.feature_flagging_and_experimentation_agentless.mock_backend_status() - assert backend_status is not None + assert isinstance(backend_status, dict) assert backend_status["requests_total"] >= 1 assert backend_status["last_path"] == CONFIG_PATH diff --git a/tests/test_the_test/test_mock_ffe_agentless_backend.py b/tests/test_the_test/test_mock_ffe_agentless_backend.py index d3568744b2b..3dc2c3e5995 100644 --- a/tests/test_the_test/test_mock_ffe_agentless_backend.py +++ b/tests/test_the_test/test_mock_ffe_agentless_backend.py @@ -122,7 +122,7 @@ def test_agentless_end_to_end_scenario_starts_backend_before_weblog() -> None: assert scenario.weblog_infra.library_container.extra_hosts == HOST_GATEWAY_EXTRA_HOSTS status = scenario.mock_backend_status() - assert status is not None + assert isinstance(status, dict) assert status["requests_total"] == 0 finally: scenario._stop_mock_backend() # noqa: SLF001 - focused lifecycle test diff --git a/utils/_remote_config.py b/utils/_remote_config.py index 5388505d7f0..c0970d9f0de 100644 --- a/utils/_remote_config.py +++ b/utils/_remote_config.py @@ -141,13 +141,13 @@ def send_state( "apply_error": "", } - state = {} + state: dict[str, Any] = {} # Agentless scenarios have no agent to relay client state via /v0.7/config: the native RC # client polls /api/v0.1/configurations directly and reports the same per-config apply state # inline on that request (LatestConfigsRequest.active_clients[0].state), instead of via a # separate follow-up request. - agentless = not context.scenario.include_agent + agentless = not getattr(context.scenario, "include_agent", True) watched_path = "/api/v0.1/configurations" if agentless else "/v0.7/config" def remote_config_applied(data: dict) -> bool: diff --git a/utils/proxy/tuf.py b/utils/proxy/tuf.py index cde699b1cd9..f59775d739f 100644 --- a/utils/proxy/tuf.py +++ b/utils/proxy/tuf.py @@ -155,7 +155,7 @@ def build_targets_content( The 'signed' portion of targets metadata (not yet wrapped with signatures) """ - custom = {"opaque_backend_state": opaque_backend_state} + custom: dict[str, str | int] = {"opaque_backend_state": opaque_backend_state} if agent_refresh_interval is not None: custom["agent_refresh_interval"] = agent_refresh_interval From 35e911a672e6d62af5be43e0a597e5b4d4ead0ff Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Wed, 2 Sep 2026 21:36:51 +0200 Subject: [PATCH 16/18] Address PR #7607 review feedback - Rename a locally-shadowing 'environment' variable to 'weblog_environment' in AgentlessEndToEndScenario.__init__, and merge two separate 'if capture_direct_egress:' blocks that were checking the exact same condition. - Default DD_API_KEY once in AgentlessEndToEndScenario itself instead of repeating the same value in every scenario declaration - safe no-op for FeatureFlaggingAgentlessEndToEndScenario, which already sets (and later re-sets, in _start_mock_backend) the identical value explicitly. Left DD_AGENTLESS_ENABLED un-defaulted: unlike DD_API_KEY, unconditionally defaulting it in the shared base would silently turn on dd-trace-py's real agentless tracer mode for FFE's scenario variants, which use a wholly separate agentless mechanism and don't want it. - Register the new utils/_context/_scenarios/debugger_agentless.py in utils/scripts/libraries_and_scenarios_rules.yml (scenario_groups: agentless), matching agentless_endtoend.py's existing entry - it was missing. - Replace @features.not_reported with real feature-parity IDs on the three test classes that had it: crashtracking (an exact existing match), trace submission and telemetry (no agentless-specific feature-parity page exists yet, so mapped to the closest existing ones - trace_data_integrity and telemetry_metrics_collected respectively; flagged this mapping for confirmation in the PR review reply since it's a judgment call, not a certainty). --- tests/test_agentless.py | 16 ++++++------- utils/_context/_scenarios/__init__.py | 8 +------ .../_context/_scenarios/agentless_endtoend.py | 24 +++++++++++-------- .../scripts/libraries_and_scenarios_rules.yml | 2 ++ 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/tests/test_agentless.py b/tests/test_agentless.py index beef5f7f0b1..6835f587f38 100644 --- a/tests/test_agentless.py +++ b/tests/test_agentless.py @@ -9,6 +9,7 @@ utils/_context/_scenarios/agentless_endtoend.py and debugger_agentless.py for the scenario setup. """ +import itertools import json import time @@ -125,7 +126,7 @@ def _find_crash_report_log() -> dict | None: @scenarios.apm_tracing_agentless -@features.not_reported +@features.trace_data_integrity class Test_Agentless_Trace_Submission: """Traces are sent directly to the intake, bypassing the Datadog Agent.""" @@ -169,8 +170,7 @@ def test_trace_submission(self): wire_length = int(headers["content-length"]) decoded_length = request["request"]["length"] assert wire_length < decoded_length, ( - f"Trace submission body doesn't look compressed: {wire_length} wire bytes vs " - f"{decoded_length} decoded bytes" + f"Trace submission body doesn't look compressed: {wire_length} wire bytes vs {decoded_length} decoded bytes" ) content = request["request"]["content"] @@ -275,12 +275,12 @@ def test_multi_flush_stats(self): sequences = [r["request"]["content"]["Stats"][0]["Sequence"] for r in requests] assert len(set(sequences)) == len(sequences), f"Sequence numbers are not unique: {sequences}" - for prev, cur in zip(sequences, sequences[1:]): + for prev, cur in itertools.pairwise(sequences): assert cur == prev + 1, f"Sequence should increment by exactly 1 per flush, got: {sequences}" @scenarios.apm_tracing_agentless -@features.not_reported +@features.telemetry_metrics_collected class Test_Agentless_Telemetry: """Instrumentation telemetry - including generate-metrics, the actual transport for internal tracer metrics like spans_created/spans_finished - has no transport setting of its own and @@ -343,7 +343,7 @@ def test_telemetry_generate_metrics(self): @scenarios.apm_tracing_agentless -@features.not_reported +@features.crashtracking class Test_Agentless_Crashtracking: """A real crash report reaches the intake through two parallel agentless mechanisms: a dedicated errors-intake endpoint, and a "logs" telemetry event - both ride the same @@ -362,9 +362,7 @@ def _trigger_crash_and_wait(self): # timeout), and the parent's os.waitpid() - and thus this HTTP response - doesn't # return until the child actually exits. Give it real headroom past the client's # normal 5s default, or this legitimately times out on a slow/busy host. - self.r = weblog.get( - "/spawn_child", params={"sleep": 0, "crash": "true", "fork": "true"}, timeout=45 - ) + self.r = weblog.get("/spawn_child", params={"sleep": 0, "crash": "true", "fork": "true"}, timeout=45) interfaces.datadog_direct.wait_for( lambda d: d["host"] == TELEMETRY_HOST and d["path"] == CRASH_ERRORS_INTAKE_PATH diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 6e4bffb2be9..2a434bd8f6c 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -7,11 +7,7 @@ from .aws_lambda import LambdaScenario from .core import Scenario, scenario_groups from .default import DefaultScenario -from .agentless_endtoend import ( - AGENTLESS_MOCK_API_KEY, - AgentlessEndToEndScenario, - FeatureFlaggingAgentlessEndToEndScenario, -) +from .agentless_endtoend import AgentlessEndToEndScenario, FeatureFlaggingAgentlessEndToEndScenario from .debugger_agentless import DebuggerAgentlessScenario from .endtoend import ( DockerScenario, @@ -873,7 +869,6 @@ class _Scenarios: rc_backend_enabled=True, weblog_env={ "DD_AGENTLESS_ENABLED": "true", - "DD_API_KEY": AGENTLESS_MOCK_API_KEY, "DD_TRACE_STATS_COMPUTATION_ENABLED": "true", "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "1", }, @@ -885,7 +880,6 @@ class _Scenarios: "Symbol DB, without a Datadog Agent.", weblog_env={ "DD_AGENTLESS_ENABLED": "true", - "DD_API_KEY": AGENTLESS_MOCK_API_KEY, "DD_DYNAMIC_INSTRUMENTATION_ENABLED": "true", "_DD_SYMBOL_DATABASE_FORCE_UPLOAD": "true", }, diff --git a/utils/_context/_scenarios/agentless_endtoend.py b/utils/_context/_scenarios/agentless_endtoend.py index 895db2d1e78..8b52c5752f0 100644 --- a/utils/_context/_scenarios/agentless_endtoend.py +++ b/utils/_context/_scenarios/agentless_endtoend.py @@ -50,18 +50,22 @@ def __init__( ) -> None: self.capture_direct_egress = capture_direct_egress - environment: dict[str, str | None] = dict(weblog_env or {}) + weblog_environment: dict[str, str | None] = dict(weblog_env or {}) + # Every current caller of DD_API_KEY here wants the same mock value, so default it + # once instead of repeating it in each scenario declaration (an explicit weblog_env + # entry, like FeatureFlaggingAgentlessEndToEndScenario's, still wins). + weblog_environment.setdefault("DD_API_KEY", AGENTLESS_MOCK_API_KEY) + + weblog_volumes: dict | None = None if capture_direct_egress: - environment.setdefault( + weblog_environment.setdefault( # The reserved .invalid domain fails closed if a request bypasses the proxy. "DD_SITE", "mock-intake.invalid", ) - environment.setdefault("DD_PROXY_HTTPS", f"http://proxy:{ProxyPorts.datadog_direct}") - environment.setdefault("HTTPS_PROXY", f"http://proxy:{ProxyPorts.datadog_direct}") + weblog_environment.setdefault("DD_PROXY_HTTPS", f"http://proxy:{ProxyPorts.datadog_direct}") + weblog_environment.setdefault("HTTPS_PROXY", f"http://proxy:{ProxyPorts.datadog_direct}") - weblog_volumes: dict | None = None - if capture_direct_egress: # The weblog talks HTTPS directly to the proxy (CONNECT tunnel), which # terminates TLS with the mitmproxy CA -- same trust anchor already # mounted into AgentContainer (utils/_context/containers.py) for its @@ -80,9 +84,9 @@ def __init__( # same env vars/root JSON already used to make the real Agent trust the test # TUF key (utils/_context/containers.py::AgentContainer.__init__). tuf_root_json = get_tuf_root_json() - environment.setdefault("DD_REMOTE_CONFIGURATION_ENABLED", "true") - environment.setdefault("DD_REMOTE_CONFIGURATION_CONFIG_ROOT", tuf_root_json) - environment.setdefault("DD_REMOTE_CONFIGURATION_DIRECTOR_ROOT", tuf_root_json) + weblog_environment.setdefault("DD_REMOTE_CONFIGURATION_ENABLED", "true") + weblog_environment.setdefault("DD_REMOTE_CONFIGURATION_CONFIG_ROOT", tuf_root_json) + weblog_environment.setdefault("DD_REMOTE_CONFIGURATION_DIRECTOR_ROOT", tuf_root_json) super().__init__( name, @@ -95,7 +99,7 @@ def __init__( use_proxy_for_weblog=capture_direct_egress, rc_api_enabled=rc_api_enabled, rc_backend_enabled=rc_backend_enabled, - weblog_env=environment, + weblog_env=weblog_environment, weblog_volumes=weblog_volumes, ) diff --git a/utils/scripts/libraries_and_scenarios_rules.yml b/utils/scripts/libraries_and_scenarios_rules.yml index 8b7f5683c4b..938927a6ebe 100644 --- a/utils/scripts/libraries_and_scenarios_rules.yml +++ b/utils/scripts/libraries_and_scenarios_rules.yml @@ -224,6 +224,8 @@ patterns: scenario_groups: appsec_rasp_scenario utils/_context/_scenarios/agentless_endtoend.py: scenario_groups: agentless + utils/_context/_scenarios/debugger_agentless.py: + scenario_groups: agentless utils/_context/_scenarios/endtoend.py: scenario_groups: end_to_end utils/_context/_scenarios/integration_frameworks.py: From 90dd0ee5fb2ca145d35934c33011d6e74682ef45 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Wed, 2 Sep 2026 21:57:50 +0200 Subject: [PATCH 17/18] Add real dd_agentless_enabled feature (id 644) for trace submission and telemetry cbeauchesne added a new DD_AGENTLESS_ENABLED entry (feature=644, owned by language-platform) to the feature-parity dashboard specifically for this - trace_data_integrity and telemetry_metrics_collected were both approximate stand-ins, not correct matches. Test_Agentless_Crashtracking keeps @features.crashtracking (317), which was already an exact existing match. --- tests/test_agentless.py | 4 ++-- utils/_features.py | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_agentless.py b/tests/test_agentless.py index 6835f587f38..e09b2ef43e8 100644 --- a/tests/test_agentless.py +++ b/tests/test_agentless.py @@ -126,7 +126,7 @@ def _find_crash_report_log() -> dict | None: @scenarios.apm_tracing_agentless -@features.trace_data_integrity +@features.dd_agentless_enabled class Test_Agentless_Trace_Submission: """Traces are sent directly to the intake, bypassing the Datadog Agent.""" @@ -280,7 +280,7 @@ def test_multi_flush_stats(self): @scenarios.apm_tracing_agentless -@features.telemetry_metrics_collected +@features.dd_agentless_enabled class Test_Agentless_Telemetry: """Instrumentation telemetry - including generate-metrics, the actual transport for internal tracer metrics like spans_created/spans_finished - has no transport setting of its own and diff --git a/utils/_features.py b/utils/_features.py index 71c67c69c1f..8be9539e80a 100644 --- a/utils/_features.py +++ b/utils/_features.py @@ -2129,6 +2129,14 @@ def crashtracking(test_object): """ return _mark_test_object(test_object, feature_id=317, owner=_Owner.language_platform) + @staticmethod + def dd_agentless_enabled(test_object): + """DD_AGENTLESS_ENABLED + + https://feature-parity.us1.prod.dog/#/?feature=644 + """ + return _mark_test_object(test_object, feature_id=644, owner=_Owner.language_platform) + @staticmethod def rasp_shell_injection(test_object): """Appsec RASP rule : shell injection From 5677be480f11ca57ce442b8d0c03e1a42852b351 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 3 Sep 2026 20:00:58 +0200 Subject: [PATCH 18/18] Ensure Otel logs / metrics point to the intake by default when agentless is enabled --- tests/test_agentless.py | 120 ++++++++++++++++++ utils/_context/_scenarios/__init__.py | 7 +- .../_context/_scenarios/agentless_endtoend.py | 1 + utils/build/docker/python/flask/app.py | 16 +++ utils/proxy/_deserializer.py | 13 +- 5 files changed, 152 insertions(+), 5 deletions(-) diff --git a/tests/test_agentless.py b/tests/test_agentless.py index e09b2ef43e8..17cc652c024 100644 --- a/tests/test_agentless.py +++ b/tests/test_agentless.py @@ -34,6 +34,13 @@ # errors-intake endpoint is just a different path on it, not a separate host, confirmed live. CRASH_ERRORS_INTAKE_PATH = "/api/v2/errorsintake" +# OpenTelemetry log/metric export are wholly separate agentless mechanisms (their own +# DD_SITE-derived host/flags), unrelated to the DD_AGENTLESS_ENABLED trace/stats/RC/telemetry +# paths above. +OTLP_HOST = "otlp.mock-intake.invalid" +OTLP_LOGS_PATH = "/v1/logs" +OTLP_METRICS_PATH = "/v1/metrics" + ROOT_SPAN_RESOURCE = "GET /" @@ -125,6 +132,33 @@ def _find_crash_report_log() -> dict | None: return None +def _find_otlp_log_record(body_substring: str) -> dict | None: + """Search every captured OTLP log-export request for a record whose body contains this text.""" + for request in _requests_at(OTLP_HOST, OTLP_LOGS_PATH): + content = request["request"]["content"] + for resource_log in content.get("resourceLogs", []): + for scope_log in resource_log.get("scopeLogs", []): + for record in scope_log.get("logRecords", []): + if body_substring in record.get("body", {}).get("stringValue", ""): + return record + return None + + +def _find_otlp_metric_datapoint(metric_name: str) -> dict | None: + """Search every captured OTLP metric-export request for a data point of this metric.""" + for request in _requests_at(OTLP_HOST, OTLP_METRICS_PATH): + content = request["request"]["content"] + for resource_metric in content.get("resourceMetrics", []): + for scope_metric in resource_metric.get("scopeMetrics", []): + for metric in scope_metric.get("metrics", []): + if metric.get("name") != metric_name: + continue + data_points = metric.get("sum", {}).get("dataPoints", []) + if data_points: + return data_points[0] + return None + + @scenarios.apm_tracing_agentless @features.dd_agentless_enabled class Test_Agentless_Trace_Submission: @@ -411,6 +445,92 @@ def test_crash_report_telemetry_logs(self): assert "SIGSEGV" in message["error"]["message"] +@scenarios.apm_tracing_agentless +@features.dd_agentless_enabled +class Test_Agentless_OTLP_Logs: + """Ordinary Python `logging` calls are forwarded as OTLP log records directly to the + intake (https://otlp./v1/logs) when DD_LOGS_OTEL_ENABLED is set, with no Datadog + Agent involved. + """ + + LOG_MESSAGE_MARKER = "[DSM] Got request with integration: None" + + def setup_otlp_log_export(self): + self.r = weblog.get("/dsm") + interfaces.datadog_direct.wait_for( + lambda d: d["host"] == OTLP_HOST + and d["path"] == OTLP_LOGS_PATH + and _find_otlp_log_record(self.LOG_MESSAGE_MARKER) is not None, + timeout=30, + ) + + def test_otlp_log_export(self): + requests = _requests_at(OTLP_HOST, OTLP_LOGS_PATH) + assert len(requests) != 0, f"No request captured on {OTLP_HOST}{OTLP_LOGS_PATH}" + + request = requests[-1] + assert request["response"]["status_code"] // 100 == 2 + + headers = _headers(request) + _assert_api_key(headers) + _assert_headers( + headers, + exact={"content-type": "application/x-protobuf"}, + present=("user-agent", "content-length"), + ) + + record = _find_otlp_log_record(self.LOG_MESSAGE_MARKER) + assert record is not None, f"No OTLP log record containing {self.LOG_MESSAGE_MARKER!r} was captured" + assert record["severityText"] == "INFO" + assert record["body"]["stringValue"] == self.LOG_MESSAGE_MARKER + assert "traceId" in record + assert "spanId" in record + + +@scenarios.apm_tracing_agentless +@features.dd_agentless_enabled +class Test_Agentless_OTLP_Metrics: + """A real, user-created OTel metric (via the standard OTel Metrics API - `GET /otel_create_metric` + calls `opentelemetry.metrics.get_meter(...).create_counter(...).add(...)` in the weblog) is + exported directly to the intake (https://otlp./v1/metrics) when DD_METRICS_OTEL_ENABLED + is set, with no Datadog Agent involved. + """ + + METRIC_NAME = "system_tests.otel_metric" + + def setup_otlp_metric_export(self): + self.r = weblog.get("/otel_create_metric") + interfaces.datadog_direct.wait_for( + lambda d: d["host"] == OTLP_HOST + and d["path"] == OTLP_METRICS_PATH + and _find_otlp_metric_datapoint(self.METRIC_NAME) is not None, + timeout=30, + ) + + def test_otlp_metric_export(self): + assert self.r.status_code == 200 + + requests = _requests_at(OTLP_HOST, OTLP_METRICS_PATH) + assert len(requests) != 0, f"No request captured on {OTLP_HOST}{OTLP_METRICS_PATH}" + + request = requests[-1] + assert request["response"]["status_code"] // 100 == 2 + + headers = _headers(request) + _assert_api_key(headers) + _assert_headers( + headers, + exact={"content-type": "application/x-protobuf"}, + present=("user-agent", "content-length"), + ) + + data_point = _find_otlp_metric_datapoint(self.METRIC_NAME) + assert data_point is not None, f"No OTLP data point for metric {self.METRIC_NAME!r} was captured" + assert int(data_point.get("asInt", data_point.get("asDouble", 0))) >= 1 + attributes = {a["key"]: a["value"] for a in data_point.get("attributes", [])} + assert attributes.get("system_tests.metric_test", {}).get("stringValue") == "true" + + @scenarios.apm_tracing_agentless @features.remote_config_object_supported class Test_Agentless_Remote_Config: diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 2a434bd8f6c..80ed1d14970 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -863,14 +863,17 @@ class _Scenarios: apm_tracing_agentless = AgentlessEndToEndScenario( "APM_TRACING_AGENTLESS", - doc="Validate direct-to-intake trace submission, client-side stats, and Remote " - "Configuration when DD_AGENTLESS_ENABLED is set, without a Datadog Agent.", + doc="Validate direct-to-intake trace submission, client-side stats, Remote " + "Configuration, and OpenTelemetry log/metric export when DD_AGENTLESS_ENABLED is set, " + "without a Datadog Agent.", rc_api_enabled=True, rc_backend_enabled=True, weblog_env={ "DD_AGENTLESS_ENABLED": "true", "DD_TRACE_STATS_COMPUTATION_ENABLED": "true", "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "1", + "DD_LOGS_OTEL_ENABLED": "true", + "DD_METRICS_OTEL_ENABLED": "true", }, ) diff --git a/utils/_context/_scenarios/agentless_endtoend.py b/utils/_context/_scenarios/agentless_endtoend.py index 8b52c5752f0..fef2518db54 100644 --- a/utils/_context/_scenarios/agentless_endtoend.py +++ b/utils/_context/_scenarios/agentless_endtoend.py @@ -65,6 +65,7 @@ def __init__( ) weblog_environment.setdefault("DD_PROXY_HTTPS", f"http://proxy:{ProxyPorts.datadog_direct}") weblog_environment.setdefault("HTTPS_PROXY", f"http://proxy:{ProxyPorts.datadog_direct}") + weblog_environment.setdefault("REQUESTS_CA_BUNDLE", "/etc/ssl/certs/ca-certificates.crt") # The weblog talks HTTPS directly to the proxy (CONNECT tunnel), which # terminates TLS with the mitmproxy CA -- same trust anchor already diff --git a/utils/build/docker/python/flask/app.py b/utils/build/docker/python/flask/app.py index fd5817be260..d629c61b3d8 100644 --- a/utils/build/docker/python/flask/app.py +++ b/utils/build/docker/python/flask/app.py @@ -65,6 +65,7 @@ import stripe import opentelemetry.baggage import opentelemetry.context +import opentelemetry.metrics import opentelemetry.propagate import opentelemetry.trace @@ -1967,6 +1968,21 @@ def otel_drop_in_default_propagator_inject(): return jsonify(result) +@app.route("/otel_create_metric", methods=["GET"]) +def otel_create_metric(): + """Record a value via the standard OTel Metrics API (opentelemetry.metrics.get_meter(...)). + + Unrelated to dd-trace-py's own runtime metrics (DD_RUNTIME_METRICS_ENABLED), which are never + fed into the OTel meter - this exercises a genuine, user-created OTel metric instead, so its + export can be verified independently of that gap. + """ + meter = opentelemetry.metrics.get_meter("system-tests") + counter = meter.create_counter("system_tests.otel_metric", description="system-tests OTel metrics test") + counter.add(1, {"system_tests.metric_test": "true"}) + + return "OK" + + # From https://github.com/open-telemetry/opentelemetry-python/issues/2432#issuecomment-1742425474 # This context manager handles correctly managing context with repeated baggage operations @contextlib.contextmanager diff --git a/utils/proxy/_deserializer.py b/utils/proxy/_deserializer.py index 7a2a96a8221..07b9307f531 100644 --- a/utils/proxy/_deserializer.py +++ b/utils/proxy/_deserializer.py @@ -189,8 +189,15 @@ def json_load(): if content_type == "application/x-protobuf": assert isinstance(content, bytes) + # dd-protocol: otlp is set by the agent's own OTLP relay path, which doesn't carry the + # exact /v1/{traces,metrics,logs} path this direct-to-intake capture does. Real OTLP + # exporters (dd-trace-py's included) never set that header themselves, so also decode + # as a *Request* whenever this is actually a request body on one of those exact paths - + # otherwise it would fall through to the unconditional (response-shaped) branches below + # and silently mis-decode direct-to-intake OTLP request payloads. dd_protocol = get_header_value("dd-protocol", message["headers"]) - if dd_protocol == "otlp" and "traces" in path: + is_direct_otlp_request = key == "request" and path in ("/v1/traces", "/v1/metrics", "/v1/logs") + if (dd_protocol == "otlp" and "traces" in path) or (is_direct_otlp_request and path == "/v1/traces"): return deserialize_otlp_v1_trace( MessageToDict( ExportTraceServiceRequest.FromString(content), @@ -198,13 +205,13 @@ def json_load(): use_integers_for_enums=True, ) ) - if dd_protocol == "otlp" and "metrics" in path: + if (dd_protocol == "otlp" and "metrics" in path) or (is_direct_otlp_request and path == "/v1/metrics"): return MessageToDict( ExportMetricsServiceRequest.FromString(content), preserving_proto_field_name=False, use_integers_for_enums=True, ) - if dd_protocol == "otlp" and "logs" in path: + if (dd_protocol == "otlp" and "logs" in path) or (is_direct_otlp_request and path == "/v1/logs"): return MessageToDict( ExportLogsServiceRequest.FromString(content), preserving_proto_field_name=False,