From 01b74c992a0414382d0af58ceb46874730a94c14 Mon Sep 17 00:00:00 2001 From: Lucas Soares Date: Thu, 19 Mar 2026 19:16:00 -0300 Subject: [PATCH 01/13] Support for console traces --- .../core/telemetry/auto_instrument.py | 22 +++--- .../unit/telemetry/test_auto_instrument.py | 77 +++++++++++++++---- 2 files changed, 77 insertions(+), 22 deletions(-) diff --git a/src/sap_cloud_sdk/core/telemetry/auto_instrument.py b/src/sap_cloud_sdk/core/telemetry/auto_instrument.py index 13acd55..d7749de 100644 --- a/src/sap_cloud_sdk/core/telemetry/auto_instrument.py +++ b/src/sap_cloud_sdk/core/telemetry/auto_instrument.py @@ -2,6 +2,7 @@ import os from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.trace.export import ConsoleSpanExporter from traceloop.sdk import Traceloop from sap_cloud_sdk.core.telemetry import Module, Operation @@ -22,24 +23,27 @@ def auto_instrument(): """ Initialize meta-instrumentation for GenAI tracing. Should be initialized before any AI frameworks. - Traces are exported to the OTEL collector endpoint configured in environment with OTEL_EXPORTER_OTLP_ENDPOINT. - - Args: + Traces are exported to the OTEL collector endpoint configured in environment with + OTEL_EXPORTER_OTLP_ENDPOINT, or printed to console when OTEL_TRACES_EXPORTER=console. """ otel_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT") + console_traces = os.getenv("OTEL_TRACES_EXPORTER", "").lower() == "console" - if not otel_endpoint: + if not otel_endpoint and not console_traces: logger.warning( "OTEL_EXPORTER_OTLP_ENDPOINT not set. Instrumentation will be disabled." ) return - if "v1/traces" not in otel_endpoint: - otel_endpoint = otel_endpoint.rstrip("/") + "/v1/traces" - - logger.info(f"Initializing auto instrumentation with endpoint: {otel_endpoint}") + if console_traces: + logger.info("Initializing auto instrumentation with console exporter") + base_exporter = ConsoleSpanExporter() + else: + if "v1/traces" not in otel_endpoint: + otel_endpoint = otel_endpoint.rstrip("/") + "/v1/traces" + logger.info(f"Initializing auto instrumentation with endpoint: {otel_endpoint}") + base_exporter = OTLPSpanExporter(endpoint=otel_endpoint) - base_exporter = OTLPSpanExporter(endpoint=otel_endpoint) exporter = GenAIAttributeTransformer(base_exporter) resource = create_resource_attributes_from_env() diff --git a/tests/core/unit/telemetry/test_auto_instrument.py b/tests/core/unit/telemetry/test_auto_instrument.py index f13e33f..b5b00ce 100644 --- a/tests/core/unit/telemetry/test_auto_instrument.py +++ b/tests/core/unit/telemetry/test_auto_instrument.py @@ -14,6 +14,7 @@ def mock_traceloop_components(): mocks = { 'traceloop': stack.enter_context(patch('sap_cloud_sdk.core.telemetry.auto_instrument.Traceloop')), 'exporter': stack.enter_context(patch('sap_cloud_sdk.core.telemetry.auto_instrument.OTLPSpanExporter')), + 'console_exporter': stack.enter_context(patch('sap_cloud_sdk.core.telemetry.auto_instrument.ConsoleSpanExporter')), 'transformer': stack.enter_context(patch('sap_cloud_sdk.core.telemetry.auto_instrument.GenAIAttributeTransformer')), 'create_resource': stack.enter_context(patch('sap_cloud_sdk.core.telemetry.auto_instrument.create_resource_attributes_from_env')), 'get_app_name': stack.enter_context(patch('sap_cloud_sdk.core.telemetry.auto_instrument._get_app_name')), @@ -24,17 +25,6 @@ def mock_traceloop_components(): class TestAutoInstrument: """Test suite for auto_instrument function.""" - def test_auto_instrument_without_endpoint(self): - """Test that auto_instrument warns when OTEL_EXPORTER_OTLP_ENDPOINT is not set.""" - with patch.dict('os.environ', {}, clear=True): - with patch('sap_cloud_sdk.core.telemetry.auto_instrument.logger') as mock_logger: - auto_instrument() - - # Should log warning about missing endpoint - mock_logger.warning.assert_called_once() - warning_message = mock_logger.warning.call_args[0][0] - assert "OTEL_EXPORTER_OTLP_ENDPOINT not set" in warning_message - def test_auto_instrument_with_endpoint_success(self, mock_traceloop_components): """Test successful auto-instrumentation with valid endpoint.""" mock_traceloop_components['get_app_name'].return_value = 'test-app' @@ -146,12 +136,73 @@ def test_auto_instrument_legacy_schema_parameter_ignored(self, mock_traceloop_co """Test that legacy_schema parameter is accepted but doesn't affect behavior.""" mock_traceloop_components['get_app_name'].return_value = 'test-app' mock_traceloop_components['create_resource'].return_value = {} - + with patch.dict('os.environ', {'OTEL_EXPORTER_OTLP_ENDPOINT': 'http://localhost:4317'}, clear=True): # Should not raise an error auto_instrument() auto_instrument() auto_instrument() - + # Verify Traceloop was initialized each time assert mock_traceloop_components['traceloop'].init.call_count == 3 + + def test_auto_instrument_with_console_exporter(self, mock_traceloop_components): + """Test that auto_instrument uses ConsoleSpanExporter when OTEL_TRACES_EXPORTER=console.""" + mock_traceloop_components['get_app_name'].return_value = 'test-app' + mock_traceloop_components['create_resource'].return_value = {} + + with patch.dict('os.environ', {'OTEL_TRACES_EXPORTER': 'console'}, clear=True): + auto_instrument() + + mock_traceloop_components['console_exporter'].assert_called_once_with() + mock_traceloop_components['exporter'].assert_not_called() + mock_traceloop_components['traceloop'].init.assert_called_once() + + def test_auto_instrument_console_exporter_case_insensitive(self, mock_traceloop_components): + """Test that OTEL_TRACES_EXPORTER=console matching is case insensitive.""" + mock_traceloop_components['get_app_name'].return_value = 'test-app' + mock_traceloop_components['create_resource'].return_value = {} + + for value in ['CONSOLE', 'Console', 'CONSOLE']: + mock_traceloop_components['console_exporter'].reset_mock() + mock_traceloop_components['traceloop'].reset_mock() + with patch.dict('os.environ', {'OTEL_TRACES_EXPORTER': value}, clear=True): + auto_instrument() + mock_traceloop_components['console_exporter'].assert_called_once_with() + + def test_auto_instrument_console_wins_when_both_set(self, mock_traceloop_components): + """Test that console exporter is used when OTEL_TRACES_EXPORTER=console, even if OTLP endpoint is also set.""" + mock_traceloop_components['get_app_name'].return_value = 'test-app' + mock_traceloop_components['create_resource'].return_value = {} + + with patch.dict('os.environ', { + 'OTEL_EXPORTER_OTLP_ENDPOINT': 'http://localhost:4317', + 'OTEL_TRACES_EXPORTER': 'console', + }, clear=True): + auto_instrument() + + mock_traceloop_components['console_exporter'].assert_called_once_with() + mock_traceloop_components['exporter'].assert_not_called() + + def test_auto_instrument_console_wraps_with_transformer(self, mock_traceloop_components): + """Test that ConsoleSpanExporter is wrapped with GenAIAttributeTransformer.""" + mock_traceloop_components['get_app_name'].return_value = 'test-app' + mock_traceloop_components['create_resource'].return_value = {} + mock_console_instance = MagicMock() + mock_traceloop_components['console_exporter'].return_value = mock_console_instance + + with patch.dict('os.environ', {'OTEL_TRACES_EXPORTER': 'console'}, clear=True): + auto_instrument() + + mock_traceloop_components['transformer'].assert_called_once_with(mock_console_instance) + + def test_auto_instrument_without_endpoint_or_console(self): + """Test that auto_instrument warns when neither OTLP endpoint nor console exporter is configured.""" + with patch.dict('os.environ', {}, clear=True): + with patch('sap_cloud_sdk.core.telemetry.auto_instrument.logger') as mock_logger: + auto_instrument() + + mock_logger.warning.assert_called_once() + warning_message = mock_logger.warning.call_args[0][0] + assert "OTEL_EXPORTER_OTLP_ENDPOINT not set" in warning_message + From b0c44f71f3e08a189d2b9cded99b3341df382e9e Mon Sep 17 00:00:00 2001 From: Lucas Soares Date: Thu, 19 Mar 2026 19:22:10 -0300 Subject: [PATCH 02/13] Support for console traces --- src/sap_cloud_sdk/core/telemetry/auto_instrument.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sap_cloud_sdk/core/telemetry/auto_instrument.py b/src/sap_cloud_sdk/core/telemetry/auto_instrument.py index d7749de..ccc6e87 100644 --- a/src/sap_cloud_sdk/core/telemetry/auto_instrument.py +++ b/src/sap_cloud_sdk/core/telemetry/auto_instrument.py @@ -27,7 +27,7 @@ def auto_instrument(): OTEL_EXPORTER_OTLP_ENDPOINT, or printed to console when OTEL_TRACES_EXPORTER=console. """ otel_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT") - console_traces = os.getenv("OTEL_TRACES_EXPORTER", "").lower() == "console" + console_traces = os.getenv("OTEL_TRACES_EXPORTER", "") == "console" if not otel_endpoint and not console_traces: logger.warning( From 4d7bffde5c2b4bc2f6c548c21a26948f089f38b3 Mon Sep 17 00:00:00 2001 From: Lucas Soares Date: Thu, 19 Mar 2026 19:25:39 -0300 Subject: [PATCH 03/13] Support for console traces --- src/sap_cloud_sdk/core/telemetry/auto_instrument.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sap_cloud_sdk/core/telemetry/auto_instrument.py b/src/sap_cloud_sdk/core/telemetry/auto_instrument.py index ccc6e87..6a124e9 100644 --- a/src/sap_cloud_sdk/core/telemetry/auto_instrument.py +++ b/src/sap_cloud_sdk/core/telemetry/auto_instrument.py @@ -27,7 +27,7 @@ def auto_instrument(): OTEL_EXPORTER_OTLP_ENDPOINT, or printed to console when OTEL_TRACES_EXPORTER=console. """ otel_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT") - console_traces = os.getenv("OTEL_TRACES_EXPORTER", "") == "console" + console_traces = os.getenv("OTEL_TRACES_EXPORTER") == "console" if not otel_endpoint and not console_traces: logger.warning( From d9970536145a5ed0632c5cb34bfe16801d148bca Mon Sep 17 00:00:00 2001 From: Lucas Soares Date: Thu, 19 Mar 2026 19:29:39 -0300 Subject: [PATCH 04/13] Case insensitiveness --- src/sap_cloud_sdk/core/telemetry/auto_instrument.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sap_cloud_sdk/core/telemetry/auto_instrument.py b/src/sap_cloud_sdk/core/telemetry/auto_instrument.py index 6a124e9..d7749de 100644 --- a/src/sap_cloud_sdk/core/telemetry/auto_instrument.py +++ b/src/sap_cloud_sdk/core/telemetry/auto_instrument.py @@ -27,7 +27,7 @@ def auto_instrument(): OTEL_EXPORTER_OTLP_ENDPOINT, or printed to console when OTEL_TRACES_EXPORTER=console. """ otel_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT") - console_traces = os.getenv("OTEL_TRACES_EXPORTER") == "console" + console_traces = os.getenv("OTEL_TRACES_EXPORTER", "").lower() == "console" if not otel_endpoint and not console_traces: logger.warning( From 47e8078c3a5855e3aac694543fda36aeeadc5538 Mon Sep 17 00:00:00 2001 From: Lucas Soares Date: Thu, 19 Mar 2026 19:36:05 -0300 Subject: [PATCH 05/13] User guide --- src/sap_cloud_sdk/core/telemetry/user-guide.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/sap_cloud_sdk/core/telemetry/user-guide.md b/src/sap_cloud_sdk/core/telemetry/user-guide.md index 8e6d3c1..f8cedc6 100644 --- a/src/sap_cloud_sdk/core/telemetry/user-guide.md +++ b/src/sap_cloud_sdk/core/telemetry/user-guide.md @@ -139,7 +139,15 @@ For production environments, you should ensure that `OTEL_EXPORTER_OTLP_ENDPOINT ### Local Development -Set the OpenTelemetry collector endpoint: +To print traces directly to the console without an OTLP collector, set: + +```bash +export OTEL_TRACES_EXPORTER=console +``` + +Then call `auto_instrument()` as usual — traces will be printed to stdout. + +To use an OTLP collector instead: ```bash export OTEL_EXPORTER_OTLP_ENDPOINT="https://otel-collector.example.com" From e20fcfbfdc94a59f816f0f2f87e8affbf5308815 Mon Sep 17 00:00:00 2001 From: Lucas Soares Date: Thu, 19 Mar 2026 19:47:24 -0300 Subject: [PATCH 06/13] Default string for type safety --- src/sap_cloud_sdk/core/telemetry/auto_instrument.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sap_cloud_sdk/core/telemetry/auto_instrument.py b/src/sap_cloud_sdk/core/telemetry/auto_instrument.py index d7749de..d3f1ee4 100644 --- a/src/sap_cloud_sdk/core/telemetry/auto_instrument.py +++ b/src/sap_cloud_sdk/core/telemetry/auto_instrument.py @@ -26,7 +26,7 @@ def auto_instrument(): Traces are exported to the OTEL collector endpoint configured in environment with OTEL_EXPORTER_OTLP_ENDPOINT, or printed to console when OTEL_TRACES_EXPORTER=console. """ - otel_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT") + otel_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") console_traces = os.getenv("OTEL_TRACES_EXPORTER", "").lower() == "console" if not otel_endpoint and not console_traces: From 50023614019da55a241d8f92f6913d76133eaa97 Mon Sep 17 00:00:00 2001 From: Lucas Soares Date: Fri, 20 Mar 2026 10:44:10 -0300 Subject: [PATCH 07/13] feat: add genai operation name to context overlay span attributes --- src/sap_cloud_sdk/core/telemetry/tracer.py | 1 + tests/core/unit/telemetry/test_tracer.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sap_cloud_sdk/core/telemetry/tracer.py b/src/sap_cloud_sdk/core/telemetry/tracer.py index 0faa4b1..4f99528 100644 --- a/src/sap_cloud_sdk/core/telemetry/tracer.py +++ b/src/sap_cloud_sdk/core/telemetry/tracer.py @@ -106,6 +106,7 @@ async def handle_request(): # Add tenant_id if set span_attrs = attributes.copy() if attributes else {} + span_attrs[_ATTR_GEN_AI_OPERATION_NAME] = span_name tenant_id = get_tenant_id() if tenant_id: span_attrs[ATTR_SAP_TENANT_ID] = tenant_id diff --git a/tests/core/unit/telemetry/test_tracer.py b/tests/core/unit/telemetry/test_tracer.py index c9a420c..76d5ab8 100644 --- a/tests/core/unit/telemetry/test_tracer.py +++ b/tests/core/unit/telemetry/test_tracer.py @@ -54,7 +54,9 @@ def mock_start_as_current_span(name, kind=None, attributes=None): with context_overlay(GenAIOperation.CHAT, attributes=custom_attrs): pass - assert captured_attributes == custom_attrs + assert captured_attributes["user.id"] == "123" + assert captured_attributes["session.id"] == "abc" + assert captured_attributes["gen_ai.operation.name"] == "chat" def test_context_overlay_with_different_span_kinds(self): """Test context overlay with different span kinds.""" From 123a124555fd817c6b7ac7c8fd6a650187542427 Mon Sep 17 00:00:00 2001 From: Lucas Soares Date: Fri, 20 Mar 2026 11:27:51 -0300 Subject: [PATCH 08/13] feat: add attribute propagation opt-in --- src/sap_cloud_sdk/core/telemetry/telemetry.py | 26 ++++ src/sap_cloud_sdk/core/telemetry/tracer.py | 136 ++++++++++-------- .../core/telemetry/user-guide.md | 49 ++++++- tests/core/unit/telemetry/test_tracer.py | 109 ++++++++++++++ 4 files changed, 261 insertions(+), 59 deletions(-) diff --git a/src/sap_cloud_sdk/core/telemetry/telemetry.py b/src/sap_cloud_sdk/core/telemetry/telemetry.py index e3211d2..ca27ac3 100644 --- a/src/sap_cloud_sdk/core/telemetry/telemetry.py +++ b/src/sap_cloud_sdk/core/telemetry/telemetry.py @@ -4,6 +4,7 @@ """ import logging +from contextlib import contextmanager from contextvars import ContextVar from typing import Optional, Dict, Any @@ -31,6 +32,9 @@ # Context variable for per-request tenant ID _tenant_id_var: ContextVar[str] = ContextVar("tenant_id", default="") +# Context variable for propagated span attributes +_propagated_attrs_var: ContextVar[Dict[str, Any]] = ContextVar("propagated_attrs", default={}) + def set_tenant_id(tenant_id: str) -> None: """Set the tenant ID for the current request context. @@ -78,6 +82,28 @@ def get_tenant_id() -> str: return _tenant_id_var.get() +def get_propagated_attributes() -> Dict[str, Any]: + """Get the propagated span attributes from the current context. + + Returns: + Dict of attributes propagated from an ancestor span with propagate=True, + or an empty dict if none are set. + """ + return _propagated_attrs_var.get() + + +@contextmanager +def _propagate_attributes(attrs: Dict[str, Any]): + """Internal: push attrs onto the propagation stack for the duration of the context.""" + current = _propagated_attrs_var.get() + merged = {**current, **attrs} + token = _propagated_attrs_var.set(merged) + try: + yield + finally: + _propagated_attrs_var.reset(token) + + def record_request_metric( module: Module, source: Optional[Module], operation: str, deprecated: bool = False ) -> None: diff --git a/src/sap_cloud_sdk/core/telemetry/tracer.py b/src/sap_cloud_sdk/core/telemetry/tracer.py index 4f99528..fc685c7 100644 --- a/src/sap_cloud_sdk/core/telemetry/tracer.py +++ b/src/sap_cloud_sdk/core/telemetry/tracer.py @@ -6,14 +6,14 @@ by auto_instrument(). """ -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from typing import Optional, Dict, Any from opentelemetry import trace from opentelemetry.trace import Status, StatusCode, Span from sap_cloud_sdk.core.telemetry.genai_operation import GenAIOperation -from sap_cloud_sdk.core.telemetry.telemetry import get_tenant_id +from sap_cloud_sdk.core.telemetry.telemetry import get_tenant_id, get_propagated_attributes, _propagate_attributes from sap_cloud_sdk.core.telemetry.constants import ATTR_SAP_TENANT_ID # OpenTelemetry GenAI semantic attribute names (avoid duplicate string literals) @@ -36,6 +36,7 @@ def context_overlay( *, attributes: Optional[Dict[str, Any]] = None, kind: trace.SpanKind = trace.SpanKind.INTERNAL, + propagate: bool = False, ): """ Create a context overlay for tracing GenAI operations. @@ -53,6 +54,8 @@ def context_overlay( (e.g., {"user.id": "123", "session.id": "abc"}) kind: Span kind - usually INTERNAL for application code. Other options: SERVER, CLIENT, PRODUCER, CONSUMER + propagate: If True, this span's attributes are passed to all nested spans + within its scope as the lowest-priority layer. Yields: The created span (available for advanced use cases like adding events) @@ -104,23 +107,26 @@ async def handle_request(): # Convert enum to string if needed span_name = str(name) - # Add tenant_id if set - span_attrs = attributes.copy() if attributes else {} + # Merge propagated attrs (lowest priority), then user attrs, then required attrs + propagated = get_propagated_attributes() + span_attrs = {**propagated, **(attributes or {})} span_attrs[_ATTR_GEN_AI_OPERATION_NAME] = span_name tenant_id = get_tenant_id() if tenant_id: span_attrs[ATTR_SAP_TENANT_ID] = tenant_id - with tracer.start_as_current_span( - span_name, kind=kind, attributes=span_attrs - ) as span: - try: - yield span - except Exception as e: - # Record the exception in the span - span.set_status(Status(StatusCode.ERROR, str(e))) - span.record_exception(e) - raise + ctx = _propagate_attributes(span_attrs) if propagate else nullcontext() + with ctx: + with tracer.start_as_current_span( + span_name, kind=kind, attributes=span_attrs + ) as span: + try: + yield span + except Exception as e: + # Record the exception in the span + span.set_status(Status(StatusCode.ERROR, str(e))) + span.record_exception(e) + raise @contextmanager @@ -131,6 +137,7 @@ def chat_span( conversation_id: Optional[str] = None, server_address: Optional[str] = None, attributes: Optional[Dict[str, Any]] = None, + propagate: bool = False, ): """ Create a span for LLM chat/completion API calls (OpenTelemetry GenAI Inference span). @@ -148,6 +155,8 @@ def chat_span( (e.g. thread or session ID). Set as gen_ai.conversation.id when provided. server_address: Optional server address. If None, server.address is not set. attributes: Optional dict of extra attributes to add or override on the span. + propagate: If True, this span's attributes are passed to all nested spans + within its scope as the lowest-priority layer. Yields: The created Span (e.g. to set gen_ai.usage.input_tokens, gen_ai.response.finish_reason). @@ -187,20 +196,23 @@ def chat_span( tenant_id = get_tenant_id() if tenant_id: base_attrs[ATTR_SAP_TENANT_ID] = tenant_id - # User attributes first, then base_attrs so required semantic keys are never overridden - span_attrs = {**(attributes or {}), **base_attrs} - - with tracer.start_as_current_span( - span_name, - kind=trace.SpanKind.CLIENT, - attributes=span_attrs, - ) as span: - try: - yield span - except Exception as e: - span.set_status(Status(StatusCode.ERROR, str(e))) - span.record_exception(e) - raise + # Propagated attrs (lowest), user attrs, required semantic keys (highest) + propagated = get_propagated_attributes() + span_attrs = {**propagated, **(attributes or {}), **base_attrs} + + ctx = _propagate_attributes(span_attrs) if propagate else nullcontext() + with ctx: + with tracer.start_as_current_span( + span_name, + kind=trace.SpanKind.CLIENT, + attributes=span_attrs, + ) as span: + try: + yield span + except Exception as e: + span.set_status(Status(StatusCode.ERROR, str(e))) + span.record_exception(e) + raise @contextmanager @@ -210,6 +222,7 @@ def execute_tool_span( tool_type: Optional[str] = None, tool_description: Optional[str] = None, attributes: Optional[Dict[str, Any]] = None, + propagate: bool = False, ): """ Create a span for tool execution in agentic workflows (OpenTelemetry GenAI Execute Tool span). @@ -223,6 +236,8 @@ def execute_tool_span( tool_type: Optional tool type (e.g. "function"). tool_description: Optional tool description. attributes: Optional dict of extra attributes to add or override on the span. + propagate: If True, this span's attributes are passed to all nested spans + within its scope as the lowest-priority layer. Yields: The created Span (e.g. to set gen_ai.tool.call.result after execution). @@ -251,20 +266,23 @@ def execute_tool_span( tenant_id = get_tenant_id() if tenant_id: base_attrs[ATTR_SAP_TENANT_ID] = tenant_id - # User attributes first, then base_attrs so required semantic keys are never overridden - span_attrs = {**(attributes or {}), **base_attrs} - - with tracer.start_as_current_span( - span_name, - kind=trace.SpanKind.INTERNAL, - attributes=span_attrs, - ) as span: - try: - yield span - except Exception as e: - span.set_status(Status(StatusCode.ERROR, str(e))) - span.record_exception(e) - raise + # Propagated attrs (lowest), user attrs, required semantic keys (highest) + propagated = get_propagated_attributes() + span_attrs = {**propagated, **(attributes or {}), **base_attrs} + + ctx = _propagate_attributes(span_attrs) if propagate else nullcontext() + with ctx: + with tracer.start_as_current_span( + span_name, + kind=trace.SpanKind.INTERNAL, + attributes=span_attrs, + ) as span: + try: + yield span + except Exception as e: + span.set_status(Status(StatusCode.ERROR, str(e))) + span.record_exception(e) + raise @contextmanager @@ -278,6 +296,7 @@ def invoke_agent_span( server_address: Optional[str] = None, kind: trace.SpanKind = trace.SpanKind.CLIENT, attributes: Optional[Dict[str, Any]] = None, + propagate: bool = False, ): """ Create a span for GenAI agent invocation (OpenTelemetry GenAI Invoke agent span). @@ -299,6 +318,8 @@ def invoke_agent_span( server_address: Optional server address. If None, server.address is not set. kind: Span kind; CLIENT for remote agents, INTERNAL for in-process. attributes: Optional dict of extra attributes to add or override on the span. + propagate: If True, this span's attributes are passed to all nested spans + within its scope as the lowest-priority layer. Yields: The created Span (e.g. to set usage, response attributes). @@ -338,20 +359,23 @@ def invoke_agent_span( tenant_id = get_tenant_id() if tenant_id: base_attrs[ATTR_SAP_TENANT_ID] = tenant_id - # User attributes first, then base_attrs so required semantic keys are never overridden - span_attrs = {**(attributes or {}), **base_attrs} - - with tracer.start_as_current_span( - span_name, - kind=kind, - attributes=span_attrs, - ) as span: - try: - yield span - except Exception as e: - span.set_status(Status(StatusCode.ERROR, str(e))) - span.record_exception(e) - raise + # Propagated attrs (lowest), user attrs, required semantic keys (highest) + propagated = get_propagated_attributes() + span_attrs = {**propagated, **(attributes or {}), **base_attrs} + + ctx = _propagate_attributes(span_attrs) if propagate else nullcontext() + with ctx: + with tracer.start_as_current_span( + span_name, + kind=kind, + attributes=span_attrs, + ) as span: + try: + yield span + except Exception as e: + span.set_status(Status(StatusCode.ERROR, str(e))) + span.record_exception(e) + raise def get_current_span() -> Span: diff --git a/src/sap_cloud_sdk/core/telemetry/user-guide.md b/src/sap_cloud_sdk/core/telemetry/user-guide.md index f8cedc6..3565a5e 100644 --- a/src/sap_cloud_sdk/core/telemetry/user-guide.md +++ b/src/sap_cloud_sdk/core/telemetry/user-guide.md @@ -4,7 +4,6 @@ - **Auto-instruments AI frameworks** - automatic tracing - **Creates custom spans** - wrap your code to trace operations and add context -- **Records metrics** - token usage for LLM calls - **Tracks tenant IDs** - per-request tenant tracking in traces and metrics ## How to use it @@ -61,7 +60,7 @@ Nest spans for complex workflows: ```python with context_overlay(GenAIOperation.RETRIEVAL): documents = retrieve_documents(query) - + with context_overlay(GenAIOperation.CHAT): response = llm.chat(messages=[ {"role": "system", "content": f"Context: {documents}"}, @@ -69,7 +68,51 @@ with context_overlay(GenAIOperation.RETRIEVAL): ]) ``` -Add events to spans: +### Propagate attributes to child spans + +Use `propagate=True` to automatically pass attributes from a parent span to all child spans within its scope. This is useful for cross-cutting attributes like `gen_ai.conversation.id`, `gen_ai.agent.name`, or custom keys that should appear on every nested span without repeating them. + +```python +with context_overlay( + GenAIOperation.INVOKE_AGENT, + attributes={"gen_ai.conversation.id": "conv-123", "user.id": "u-456"}, + propagate=True +): + # Both child spans automatically receive gen_ai.conversation.id and user.id + with chat_span(model="gpt-4", provider="openai") as span: + response = client.chat.completions.create(...) + + with execute_tool_span(tool_name="get_weather"): + result = call_weather_api(location) +``` + +All four span functions support `propagate=True`: `context_overlay`, `chat_span`, `execute_tool_span`, and `invoke_agent_span`. + +**Priority rules** — child spans always win over propagated values (highest to lowest): +1. Required semantic keys set by the span function (e.g. `gen_ai.operation.name`) — always wins +2. User-provided `attributes` on the child span +3. Propagated attrs from ancestors — lowest priority, easily overridden + +```python +# Even if the parent propagated gen_ai.operation.name="invoke_agent", +# the child chat_span always sets it to "chat" +with invoke_agent_span(provider="openai", propagate=True): + with chat_span(model="gpt-4", provider="openai") as span: + pass # span has gen_ai.operation.name="chat", not "invoke_agent" +``` + +Propagation is scoped: once the `propagate=True` span exits, its attributes are no longer passed to subsequent sibling spans. + +Nesting multiple `propagate=True` spans accumulates attributes from all levels: + +```python +with context_overlay(GenAIOperation.INVOKE_AGENT, attributes={"session": "s1"}, propagate=True): + with chat_span("gpt-4", "openai", attributes={"turn": "1"}, propagate=True): + with execute_tool_span("search"): + pass # receives both session="s1" and turn="1" +``` + +### Add events to spans ```python with context_overlay(GenAIOperation.EMBEDDINGS) as span: diff --git a/tests/core/unit/telemetry/test_tracer.py b/tests/core/unit/telemetry/test_tracer.py index 76d5ab8..4468a98 100644 --- a/tests/core/unit/telemetry/test_tracer.py +++ b/tests/core/unit/telemetry/test_tracer.py @@ -872,3 +872,112 @@ def mock_start_as_current_span(name, kind=None, attributes=None): span.add_event("agent_step") mock_span.add_event.assert_called_once_with("agent_step") + + +class TestPropagate: + """Test suite for propagate=True attribute propagation across nested spans.""" + + def _make_tracer(self, mock_span, capture_list): + """Helper: return a mock tracer that records (name, attributes) into capture_list.""" + mock_tracer = MagicMock() + + @contextmanager + def mock_start_as_current_span(name, kind=None, attributes=None): + capture_list.append({"name": name, "attributes": dict(attributes or {})}) + yield mock_span + + mock_tracer.start_as_current_span = mock_start_as_current_span + return mock_tracer + + def test_context_overlay_propagate_passes_attributes_to_child_span(self): + """Parent context_overlay(propagate=True) attrs appear on nested context_overlay.""" + mock_span = MagicMock() + captures = [] + mock_tracer = self._make_tracer(mock_span, captures) + + with patch('opentelemetry.trace.get_tracer', return_value=mock_tracer): + with context_overlay(GenAIOperation.CHAT, attributes={"x": 1}, propagate=True): + with context_overlay(GenAIOperation.EMBEDDINGS): + pass + + child_attrs = captures[1]["attributes"] + assert child_attrs.get("x") == 1 + + def test_chat_span_propagate_passes_attributes_to_child_span(self): + """chat_span(propagate=True) attrs appear on a nested context_overlay.""" + mock_span = MagicMock() + captures = [] + mock_tracer = self._make_tracer(mock_span, captures) + + with patch('opentelemetry.trace.get_tracer', return_value=mock_tracer): + with chat_span("gpt-4", "openai", attributes={"agent.name": "bot"}, propagate=True): + with context_overlay(GenAIOperation.CHAT): + pass + + child_attrs = captures[1]["attributes"] + assert child_attrs.get("agent.name") == "bot" + + def test_propagate_does_not_override_child_base_attrs(self): + """Propagated gen_ai.operation.name from parent does not override child's own value.""" + mock_span = MagicMock() + captures = [] + mock_tracer = self._make_tracer(mock_span, captures) + + with patch('opentelemetry.trace.get_tracer', return_value=mock_tracer): + with context_overlay(GenAIOperation.CHAT, propagate=True): + # parent sets gen_ai.operation.name="chat"; child should keep its own + with chat_span("gpt-4", "openai"): + pass + + child_attrs = captures[1]["attributes"] + assert child_attrs["gen_ai.operation.name"] == "chat" + assert child_attrs["gen_ai.request.model"] == "gpt-4" + + def test_propagate_false_does_not_pollute_sibling_spans(self): + """After a propagate=True span exits, its attrs don't appear on a sibling span.""" + mock_span = MagicMock() + captures = [] + mock_tracer = self._make_tracer(mock_span, captures) + + with patch('opentelemetry.trace.get_tracer', return_value=mock_tracer): + with context_overlay(GenAIOperation.CHAT, attributes={"secret": "yes"}, propagate=True): + pass + # sibling — propagation should be reset after the first span exits + with context_overlay(GenAIOperation.EMBEDDINGS): + pass + + sibling_attrs = captures[1]["attributes"] + assert "secret" not in sibling_attrs + + def test_propagate_accumulates_across_nested_levels(self): + """Three levels deep, each with propagate=True; deepest gets attrs from all parents.""" + mock_span = MagicMock() + captures = [] + mock_tracer = self._make_tracer(mock_span, captures) + + with patch('opentelemetry.trace.get_tracer', return_value=mock_tracer): + with context_overlay(GenAIOperation.CHAT, attributes={"level": "1", "from_l1": "yes"}, propagate=True): + with context_overlay(GenAIOperation.EMBEDDINGS, attributes={"level": "2", "from_l2": "yes"}, propagate=True): + with context_overlay(GenAIOperation.RETRIEVAL): + pass + + deepest_attrs = captures[2]["attributes"] + # from_l1 comes from level 1, from_l2 comes from level 2 + assert deepest_attrs.get("from_l1") == "yes" + assert deepest_attrs.get("from_l2") == "yes" + # level was set by both; level 2 overwrites level 1 in propagation stack + assert deepest_attrs.get("level") == "2" + + def test_propagate_default_is_false(self): + """Without propagate param, child spans do not receive parent custom attrs.""" + mock_span = MagicMock() + captures = [] + mock_tracer = self._make_tracer(mock_span, captures) + + with patch('opentelemetry.trace.get_tracer', return_value=mock_tracer): + with context_overlay(GenAIOperation.CHAT, attributes={"custom": "val"}): + with context_overlay(GenAIOperation.EMBEDDINGS): + pass + + child_attrs = captures[1]["attributes"] + assert "custom" not in child_attrs From ea6a1bad37da852525c0c47d3d8461d71959158b Mon Sep 17 00:00:00 2001 From: Lucas Soares Date: Fri, 20 Mar 2026 13:21:28 -0300 Subject: [PATCH 09/13] chore: moved internal function to approriate file --- src/sap_cloud_sdk/core/telemetry/telemetry.py | 13 ------------- src/sap_cloud_sdk/core/telemetry/tracer.py | 14 +++++++++++++- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/sap_cloud_sdk/core/telemetry/telemetry.py b/src/sap_cloud_sdk/core/telemetry/telemetry.py index ca27ac3..c69ba6a 100644 --- a/src/sap_cloud_sdk/core/telemetry/telemetry.py +++ b/src/sap_cloud_sdk/core/telemetry/telemetry.py @@ -4,7 +4,6 @@ """ import logging -from contextlib import contextmanager from contextvars import ContextVar from typing import Optional, Dict, Any @@ -92,18 +91,6 @@ def get_propagated_attributes() -> Dict[str, Any]: return _propagated_attrs_var.get() -@contextmanager -def _propagate_attributes(attrs: Dict[str, Any]): - """Internal: push attrs onto the propagation stack for the duration of the context.""" - current = _propagated_attrs_var.get() - merged = {**current, **attrs} - token = _propagated_attrs_var.set(merged) - try: - yield - finally: - _propagated_attrs_var.reset(token) - - def record_request_metric( module: Module, source: Optional[Module], operation: str, deprecated: bool = False ) -> None: diff --git a/src/sap_cloud_sdk/core/telemetry/tracer.py b/src/sap_cloud_sdk/core/telemetry/tracer.py index fc685c7..40d6ff6 100644 --- a/src/sap_cloud_sdk/core/telemetry/tracer.py +++ b/src/sap_cloud_sdk/core/telemetry/tracer.py @@ -13,7 +13,7 @@ from opentelemetry.trace import Status, StatusCode, Span from sap_cloud_sdk.core.telemetry.genai_operation import GenAIOperation -from sap_cloud_sdk.core.telemetry.telemetry import get_tenant_id, get_propagated_attributes, _propagate_attributes +from sap_cloud_sdk.core.telemetry.telemetry import get_tenant_id, get_propagated_attributes, _propagated_attrs_var from sap_cloud_sdk.core.telemetry.constants import ATTR_SAP_TENANT_ID # OpenTelemetry GenAI semantic attribute names (avoid duplicate string literals) @@ -30,6 +30,18 @@ _ATTR_SERVER_ADDRESS = "server.address" +@contextmanager +def _propagate_attributes(attrs: Dict[str, Any]): + """Push attrs onto the propagation stack for the duration of the context.""" + current = _propagated_attrs_var.get() + merged = {**current, **attrs} + token = _propagated_attrs_var.set(merged) + try: + yield + finally: + _propagated_attrs_var.reset(token) + + @contextmanager def context_overlay( name: GenAIOperation, From 2774d31c039f0f7d8ac9984853d9d6604217fb82 Mon Sep 17 00:00:00 2001 From: Lucas Soares Date: Fri, 20 Mar 2026 16:26:15 -0300 Subject: [PATCH 10/13] docs: trimm docs for conciseness --- src/sap_cloud_sdk/core/telemetry/tracer.py | 65 ++-------------------- 1 file changed, 5 insertions(+), 60 deletions(-) diff --git a/src/sap_cloud_sdk/core/telemetry/tracer.py b/src/sap_cloud_sdk/core/telemetry/tracer.py index 40d6ff6..70e6636 100644 --- a/src/sap_cloud_sdk/core/telemetry/tracer.py +++ b/src/sap_cloud_sdk/core/telemetry/tracer.py @@ -53,12 +53,6 @@ def context_overlay( """ Create a context overlay for tracing GenAI operations. - Works in both sync and async code. The span is automatically closed - when exiting the context, and exceptions are automatically recorded. - This context manager integrates seamlessly with the auto-instrumentation - provided by auto_instrument(), allowing you to create parent spans that - wrap auto-instrumented AI framework calls. - Args: name: GenAI operation name following OpenTelemetry semantic conventions. Example: GenAIOperation.CHAT, GenAIOperation.EMBEDDINGS @@ -72,47 +66,11 @@ def context_overlay( Yields: The created span (available for advanced use cases like adding events) - Examples: - Basic GenAI operation: - ```python - from sap_cloud_sdk.core.telemetry import context_overlay, GenAIOperation - - with context_overlay(GenAIOperation.CHAT): - response = llm.chat(message) - ``` - - With custom attributes: + Example: ```python - with context_overlay( - name=GenAIOperation.CHAT, - attributes={"user.id": "123", "session.id": "abc"} - ): + with context_overlay(GenAIOperation.CHAT, attributes={"user.id": "123"}): response = llm.chat(message) ``` - - In async code (works the same): - ```python - async def handle_request(): - with context_overlay(GenAIOperation.CHAT): - result = await llm.chat_async(message) - ``` - - Nested spans: - ```python - with context_overlay(GenAIOperation.RETRIEVAL): - documents = retrieve_documents(query) - - with context_overlay(GenAIOperation.CHAT): - response = llm.chat(documents) - ``` - - Advanced usage with span events: - ```python - with context_overlay(GenAIOperation.EMBEDDINGS) as span: - span.add_event("processing_started") - embeddings = generate_embeddings(text) - span.add_event("processing_completed") - ``` """ tracer = trace.get_tracer(__name__) @@ -173,24 +131,11 @@ def chat_span( Yields: The created Span (e.g. to set gen_ai.usage.input_tokens, gen_ai.response.finish_reason). - Examples: - Agentic workflow with chat and tool execution: + Example: ```python - from sap_cloud_sdk.core.telemetry import chat_span, execute_tool_span - - with chat_span(model="gpt-4", provider="openai") as span: - response = client.chat.completions.create( - messages=[{"role": "user", "content": "What's the weather?"}], - tools=[weather_tool] - ) + with chat_span(model="gpt-4", provider="openai", conversation_id="cid") as span: + response = client.chat.completions.create(...) span.set_attribute("gen_ai.usage.input_tokens", response.usage.prompt_tokens) - span.set_attribute("gen_ai.response.finish_reason", response.choices[0].finish_reason) - - if response.choices[0].message.tool_calls: - tool_call = response.choices[0].message.tool_calls[0] - with execute_tool_span(tool_name=tool_call.function.name) as tool_span: - result = execute_function(tool_call.function.name, tool_call.function.arguments) - tool_span.set_attribute("gen_ai.tool.call.result", result) ``` """ tracer = trace.get_tracer(__name__) From 42878e7cc972d672d30513b8a973b08f58634020 Mon Sep 17 00:00:00 2001 From: Lucas Soares Date: Mon, 23 Mar 2026 10:46:26 -0300 Subject: [PATCH 11/13] docs: new mental model --- .../core/telemetry/user-guide.md | 279 ++++++++---------- 1 file changed, 127 insertions(+), 152 deletions(-) diff --git a/src/sap_cloud_sdk/core/telemetry/user-guide.md b/src/sap_cloud_sdk/core/telemetry/user-guide.md index 3565a5e..579650a 100644 --- a/src/sap_cloud_sdk/core/telemetry/user-guide.md +++ b/src/sap_cloud_sdk/core/telemetry/user-guide.md @@ -1,14 +1,22 @@ # Telemetry User Guide -## What it does +## How it works -- **Auto-instruments AI frameworks** - automatic tracing -- **Creates custom spans** - wrap your code to trace operations and add context -- **Tracks tenant IDs** - per-request tenant tracking in traces and metrics +Telemetry has two layers that work together: -## How to use it +- **Auto-instrumentation** handles the *what*: LLM calls, token counts, latency, model names — automatically, by calling auto_instrument() once. +- **Custom spans** handle the *who* and *why*: which agent, which user, which operation — context that autoinstrumentation can't infer. -### Auto-instrument AI frameworks +The primary pattern is to wrap autoinstrumented calls in a parent span that carries your business context: + +``` +invoke_agent span ← you create this (agent name, tenant, session, operation type) + └─ chat span ← autoinstrumentation creates this (model, tokens, latency) +``` + +## Quick start + +### 1. Enable auto-instrumentation Call before importing AI libraries: @@ -18,225 +26,192 @@ from sap_cloud_sdk.core.telemetry import auto_instrument auto_instrument() from litellm import completion -# All LLM calls are now automatically traced +# LLM calls are now automatically traced ``` -### Create custom spans +### 2. Add business context with a parent span -Wrap operations to trace them: +Wrap your LLM calls to add the context autoinstrumentation can't provide: ```python -from sap_cloud_sdk.core.telemetry import context_overlay, GenAIOperation +from sap_cloud_sdk.core.telemetry import invoke_agent_span -with context_overlay(GenAIOperation.CHAT): - response = llm.chat(message) +with invoke_agent_span(provider="openai", agent_name="SupportBot", conversation_id="conv-123"): + # autoinstrumented LLM call is a child of this span + response = client.chat.completions.create(...) ``` -Add custom attributes: +### 3. Set tenant ID at the request boundary ```python -with context_overlay( - GenAIOperation.CHAT, - attributes={"user.id": "123", "feature": "support"} -): - response = llm.chat(message) +from sap_cloud_sdk.core.telemetry import set_tenant_id + +def handle_request(request): + set_tenant_id(extract_tenant_from_jwt(request)) ``` -Available operations: +--- -```python -GenAIOperation.CHAT # Chat completion -GenAIOperation.TEXT_COMPLETION # Text completion -GenAIOperation.EMBEDDINGS # Embedding generation -GenAIOperation.GENERATE_CONTENT # Multimodal content generation -GenAIOperation.RETRIEVAL # Document/context retrieval -GenAIOperation.EXECUTE_TOOL # Tool execution -GenAIOperation.CREATE_AGENT # Agent creation -GenAIOperation.INVOKE_AGENT # Agent invocation -``` +## Span functions -Nest spans for complex workflows: +For operations following [OpenTelemetry GenAI conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-agent-spans/): ```python -with context_overlay(GenAIOperation.RETRIEVAL): - documents = retrieve_documents(query) +from sap_cloud_sdk.core.telemetry import chat_span, execute_tool_span, invoke_agent_span + +# Agent invocation — top-level parent span for an agent turn +with invoke_agent_span(provider="openai", agent_name="SupportBot", conversation_id="cid"): + response = client.beta.threads.runs.create(...) + +# LLM chat call — use when autoinstrumentation is not available +with chat_span(model="gpt-4", provider="openai", conversation_id="cid") as span: + response = client.chat.completions.create(...) - with context_overlay(GenAIOperation.CHAT): - response = llm.chat(messages=[ - {"role": "system", "content": f"Context: {documents}"}, - {"role": "user", "content": query} - ]) +# Tool execution +with execute_tool_span(tool_name="get_weather", tool_type="mcp", tool_description="weather mcp server"): + result = call_weather_api(location) ``` -### Propagate attributes to child spans +### Generic spans -Use `propagate=True` to automatically pass attributes from a parent span to all child spans within its scope. This is useful for cross-cutting attributes like `gen_ai.conversation.id`, `gen_ai.agent.name`, or custom keys that should appear on every nested span without repeating them. +Use `context_overlay` for operations without a dedicated function: ```python -with context_overlay( - GenAIOperation.INVOKE_AGENT, - attributes={"gen_ai.conversation.id": "conv-123", "user.id": "u-456"}, - propagate=True -): - # Both child spans automatically receive gen_ai.conversation.id and user.id - with chat_span(model="gpt-4", provider="openai") as span: - response = client.chat.completions.create(...) +from sap_cloud_sdk.core.telemetry import context_overlay, GenAIOperation - with execute_tool_span(tool_name="get_weather"): - result = call_weather_api(location) +with context_overlay(GenAIOperation.RETRIEVAL, attributes={"index": "knowledge-base"}): + documents = retrieve_documents(query) ``` -All four span functions support `propagate=True`: `context_overlay`, `chat_span`, `execute_tool_span`, and `invoke_agent_span`. - -**Priority rules** — child spans always win over propagated values (highest to lowest): -1. Required semantic keys set by the span function (e.g. `gen_ai.operation.name`) — always wins -2. User-provided `attributes` on the child span -3. Propagated attrs from ancestors — lowest priority, easily overridden +Available operations: ```python -# Even if the parent propagated gen_ai.operation.name="invoke_agent", -# the child chat_span always sets it to "chat" -with invoke_agent_span(provider="openai", propagate=True): - with chat_span(model="gpt-4", provider="openai") as span: - pass # span has gen_ai.operation.name="chat", not "invoke_agent" +GenAIOperation.CHAT +GenAIOperation.TEXT_COMPLETION +GenAIOperation.EMBEDDINGS +GenAIOperation.GENERATE_CONTENT +GenAIOperation.RETRIEVAL +GenAIOperation.EXECUTE_TOOL +GenAIOperation.CREATE_AGENT +GenAIOperation.INVOKE_AGENT ``` -Propagation is scoped: once the `propagate=True` span exits, its attributes are no longer passed to subsequent sibling spans. +--- -Nesting multiple `propagate=True` spans accumulates attributes from all levels: +## Adding attributes + +### To the current span + +Add attributes to whichever span is currently active — including autoinstrumented ones: ```python -with context_overlay(GenAIOperation.INVOKE_AGENT, attributes={"session": "s1"}, propagate=True): - with chat_span("gpt-4", "openai", attributes={"turn": "1"}, propagate=True): - with execute_tool_span("search"): - pass # receives both session="s1" and turn="1" +from sap_cloud_sdk.core.telemetry import add_span_attribute + +with invoke_agent_span(provider="openai", agent_name="SupportBot"): + response = client.chat.completions.create(...) + add_span_attribute("response.length", len(response.choices[0].message.content)) ``` -### Add events to spans +### To a specific span + +Every span function yields the span for direct access: ```python -with context_overlay(GenAIOperation.EMBEDDINGS) as span: - span.add_event("preprocessing_started") - text = preprocess_text(raw_text) - - span.add_event("embedding_generation_started") - embeddings = generate_embeddings(text) - - span.add_event("completed", attributes={ - "embedding_dim": len(embeddings) - }) +with invoke_agent_span(provider="openai", agent_name="SupportBot") as span: + span.add_event("tool_selected", attributes={"tool": "search"}) + response = client.chat.completions.create(...) ``` -### GenAI-specific spans +### Propagating parent attributes to child spans -For LLM calls following [OpenTelemetry GenAI conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-agent-spans/): +By default, attributes set on a parent span stay on that span. If you need attributes to also appear on child spans — for example, to filter by `user.id` at the LLM span level in your observability backend — use `propagate=True`: ```python -from sap_cloud_sdk.core.telemetry import chat_span, execute_tool_span, invoke_agent_span +with invoke_agent_span( + provider="openai", + agent_name="SupportBot", + attributes={"user.id": "u-456"}, + propagate=True +): + # child spans automatically receive user.id + with execute_tool_span("search"): + ... + with chat_span("gpt-4", "openai"): + ... +``` -# LLM chat calls -with chat_span(model="gpt-4", provider="openai", conversation_id="cid") as span: - response = client.chat.completions.create(...) +> **Note:** `propagate=True` is an escape hatch for backends that require attributes to appear on every span individually. In most cases, querying by the parent span is sufficient and preferred. -# Tool execution -with execute_tool_span(tool_name="get_weather", tool_type="mcp", tool_description="weather mcp server"): - result = call_weather_api(location) +**Priority rules** — child span values always win (highest to lowest): +1. Required semantic keys set by the span function (e.g. `gen_ai.operation.name`) +2. User-provided `attributes` on the child span +3. Propagated attributes from ancestors -# Agent invocation -with invoke_agent_span(provider="openai", agent_name="SupportBot", agent_id="id", agent_description="support agent", conversation_id="cid"): - response = client.beta.threads.runs.create(...) -``` +Propagation is scoped: once the parent span exits, its attributes stop propagating to subsequent spans. -### Track tenant ID +--- -Set at request entry point: +## Complete example ```python -from sap_cloud_sdk.core.telemetry import set_tenant_id +from sap_cloud_sdk.core.telemetry import ( + auto_instrument, + invoke_agent_span, + execute_tool_span, + set_tenant_id, + add_span_attribute, +) -def handle_request(request): - tenant_id = extract_tenant_from_jwt(request) - set_tenant_id(tenant_id) -``` +auto_instrument() -Thread-safe and async-safe. Automatic Propagation. +from litellm import completion -### Access current span +async def handle_request(query: str, user_id: str): + set_tenant_id("bh7sjh...") -```python -from sap_cloud_sdk.core.telemetry import get_current_span, add_span_attribute + # Parent span carries business context for the whole agent turn. + # Autoinstrumentation creates the child LLM span automatically. + with invoke_agent_span( + provider="openai", + agent_name="SupportBot", + attributes={"user.id": user_id} + ): + documents = await retrieve_knowledge_base(query) + add_span_attribute("documents.retrieved", len(documents)) -span = get_current_span() -if span.is_recording(): - span.set_attribute("custom.key", "value") + response = completion( + model="gpt-4", + messages=[ + {"role": "system", "content": f"Context: {documents}"}, + {"role": "user", "content": query} + ] + ) -# Or use the helper -add_span_attribute("request.id", request_id) + return response ``` ## Configuration ### Production -For production environments, you should ensure that `OTEL_EXPORTER_OTLP_ENDPOINT` is configured and points to the expected OTLP endpoint. This variable is a standard environment variable from the OpenTelemetry libraries. +Ensure `OTEL_EXPORTER_OTLP_ENDPOINT` points to your OTLP endpoint. -### Local Development +### Local development -To print traces directly to the console without an OTLP collector, set: +Print traces to console: ```bash export OTEL_TRACES_EXPORTER=console ``` -Then call `auto_instrument()` as usual — traces will be printed to stdout. - -To use an OTLP collector instead: +Use an OTLP collector: ```bash export OTEL_EXPORTER_OTLP_ENDPOINT="https://otel-collector.example.com" ``` -### System Role - -Set via environment variable: +### System role ```bash export APPFND_CONHOS_SYSTEM_ROLE="S4HC" ``` - -## Complete Example - -```python -from sap_cloud_sdk.core.telemetry import ( - auto_instrument, - context_overlay, - GenAIOperation, - set_tenant_id, - add_span_attribute -) - -auto_instrument() - -from litellm import completion - -async def handle_customer_query(query: str, user_id: str): - set_tenant_id("bh7sjh...") - - with context_overlay( - GenAIOperation.RETRIEVAL, - attributes={"user.id": user_id, "query.type": "support"} - ): - documents = await retrieve_knowledge_base(query) - add_span_attribute("documents.count", len(documents)) - - with context_overlay(GenAIOperation.CHAT): - response = completion( - model="gpt-4", - messages=[ - {"role": "system", "content": f"Context: {documents}"}, - {"role": "user", "content": query} - ] - ) - add_span_attribute("response.length", len(response.choices[0].message.content)) - - return response From 7d96beefc41963da44a72d0f4d2f37d9837778c8 Mon Sep 17 00:00:00 2001 From: Lucas Soares Date: Mon, 23 Mar 2026 12:14:29 -0300 Subject: [PATCH 12/13] docs: conciseness --- src/sap_cloud_sdk/core/telemetry/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sap_cloud_sdk/core/telemetry/user-guide.md b/src/sap_cloud_sdk/core/telemetry/user-guide.md index 579650a..dc88f52 100644 --- a/src/sap_cloud_sdk/core/telemetry/user-guide.md +++ b/src/sap_cloud_sdk/core/telemetry/user-guide.md @@ -140,7 +140,7 @@ with invoke_agent_span( ... ``` -> **Note:** `propagate=True` is an escape hatch for backends that require attributes to appear on every span individually. In most cases, querying by the parent span is sufficient and preferred. +> **Note:** `propagate=True` is specific for backends that require attributes to appear on every span individually. In most cases, querying by the parent span is sufficient and preferred. **Priority rules** — child span values always win (highest to lowest): 1. Required semantic keys set by the span function (e.g. `gen_ai.operation.name`) From 43c4e603a1303f795eede7098a0ea52072556ce9 Mon Sep 17 00:00:00 2001 From: Lucas Soares Date: Mon, 23 Mar 2026 12:31:23 -0300 Subject: [PATCH 13/13] chore: linting --- src/sap_cloud_sdk/core/telemetry/telemetry.py | 4 +++- src/sap_cloud_sdk/core/telemetry/tracer.py | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/sap_cloud_sdk/core/telemetry/telemetry.py b/src/sap_cloud_sdk/core/telemetry/telemetry.py index c69ba6a..c105109 100644 --- a/src/sap_cloud_sdk/core/telemetry/telemetry.py +++ b/src/sap_cloud_sdk/core/telemetry/telemetry.py @@ -32,7 +32,9 @@ _tenant_id_var: ContextVar[str] = ContextVar("tenant_id", default="") # Context variable for propagated span attributes -_propagated_attrs_var: ContextVar[Dict[str, Any]] = ContextVar("propagated_attrs", default={}) +_propagated_attrs_var: ContextVar[Dict[str, Any]] = ContextVar( + "propagated_attrs", default={} +) def set_tenant_id(tenant_id: str) -> None: diff --git a/src/sap_cloud_sdk/core/telemetry/tracer.py b/src/sap_cloud_sdk/core/telemetry/tracer.py index 70e6636..5cca857 100644 --- a/src/sap_cloud_sdk/core/telemetry/tracer.py +++ b/src/sap_cloud_sdk/core/telemetry/tracer.py @@ -13,7 +13,11 @@ from opentelemetry.trace import Status, StatusCode, Span from sap_cloud_sdk.core.telemetry.genai_operation import GenAIOperation -from sap_cloud_sdk.core.telemetry.telemetry import get_tenant_id, get_propagated_attributes, _propagated_attrs_var +from sap_cloud_sdk.core.telemetry.telemetry import ( + get_tenant_id, + get_propagated_attributes, + _propagated_attrs_var, +) from sap_cloud_sdk.core.telemetry.constants import ATTR_SAP_TENANT_ID # OpenTelemetry GenAI semantic attribute names (avoid duplicate string literals)