diff --git a/instrumentation-loongsuite/README.md b/instrumentation-loongsuite/README.md index 325f02740..8c6732cee 100644 --- a/instrumentation-loongsuite/README.md +++ b/instrumentation-loongsuite/README.md @@ -1,7 +1,7 @@ | Instrumentation | Supported Packages | Metrics support | Semconv status | | --------------- | ------------------ | --------------- | -------------- | -| [loongsuite-instrumentation-agentscope](./loongsuite-instrumentation-agentscope) | agentscope >= 1.0.0 | No | development +| [loongsuite-instrumentation-agentscope](./loongsuite-instrumentation-agentscope) | agentscope >= 1.0.0, < 3.0.0 | No | development | [loongsuite-instrumentation-agno](./loongsuite-instrumentation-agno) | agno >= 2.0.0, < 3 | No | development | [loongsuite-instrumentation-algotune](./loongsuite-instrumentation-algotune) | algotune | No | development | [loongsuite-instrumentation-bfclv4](./loongsuite-instrumentation-bfclv4) | bfcl-eval >= 4.0.0 | No | development diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/CHANGELOG.md b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/CHANGELOG.md index 423fc7af7..de22189d8 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/CHANGELOG.md +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Add version-aware AgentScope v2 middleware instrumentation while preserving + AgentScope v1 compatibility. + ## Version 0.5.0 (2026-05-11) ### Added diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/pyproject.toml b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/pyproject.toml index 026d4dafc..ea0d5fdb2 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/pyproject.toml +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/pyproject.toml @@ -30,12 +30,13 @@ dependencies = [ "opentelemetry-instrumentation >= 0.58b0", "opentelemetry-semantic-conventions >= 0.58b0", "opentelemetry-util-genai", + "packaging >= 18.0", "wrapt >= 1.17.3, < 2.0.0", ] [project.optional-dependencies] instruments = [ - "agentscope >= 1.0.0", + "agentscope >= 1.0.0, < 3.0.0", ] test = [ @@ -43,9 +44,9 @@ test = [ "pytest-asyncio ~= 0.23.0", "pytest-cov ~= 4.1.0", "pytest-vcr ~= 1.0.2", - "vcrpy ~= 5.1.0", + "vcrpy >= 8.1.1", "pyyaml ~= 6.0", - "agentscope >= 1.0.0", + "agentscope >= 1.0.0, < 3.0.0", ] [project.entry-points.opentelemetry_instrumentor] diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/src/opentelemetry/instrumentation/agentscope/__init__.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/src/opentelemetry/instrumentation/agentscope/__init__.py index bb210120d..2002bd3f9 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/src/opentelemetry/instrumentation/agentscope/__init__.py +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/src/opentelemetry/instrumentation/agentscope/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. """ -AgentScope instrumentation supporting `agentscope >= 1.0.0`. +AgentScope instrumentation supporting `agentscope >= 1.0.0, < 3.0.0`. Usage ----- @@ -49,25 +49,22 @@ async def call_model(): from __future__ import annotations import logging +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as metadata_version from typing import Any, Collection from wrapt import wrap_function_wrapper from opentelemetry import trace as trace_api -from opentelemetry.instrumentation.agentscope.package import _instruments +from opentelemetry.instrumentation.agentscope.package import ( + get_installed_instrumentation_dependencies, +) from opentelemetry.instrumentation.agentscope.version import __version__ from opentelemetry.instrumentation.instrumentor import BaseInstrumentor from opentelemetry.instrumentation.utils import unwrap from opentelemetry.semconv.schemas import Schemas from opentelemetry.util.genai.extended_handler import ExtendedTelemetryHandler -from ._wrapper import ( - AgentScopeAgentWrapper, - AgentScopeChatModelWrapper, - AgentScopeEmbeddingModelWrapper, -) -from .patch import wrap_formatter_format, wrap_tool_call - logger = logging.getLogger(__name__) _MODEL_MODULE = "agentscope.model" @@ -94,9 +91,10 @@ def __init__(self): self._handler = ( None # ExtendedTelemetryHandler handles all other operations ) + self._agentscope_major = None def instrumentation_dependencies(self) -> Collection[str]: - return _instruments + return get_installed_instrumentation_dependencies() def _setup_tracing_patch(self, wrapped, instance, args, kwargs): """Replace setup_tracing with no-op to use OTEL instead.""" @@ -127,6 +125,23 @@ def _instrument(self, **kwargs: Any) -> None: schema_url=Schemas.V1_37_0.value, ) + self._agentscope_major = _get_agentscope_major() + if self._agentscope_major >= 2: + self._instrument_v2() + else: + self._instrument_v1() + + def _instrument_v1(self) -> None: + from ._wrapper import ( # noqa: PLC0415 + AgentScopeAgentWrapper, + AgentScopeChatModelWrapper, + AgentScopeEmbeddingModelWrapper, + ) + from .patch import ( # noqa: PLC0415 + wrap_formatter_format, + wrap_tool_call, + ) + # Instrument ChatModelBase try: chat_wrapper = AgentScopeChatModelWrapper(handler=self._handler) @@ -224,21 +239,48 @@ def wrap_formatter_with_tracer(wrapped, instance, args, kwargs): def _uninstrument(self, **kwargs: Any) -> None: """Disable AgentScope instrumentation.""" + del kwargs + if self._agentscope_major is None: + self._agentscope_major = _get_agentscope_major() + if self._agentscope_major >= 2: + self._uninstrument_v2() + else: + self._uninstrument_v1() + self._handler = None + self._tracer = None + self._agentscope_major = None + + def _uninstrument_v1(self) -> None: try: - AgentScopeChatModelWrapper.restore_original_methods() - logger.debug("Restored ChatModelBase methods") + from ._wrapper import ( # noqa: PLC0415 + AgentScopeAgentWrapper, + AgentScopeChatModelWrapper, + AgentScopeEmbeddingModelWrapper, + ) + except Exception as e: + logger.warning(f"Failed to import AgentScope wrappers: {e}") + AgentScopeAgentWrapper = None + AgentScopeChatModelWrapper = None + AgentScopeEmbeddingModelWrapper = None + + try: + if AgentScopeChatModelWrapper is not None: + AgentScopeChatModelWrapper.restore_original_methods() + logger.debug("Restored ChatModelBase methods") except Exception as e: logger.warning(f"Failed to restore ChatModelBase: {e}") try: - AgentScopeAgentWrapper.restore_original_methods() - logger.debug("Restored AgentBase methods") + if AgentScopeAgentWrapper is not None: + AgentScopeAgentWrapper.restore_original_methods() + logger.debug("Restored AgentBase methods") except Exception as e: logger.warning(f"Failed to restore AgentBase: {e}") try: - AgentScopeEmbeddingModelWrapper.restore_original_methods() - logger.debug("Restored EmbeddingModelBase methods") + if AgentScopeEmbeddingModelWrapper is not None: + AgentScopeEmbeddingModelWrapper.restore_original_methods() + logger.debug("Restored EmbeddingModelBase methods") except Exception as e: logger.warning(f"Failed to restore EmbeddingModelBase: {e}") @@ -301,3 +343,50 @@ def _uninstrument(self, **kwargs: Any) -> None: logger.warning( f"Failed to uninstrument _check_tracing_enabled: {e}" ) + + def _instrument_v2(self) -> None: + from ._v2_middleware import ( # noqa: PLC0415 + AgentScopeV2Middleware, + append_loongsuite_middleware, + ) + + try: + + def wrap_agent_init(wrapped, instance, args, kwargs): + args, kwargs = append_loongsuite_middleware( + args, + kwargs, + AgentScopeV2Middleware(handler=lambda: self._handler), + ) + return wrapped(*args, **kwargs) + + wrap_function_wrapper( + module=_AGENT_MODULE, + name="Agent.__init__", + wrapper=wrap_agent_init, + ) + logger.debug("Instrumented AgentScope v2 Agent") + except Exception as e: + logger.warning(f"Failed to instrument AgentScope v2 Agent: {e}") + + def _uninstrument_v2(self) -> None: + try: + import agentscope.agent # noqa: PLC0415 + + unwrap(agentscope.agent.Agent, "__init__") + logger.debug("Uninstrumented AgentScope v2 Agent") + except Exception as e: + logger.warning(f"Failed to uninstrument AgentScope v2 Agent: {e}") + + +def _get_agentscope_major() -> int: + try: + installed_version = metadata_version("agentscope") + except PackageNotFoundError: + return 1 + + major_text = installed_version.split(".", 1)[0] + try: + return int(major_text) + except ValueError: + return 1 diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/src/opentelemetry/instrumentation/agentscope/_v2_middleware.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/src/opentelemetry/instrumentation/agentscope/_v2_middleware.py new file mode 100644 index 000000000..f70d06d8f --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/src/opentelemetry/instrumentation/agentscope/_v2_middleware.py @@ -0,0 +1,513 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AgentScope v2 middleware instrumentation.""" + +from __future__ import annotations + +import inspect +import json +import logging +import timeit +from collections.abc import AsyncGenerator, Awaitable, Callable, Sequence +from contextvars import ContextVar +from dataclasses import asdict, is_dataclass +from typing import Any + +from agentscope.agent import Agent +from agentscope.message import Msg +from agentscope.middleware import MiddlewareBase +from agentscope.model import ChatModelBase, ChatResponse +from agentscope.tool import ToolResponse + +from opentelemetry.context import Context, get_current +from opentelemetry.util.genai.extended_handler import ExtendedTelemetryHandler +from opentelemetry.util.genai.extended_types import ( + ExecuteToolInvocation, + InvokeAgentInvocation, + ReactStepInvocation, +) +from opentelemetry.util.genai.types import ( + Error, + FunctionToolDefinition, + InputMessage, + LLMInvocation, + OutputMessage, + Reasoning, + Text, + ToolCall, + ToolCallResponse, +) + +logger = logging.getLogger(__name__) + +_MIDDLEWARE_PARAMETER = "middlewares" +_FIRST_TOKEN_EVENT_TYPES = { + "text_block_delta", + "thinking_block_delta", + "tool_call_delta", +} + + +def append_loongsuite_middleware( + args: tuple[Any, ...], + kwargs: dict[str, Any], + middleware: "AgentScopeV2Middleware", +) -> tuple[tuple[Any, ...], dict[str, Any]]: + """Append LoongSuite middleware to AgentScope v2 Agent.__init__ inputs.""" + if _MIDDLEWARE_PARAMETER in kwargs: + kwargs = dict(kwargs) + kwargs[_MIDDLEWARE_PARAMETER] = _append_once( + kwargs.get(_MIDDLEWARE_PARAMETER), middleware + ) + return args, kwargs + + middleware_position = _middleware_arg_position() + if middleware_position is not None and len(args) > middleware_position: + updated_args = list(args) + updated_args[middleware_position] = _append_once( + updated_args[middleware_position], + middleware, + ) + return tuple(updated_args), kwargs + + kwargs = dict(kwargs) + kwargs[_MIDDLEWARE_PARAMETER] = [middleware] + return args, kwargs + + +def _append_once( + middlewares: Sequence[MiddlewareBase] | None, + middleware: "AgentScopeV2Middleware", +) -> list[MiddlewareBase]: + result = list(middlewares or []) + if any(isinstance(item, AgentScopeV2Middleware) for item in result): + return result + result.append(middleware) + return result + + +class AgentScopeV2Middleware(MiddlewareBase): + """LoongSuite telemetry adapter for AgentScope v2 middleware hooks.""" + + def __init__( + self, handler: Callable[[], ExtendedTelemetryHandler | None] + ) -> None: + self._handler = handler + self._react_round: ContextVar[int] = ContextVar( + "loongsuite_agentscope_v2_react_round", + default=0, + ) + + async def on_reply( + self, + agent: Agent, + input_kwargs: dict, + next_handler: Callable[..., AsyncGenerator], + ) -> AsyncGenerator: + handler = self._handler() + if handler is None: + async for item in next_handler(**input_kwargs): + yield item + return + + invocation = _create_agent_invocation(agent, input_kwargs) + handler.start_invoke_agent(invocation) + round_token = self._react_round.set(0) + first_token_seen = False + last_msg = None + closed = False + try: + async for item in next_handler(**input_kwargs): + if not first_token_seen and _is_first_token_event(item): + invocation.monotonic_first_token_s = timeit.default_timer() + first_token_seen = True + if isinstance(item, Msg): + last_msg = item + yield item + except BaseException as exc: + handler.fail_invoke_agent( + invocation, + Error(message=str(exc) or type(exc).__name__, type=type(exc)), + ) + closed = True + raise + else: + if last_msg is not None: + invocation.output_messages = [_message_to_output(last_msg)] + if last_msg.usage is not None: + invocation.input_tokens = last_msg.usage.input_tokens + invocation.output_tokens = last_msg.usage.output_tokens + handler.stop_invoke_agent(invocation) + closed = True + finally: + self._react_round.reset(round_token) + if not closed: + handler.stop_invoke_agent(invocation) + + async def on_model_call( + self, + agent: Agent, + input_kwargs: dict, + next_handler: Callable[ + ..., + Awaitable[ChatResponse | AsyncGenerator[ChatResponse, None]], + ], + ) -> ChatResponse | AsyncGenerator[ChatResponse, None]: + model = input_kwargs.get("current_model") + if not isinstance(model, ChatModelBase): + return await next_handler(**input_kwargs) + + handler = self._handler() + if handler is None: + return await next_handler(**input_kwargs) + + invocation = _create_llm_invocation(model, input_kwargs) + span_context = get_current() + started = False + if not _is_streaming_model(model, input_kwargs): + handler.start_llm(invocation, context=span_context) + started = True + try: + result = await next_handler(**input_kwargs) + if inspect.isasyncgen(result): + return self._wrap_model_stream( + result, + invocation, + span_context, + handler, + span_started=started, + ) + + if not started: + handler.start_llm(invocation, context=span_context) + started = True + _finish_llm_invocation(invocation, result) + handler.stop_llm(invocation) + return result + except BaseException as exc: + if not started: + handler.start_llm(invocation, context=span_context) + handler.fail_llm( + invocation, + Error(message=str(exc) or type(exc).__name__, type=type(exc)), + ) + raise + + async def _wrap_model_stream( + self, + result: AsyncGenerator[ChatResponse, None], + invocation: LLMInvocation, + span_context: Context, + handler: ExtendedTelemetryHandler, + *, + span_started: bool, + ) -> AsyncGenerator[ChatResponse, None]: + first_token_seen = False + last_chunk = None + closed = False + if not span_started: + handler.start_llm(invocation, context=span_context) + span_started = True + try: + async for chunk in result: + if not first_token_seen: + invocation.monotonic_first_token_s = timeit.default_timer() + first_token_seen = True + last_chunk = chunk + yield chunk + except BaseException as exc: + handler.fail_llm( + invocation, + Error(message=str(exc) or type(exc).__name__, type=type(exc)), + ) + closed = True + raise + else: + _finish_llm_invocation(invocation, last_chunk) + handler.stop_llm(invocation) + closed = True + finally: + if span_started and not closed: + handler.stop_llm(invocation) + + async def on_acting( + self, + agent: Agent, + input_kwargs: dict, + next_handler: Callable[..., AsyncGenerator], + ) -> AsyncGenerator: + handler = self._handler() + if handler is None: + async for item in next_handler(**input_kwargs): + yield item + return + + tool_call = input_kwargs.get("tool_call") + react_invocation = ReactStepInvocation(round=self._next_react_round()) + handler.start_react_step(react_invocation, context=get_current()) + invocation = ExecuteToolInvocation( + tool_name=getattr(tool_call, "name", "unknown_tool"), + tool_call_id=getattr(tool_call, "id", None), + tool_call_arguments=_loads_json(getattr(tool_call, "input", None)), + provider="agentscope", + ) + handler.start_execute_tool(invocation) + last_item = None + tool_closed = False + react_closed = False + try: + async for item in next_handler(**input_kwargs): + last_item = item + yield item + except BaseException as exc: + error = Error( + message=str(exc) or type(exc).__name__, type=type(exc) + ) + handler.fail_execute_tool( + invocation, + error, + ) + tool_closed = True + handler.fail_react_step(react_invocation, error) + react_closed = True + raise + else: + if isinstance(last_item, ToolResponse): + invocation.tool_call_result = _jsonable( + _blocks_to_parts(last_item.content) + ) + elif last_item is not None: + invocation.tool_call_result = str(last_item) + handler.stop_execute_tool(invocation) + tool_closed = True + react_invocation.finish_reason = "tool_calls" + handler.stop_react_step(react_invocation) + react_closed = True + finally: + if not tool_closed: + handler.stop_execute_tool(invocation) + if not react_closed: + handler.stop_react_step(react_invocation) + + def _next_react_round(self) -> int: + current = self._react_round.get() + 1 + self._react_round.set(current) + return current + + +def _create_agent_invocation( + agent: Agent, + input_kwargs: dict[str, Any], +) -> InvokeAgentInvocation: + model = getattr(agent, "model", None) + request_model = getattr(model, "model", None) + provider = _get_provider_name(model) + inputs = input_kwargs.get("inputs") + return InvokeAgentInvocation( + provider=provider, + agent_name=getattr(agent, "name", "unknown_agent"), + agent_id=getattr(getattr(agent, "state", None), "session_id", None), + conversation_id=getattr( + getattr(agent, "state", None), "session_id", None + ), + request_model=request_model, + input_messages=_messages_to_inputs(inputs), + system_instruction=[ + Text(content=getattr(agent, "_system_prompt", "")) + ], + ) + + +def _create_llm_invocation( + model: ChatModelBase, + input_kwargs: dict[str, Any], +) -> LLMInvocation: + invocation = LLMInvocation( + request_model=getattr(model, "model", None), + provider=_get_provider_name(model), + input_messages=_messages_to_inputs(input_kwargs.get("messages")), + tool_definitions=_tool_definitions(input_kwargs.get("tools")), + ) + parameters = getattr(model, "parameters", None) + for source in (parameters, input_kwargs): + _set_if_present(invocation, "temperature", source) + _set_if_present(invocation, "top_p", source) + _set_if_present(invocation, "max_tokens", source) + return invocation + + +def _finish_llm_invocation( + invocation: LLMInvocation, + response: ChatResponse | None, +) -> None: + if response is None: + return + invocation.response_id = getattr(response, "id", None) + invocation.output_messages = [_chat_response_to_output(response)] + usage = getattr(response, "usage", None) + if usage is not None: + invocation.input_tokens = getattr(usage, "input_tokens", None) + invocation.output_tokens = getattr(usage, "output_tokens", None) + + +def _messages_to_inputs(value: Any) -> list[InputMessage]: + if value is None: + return [] + if isinstance(value, Msg): + return [_message_to_input(value)] + if isinstance(value, list): + return [ + _message_to_input(item) for item in value if isinstance(item, Msg) + ] + return [] + + +def _message_to_input(msg: Msg) -> InputMessage: + return InputMessage(role=msg.role, parts=_blocks_to_parts(msg.content)) + + +def _message_to_output(msg: Msg) -> OutputMessage: + return OutputMessage( + role=msg.role, + parts=_blocks_to_parts(msg.content), + finish_reason="stop", + ) + + +def _chat_response_to_output(response: ChatResponse) -> OutputMessage: + finish_reason = "stop" + if any( + getattr(block, "type", None) == "tool_call" + for block in response.content + ): + finish_reason = "tool_calls" + return OutputMessage( + role="assistant", + parts=_blocks_to_parts(response.content), + finish_reason=finish_reason, + ) + + +def _blocks_to_parts(blocks: Sequence[Any]) -> list[Any]: + parts = [] + for block in blocks: + block_type = getattr(block, "type", None) + if block_type == "text": + parts.append(Text(content=getattr(block, "text", ""))) + elif block_type == "thinking": + parts.append(Reasoning(content=getattr(block, "thinking", ""))) + elif block_type == "tool_call": + parts.append( + ToolCall( + id=getattr(block, "id", None), + name=getattr(block, "name", ""), + arguments=_loads_json(getattr(block, "input", None)), + ) + ) + elif block_type == "tool_result": + parts.append( + ToolCallResponse( + id=getattr(block, "id", None), + response=getattr(block, "output", ""), + ) + ) + return parts + + +def _tool_definitions(tools: list[dict[str, Any]] | None) -> list[Any]: + if not tools: + return [] + definitions = [] + for tool in tools: + function = tool.get("function") if isinstance(tool, dict) else None + if not isinstance(function, dict): + continue + definitions.append( + FunctionToolDefinition( + name=function.get("name", ""), + description=function.get("description"), + parameters=function.get("parameters"), + ) + ) + return definitions + + +def _get_provider_name(model: Any) -> str: + class_name = model.__class__.__name__.lower() if model is not None else "" + if "dashscope" in class_name: + return "dashscope" + if "openai" in class_name: + return "openai" + if "anthropic" in class_name: + return "anthropic" + if "gemini" in class_name: + return "gcp.gen_ai" + if "ollama" in class_name: + return "ollama" + return "agentscope" + + +def _is_first_token_event(item: Any) -> bool: + event_type = getattr(item, "type", None) + return event_type in _FIRST_TOKEN_EVENT_TYPES + + +def _middleware_arg_position() -> int | None: + try: + parameters = list(inspect.signature(Agent.__init__).parameters) + return parameters.index(_MIDDLEWARE_PARAMETER) - 1 + except (TypeError, ValueError): + return None + + +def _is_streaming_model( + model: ChatModelBase, input_kwargs: dict[str, Any] +) -> bool: + if "stream" in input_kwargs: + return bool(input_kwargs["stream"]) + return bool(getattr(model, "stream", False)) + + +def _loads_json(value: Any) -> Any: + if not isinstance(value, str): + return value + try: + return json.loads(value) + except ValueError: + return value + + +def _jsonable(value: Any) -> Any: + if is_dataclass(value): + return _jsonable(asdict(value)) + if isinstance(value, list | tuple): + return [_jsonable(item) for item in value] + if isinstance(value, dict): + return {key: _jsonable(item) for key, item in value.items()} + return value + + +def _set_if_present( + invocation: LLMInvocation, + field_name: str, + source: Any, +) -> None: + value = ( + source.get(field_name) + if isinstance(source, dict) + else getattr(source, field_name, None) + ) + if value is not None: + setattr(invocation, field_name, value) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/src/opentelemetry/instrumentation/agentscope/package.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/src/opentelemetry/instrumentation/agentscope/package.py index c140bd607..6bbe8643b 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/src/opentelemetry/instrumentation/agentscope/package.py +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/src/opentelemetry/instrumentation/agentscope/package.py @@ -12,6 +12,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -_instruments = ("agentscope >= 1.0.0",) +from __future__ import annotations + +from importlib.metadata import PackageNotFoundError, version + +from packaging.requirements import Requirement + +_instruments_v1 = ("agentscope >= 1.0.0, < 2.0.0",) +_instruments_v2 = ("agentscope >= 2.0.0, < 3.0.0",) +_instruments = ("agentscope >= 1.0.0, < 3.0.0",) _supports_metrics = False + + +def get_installed_instrumentation_dependencies(): + """Return the AgentScope dependency range matching the installed major.""" + try: + installed_version = version("agentscope") + except PackageNotFoundError: + return _instruments + + for requirement in (_instruments_v2[0], _instruments_v1[0]): + if Requirement(requirement).specifier.contains( + installed_version, + prereleases=True, + ): + return (requirement,) + + return _instruments diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/cassettes/test_v2_agent_concurrent_e2e.yaml b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/cassettes/test_v2_agent_concurrent_e2e.yaml new file mode 100644 index 000000000..fcf53c5b7 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/cassettes/test_v2_agent_concurrent_e2e.yaml @@ -0,0 +1,242 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "Reply with exactly one short sentence." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Say OK for request 2." + } + ] + } + ], + "model": "qwen-plus", + "max_tokens": 16, + "stream": false, + "enable_thinking": false + } + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '258' + Content-Type: + - application/json + Host: + - dashscope.aliyuncs.com + User-Agent: + - AsyncOpenAI/Python 2.40.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - async:asyncio + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 2.40.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.13 + authorization: + - + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions + response: + body: + string: |- + { + "model": "qwen-plus", + "id": "chatcmpl-0d58865c-c0c4-97df-9ef5-841029ada056", + "choices": [ + { + "message": { + "content": "OK", + "role": "assistant" + }, + "index": 0, + "finish_reason": "stop" + } + ], + "created": 1780388664, + "object": "chat.completion", + "usage": { + "total_tokens": 28, + "completion_tokens": 1, + "prompt_tokens": 27, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + headers: + content-length: + - '328' + content-type: + - application/json + date: + - Tue, 02 Jun 2026 08:24:24 GMT + req-arrive-time: + - '1780388664520' + req-cost-time: + - '378' + resp-start-time: + - '1780388664899' + server: + - istio-envoy + transfer-encoding: + - chunked + vary: + - Origin,Access-Control-Request-Method,Access-Control-Request-Headers, Accept-Encoding + x-dashscope-call-gateway: + - 'true' + x-envoy-upstream-service-time: + - '378' + x-request-id: + - 0d58865c-c0c4-97df-9ef5-841029ada056 + status: + code: 200 + message: OK +- request: + body: |- + { + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "Reply with exactly one short sentence." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Say OK for request 1." + } + ] + } + ], + "model": "qwen-plus", + "max_tokens": 16, + "stream": false, + "enable_thinking": false + } + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '258' + Content-Type: + - application/json + Host: + - dashscope.aliyuncs.com + User-Agent: + - AsyncOpenAI/Python 2.40.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - async:asyncio + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 2.40.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.13 + authorization: + - + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions + response: + body: + string: |- + { + "model": "qwen-plus", + "id": "chatcmpl-b1d9db3b-e07f-9dea-a72e-79a244bac8d2", + "choices": [ + { + "message": { + "content": "OK for request 1.", + "role": "assistant" + }, + "index": 0, + "finish_reason": "stop" + } + ], + "created": 1780388664, + "object": "chat.completion", + "usage": { + "total_tokens": 33, + "completion_tokens": 6, + "prompt_tokens": 27, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + headers: + content-length: + - '343' + content-type: + - application/json + date: + - Tue, 02 Jun 2026 08:24:24 GMT + req-arrive-time: + - '1780388664535' + req-cost-time: + - '510' + resp-start-time: + - '1780388665045' + server: + - istio-envoy + transfer-encoding: + - chunked + vary: + - Origin,Access-Control-Request-Method,Access-Control-Request-Headers, Accept-Encoding + x-dashscope-call-gateway: + - 'true' + x-envoy-upstream-service-time: + - '509' + x-request-id: + - b1d9db3b-e07f-9dea-a72e-79a244bac8d2 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/cassettes/test_v2_agent_non_streaming_e2e.yaml b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/cassettes/test_v2_agent_non_streaming_e2e.yaml new file mode 100644 index 000000000..e6cb0678f --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/cassettes/test_v2_agent_non_streaming_e2e.yaml @@ -0,0 +1,122 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "Reply with exactly: OK" + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Say OK." + } + ] + } + ], + "model": "qwen-plus", + "max_tokens": 16, + "stream": false, + "enable_thinking": false + } + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '228' + Content-Type: + - application/json + Host: + - dashscope.aliyuncs.com + User-Agent: + - AsyncOpenAI/Python 2.40.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - async:asyncio + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 2.40.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.13 + authorization: + - + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions + response: + body: + string: |- + { + "model": "qwen-plus", + "id": "chatcmpl-d1fc4eda-fe73-9625-9108-2b83bca90dbb", + "choices": [ + { + "message": { + "content": "OK", + "role": "assistant" + }, + "index": 0, + "finish_reason": "stop" + } + ], + "created": 1780388663, + "object": "chat.completion", + "usage": { + "total_tokens": 22, + "completion_tokens": 1, + "prompt_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + headers: + content-length: + - '328' + content-type: + - application/json + date: + - Tue, 02 Jun 2026 08:24:23 GMT + req-arrive-time: + - '1780388663479' + req-cost-time: + - '330' + resp-start-time: + - '1780388663810' + server: + - istio-envoy + transfer-encoding: + - chunked + vary: + - Origin,Access-Control-Request-Method,Access-Control-Request-Headers, Accept-Encoding + x-dashscope-call-gateway: + - 'true' + x-envoy-upstream-service-time: + - '330' + x-request-id: + - d1fc4eda-fe73-9625-9108-2b83bca90dbb + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/cassettes/test_v2_agent_streaming_e2e.yaml b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/cassettes/test_v2_agent_streaming_e2e.yaml new file mode 100644 index 000000000..36f3135a5 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/cassettes/test_v2_agent_streaming_e2e.yaml @@ -0,0 +1,111 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "Reply with a short sentence." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Say hello in one sentence." + } + ] + } + ], + "model": "qwen-plus", + "max_tokens": 16, + "stream": true, + "stream_options": { + "include_usage": true + }, + "enable_thinking": false + } + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '292' + Content-Type: + - application/json + Host: + - dashscope.aliyuncs.com + User-Agent: + - AsyncOpenAI/Python 2.40.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - async:asyncio + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 2.40.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.13 + authorization: + - + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions + response: + body: + string: |+ + data: {"model":"qwen-plus","id":"chatcmpl-23d2afca-b6f8-9fb2-9c79-a40825fe928d","created":1780388664,"object":"chat.completion.chunk","usage":null,"choices":[{"logprobs":null,"index":0,"delta":{"content":"","role":"assistant"},"finish_reason":null}]} + + data: {"model":"qwen-plus","id":"chatcmpl-23d2afca-b6f8-9fb2-9c79-a40825fe928d","choices":[{"delta":{"content":"Hello"},"index":0,"finish_reason":null,"logprobs":null}],"created":1780388664,"object":"chat.completion.chunk","usage":null} + + data: {"model":"qwen-plus","id":"chatcmpl-23d2afca-b6f8-9fb2-9c79-a40825fe928d","choices":[{"delta":{"content":"!"},"index":0,"finish_reason":null,"logprobs":null}],"created":1780388664,"object":"chat.completion.chunk","usage":null} + + data: {"model":"qwen-plus","id":"chatcmpl-23d2afca-b6f8-9fb2-9c79-a40825fe928d","choices":[{"delta":{"content":""},"index":0,"finish_reason":"stop","logprobs":null}],"created":1780388664,"object":"chat.completion.chunk","usage":null} + + data: {"model":"qwen-plus","id":"chatcmpl-23d2afca-b6f8-9fb2-9c79-a40825fe928d","choices":[],"created":1780388664,"object":"chat.completion.chunk","usage":{"total_tokens":27,"completion_tokens":2,"prompt_tokens":25,"prompt_tokens_details":{"cached_tokens":0}}} + + data: [DONE] + + headers: + content-type: + - text/event-stream;charset=utf-8 + date: + - Tue, 02 Jun 2026 08:24:24 GMT + req-arrive-time: + - '1780388663952' + req-cost-time: + - '372' + resp-start-time: + - '1780388664324' + server: + - istio-envoy + transfer-encoding: + - chunked + vary: + - Origin + x-dashscope-call-gateway: + - 'true' + x-envoy-upstream-service-time: + - '372' + x-request-id: + - 23d2afca-b6f8-9fb2-9c79-a40825fe928d + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/conftest.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/conftest.py index 5cf7779d6..2e94d564e 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/conftest.py +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/conftest.py @@ -15,8 +15,12 @@ # -*- coding: utf-8 -*- """Test Configuration""" +import asyncio +import inspect import json import os +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path import pytest import yaml @@ -27,6 +31,49 @@ if "DASHSCOPE_API_KEY" not in os.environ: os.environ["DASHSCOPE_API_KEY"] = "test_api_key" +# vcrpy's aiohttp stub still references a mixin removed by newer aiohttp +# releases. AgentScope tests use VCR for replay and should not fail during +# pytest marker setup just because aiohttp is importable in the environment. +try: + import aiohttp.streams # type: ignore[import-not-found] + + if not hasattr(aiohttp.streams, "AsyncStreamReaderMixin"): + aiohttp.streams.AsyncStreamReaderMixin = object +except ImportError: + pass + +try: + import aiohttp # type: ignore[import-not-found] + import vcr.stubs.aiohttp_stubs as aiohttp_stubs + + if ( + "stream_writer" + in inspect.signature(aiohttp.ClientResponse.__init__).parameters + ): + + class _CompatStreamWriter: + output_size = 0 + + class _CompatMockClientResponse(aiohttp_stubs.MockClientResponse): + def __init__(self, method, url, request_info=None): + aiohttp.ClientResponse.__init__( + self, + method=method, + url=url, + writer=None, + continue100=None, + timer=None, + request_info=request_info, + traces=None, + loop=asyncio.get_event_loop(), + session=None, + stream_writer=_CompatStreamWriter(), + ) + + aiohttp_stubs.MockClientResponse = _CompatMockClientResponse +except ImportError: + pass + from opentelemetry.instrumentation.agentscope import AgentScopeInstrumentor from opentelemetry.sdk._logs import LoggerProvider from opentelemetry.sdk._logs.export import ( @@ -45,6 +92,19 @@ OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, ) +_V2_TEST_FILE = "test_v2_instrumentation.py" + + +def _agentscope_major() -> int: + try: + installed_version = version("agentscope") + except PackageNotFoundError: + return 1 + try: + return int(installed_version.split(".", 1)[0]) + except ValueError: + return 1 + def pytest_configure(config: pytest.Config): # Configure pytest-asyncio to auto-detect async test functions @@ -67,6 +127,19 @@ def pytest_configure(config: pytest.Config): config.option.api_key = api_key +def pytest_ignore_collect(collection_path, config): # noqa: ARG001 + path = Path(str(collection_path)) + if not path.name.startswith("test_") or path.suffix != ".py": + return None + + major = _agentscope_major() + if major >= 2: + return path.name != _V2_TEST_FILE + if path.name == _V2_TEST_FILE: + return True + return None + + # ==================== Exporters and Readers ==================== @@ -245,8 +318,8 @@ def vcr_config(): """Configure VCR for recording and replaying HTTP requests""" return { "filter_headers": [ - ("authorization", "Bearer test_api_key"), - ("api-key", "test_api_key"), + ("authorization", ""), + ("api-key", ""), ], "decode_compressed_response": True, "before_record_response": scrub_response_headers, diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/requirements.latest.txt b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/requirements.latest.txt index 1c66fa2a0..a2c5f1708 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/requirements.latest.txt +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/requirements.latest.txt @@ -16,33 +16,26 @@ # WARNING: NOT HERMETIC !!!!!!!!!! # ******************************** # -# This "requirements.txt" is installed in conjunction -# with multiple other dependencies in the top-level "tox-loongsuite.ini" -# file. In particular, please see: -# -# agentscope-latest: {[testenv]test_deps} -# agentscope-latest: -r {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/requirements.latest.txt -# -# This provides additional dependencies, namely: -# -# opentelemetry-api -# opentelemetry-sdk -# opentelemetry-semantic-conventions -# -# ... with a "dev" version based on the latest distribution. +# This file intentionally uses stable OpenTelemetry dependencies instead of +# the shared core-dev test_deps. AgentScope v2 depends on stable OTel exporter +# packages, which conflict with opentelemetry-test-utils from the core repo. # This variant of the requirements aims to test the system using # the newest supported version of external dependencies. -agentscope>=1.0.0 +agentscope>=2.0.0,<3.0.0 pytest pytest-asyncio pytest-cov pytest-vcr>=1.0.2 -vcrpy>=5.1.0 +vcrpy>=8.1.1 pyyaml>=6.0 +packaging>=18.0 +opentelemetry-api>=1.39.0 +opentelemetry-sdk>=1.39.0 +opentelemetry-instrumentation>=0.60b0 +opentelemetry-semantic-conventions>=0.60b0 wrapt>=1.17.3,<2.0.0 --e opentelemetry-instrumentation -e instrumentation-loongsuite/loongsuite-instrumentation-agentscope -e util/opentelemetry-util-genai diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/requirements.oldest.txt b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/requirements.oldest.txt index dd213baef..2dfff29a1 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/requirements.oldest.txt +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/requirements.oldest.txt @@ -15,13 +15,15 @@ # This variant of the requirements aims to test the system using # the oldest supported version of external dependencies. -agentscope>=1.0.0 +agentscope>=1.0.0,<2.0.0 pytest pytest-asyncio pytest-cov pytest-vcr>=1.0.2 -vcrpy>=5.1.0 +vcrpy>=8.1.1 pyyaml>=6.0 +packaging>=18.0 +aiohttp<3.9; python_version < "3.12" opentelemetry-api==1.37 opentelemetry-sdk==1.37 opentelemetry-instrumentation==0.58b0 diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/test_v2_instrumentation.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/test_v2_instrumentation.py new file mode 100644 index 000000000..ab2254a55 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/test_v2_instrumentation.py @@ -0,0 +1,406 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AgentScope v2 instrumentation tests.""" + +from __future__ import annotations + +import asyncio +import importlib.metadata +import os +from types import SimpleNamespace + +import pytest + +agentscope = pytest.importorskip("agentscope") +if not importlib.metadata.version("agentscope").startswith("2."): + pytest.skip( + "AgentScope v2 tests require agentscope>=2,<3", allow_module_level=True + ) + +from agentscope.agent import Agent # noqa: E402 +from agentscope.credential import DashScopeCredential # noqa: E402 +from agentscope.message import TextBlock, UserMsg # noqa: E402 +from agentscope.model import ChatResponse, DashScopeChatModel # noqa: E402 +from agentscope.tool import ToolResponse # noqa: E402 + +from opentelemetry.instrumentation.agentscope._v2_middleware import ( # noqa: E402 + AgentScopeV2Middleware, +) +from opentelemetry.instrumentation.agentscope.package import ( # noqa: E402 + get_installed_instrumentation_dependencies, +) +from opentelemetry.trace.status import StatusCode # noqa: E402 + + +def test_v2_dependency_detection(): + assert get_installed_instrumentation_dependencies() == ( + "agentscope >= 2.0.0, < 3.0.0", + ) + + +def test_instrumentor_injects_v2_middleware(instrument): + model = _make_model(stream=False) + agent = Agent( + name="middleware_probe", + system_prompt="Reply briefly.", + model=model, + ) + + assert any( + isinstance(middleware, AgentScopeV2Middleware) + for middleware in agent._reply_middlewares + ) + assert any( + isinstance(middleware, AgentScopeV2Middleware) + for middleware in agent._model_call_middlewares + ) + assert any( + isinstance(middleware, AgentScopeV2Middleware) + for middleware in agent._acting_middlewares + ) + + +def test_v2_uninstrument_removes_agent_patch(instrument): + instrument.uninstrument() + + agent = Agent( + name="uninstrument_probe", + system_prompt="Reply briefly.", + model=_make_model(stream=False), + ) + + assert not any( + isinstance(middleware, AgentScopeV2Middleware) + for middleware in agent._reply_middlewares + ) + assert not any( + isinstance(middleware, AgentScopeV2Middleware) + for middleware in agent._model_call_middlewares + ) + + +async def test_v2_existing_agent_middleware_noops_after_uninstrument( + instrument, span_exporter +): + agent = Agent( + name="existing_agent", + system_prompt="Reply briefly.", + model=_make_model(stream=False), + ) + middleware = _middleware(agent._model_call_middlewares) + instrument.uninstrument() + + async def model_handler(**kwargs): + del kwargs + return ChatResponse(content=[TextBlock(text="ok")], is_last=True) + + response = await middleware.on_model_call( + agent, + { + "current_model": agent.model, + "messages": [UserMsg(name="user", content="hello")], + }, + model_handler, + ) + + assert response.content + assert not span_exporter.get_finished_spans() + + +async def test_v2_model_call_error_path(instrument, span_exporter): + agent = Agent( + name="error_agent", + system_prompt="Reply briefly.", + model=_make_model(stream=False), + ) + middleware = _middleware(agent._model_call_middlewares) + + async def failing_handler(**kwargs): + del kwargs + raise RuntimeError("model failed") + + with pytest.raises(RuntimeError, match="model failed"): + await middleware.on_model_call( + agent, + { + "current_model": agent.model, + "messages": [UserMsg(name="user", content="fail")], + }, + failing_handler, + ) + + span = _spans_by_operation(span_exporter.get_finished_spans(), "chat")[0] + assert span.status.status_code == StatusCode.ERROR + assert span.attributes["error.type"] == "RuntimeError" + + +async def test_v2_streaming_model_call_error_path(instrument, span_exporter): + agent = Agent( + name="stream_error_agent", + system_prompt="Reply briefly.", + model=_make_model(stream=True), + ) + middleware = _middleware(agent._model_call_middlewares) + + async def failing_stream(): + yield ChatResponse(content=[TextBlock(text="partial")], is_last=False) + raise RuntimeError("stream failed") + + async def stream_handler(**kwargs): + del kwargs + return failing_stream() + + stream = await middleware.on_model_call( + agent, + { + "current_model": agent.model, + "messages": [UserMsg(name="user", content="fail")], + }, + stream_handler, + ) + + with pytest.raises(RuntimeError, match="stream failed"): + async for _ in stream: + pass + + span = _spans_by_operation(span_exporter.get_finished_spans(), "chat")[0] + assert span.status.status_code == StatusCode.ERROR + assert span.attributes["error.type"] == "RuntimeError" + + +async def test_v2_tool_acting_hook(instrument, span_exporter): + agent = Agent( + name="tool_agent", + system_prompt="Use tools.", + model=_make_model(stream=False), + ) + middleware = _middleware(agent._acting_middlewares) + tool_call = SimpleNamespace( + name="lookup_weather", + id="tool-call-1", + input='{"city": "Hangzhou"}', + ) + + async def tool_handler(**kwargs): + del kwargs + yield ToolResponse(content=[TextBlock(text="sunny")]) + + results = [ + item + async for item in middleware.on_acting( + agent, + {"tool_call": tool_call}, + tool_handler, + ) + ] + + assert results + tool_span = _spans_by_operation( + span_exporter.get_finished_spans(), "execute_tool" + )[0] + assert tool_span.attributes["gen_ai.tool.name"] == "lookup_weather" + + +async def test_v2_tool_result_content_capture( + instrument_with_content, + span_exporter, +): + agent = Agent( + name="tool_content_agent", + system_prompt="Use tools.", + model=_make_model(stream=False), + ) + middleware = _middleware(agent._acting_middlewares) + tool_call = SimpleNamespace( + name="lookup_weather", + id="tool-call-content", + input='{"city": "Hangzhou"}', + ) + + async def tool_handler(**kwargs): + del kwargs + yield ToolResponse(content=[TextBlock(text="sunny")]) + + results = [ + item + async for item in middleware.on_acting( + agent, + {"tool_call": tool_call}, + tool_handler, + ) + ] + + assert results + tool_span = _spans_by_operation( + span_exporter.get_finished_spans(), "execute_tool" + )[0] + assert tool_span.attributes["gen_ai.tool.call.result"] == ( + '[{"content":"sunny","type":"text"}]' + ) + + +async def test_v2_react_many_tools_telemetry(instrument, span_exporter): + agent = Agent( + name="react_tool_agent", + system_prompt="Use tools.", + model=_make_model(stream=False), + ) + middleware = _middleware(agent._acting_middlewares) + + for idx, name in enumerate( + [ + "lookup_weather", + "search_docs", + "calculate_total", + "write_summary", + ], + start=1, + ): + tool_call = SimpleNamespace( + name=name, + id=f"tool-call-{idx}", + input=f'{{"idx": {idx}}}', + ) + + async def tool_handler(**kwargs): + del kwargs + yield ToolResponse(content=[TextBlock(text=f"result {idx}")]) + + results = [ + item + async for item in middleware.on_acting( + agent, + {"tool_call": tool_call}, + tool_handler, + ) + ] + assert results + + spans = span_exporter.get_finished_spans() + react_spans = _spans_by_operation(spans, "react") + tool_spans = _spans_by_operation(spans, "execute_tool") + + assert [span.attributes["gen_ai.react.round"] for span in react_spans] == [ + 1, + 2, + 3, + 4, + ] + assert {span.attributes["gen_ai.tool.name"] for span in tool_spans} == { + "lookup_weather", + "search_docs", + "calculate_total", + "write_summary", + } + react_span_ids = {span.context.span_id for span in react_spans} + assert {span.parent.span_id for span in tool_spans} == react_span_ids + + +@pytest.mark.vcr() +async def test_v2_agent_non_streaming_e2e(instrument, span_exporter): + model = _make_model(stream=False) + agent = Agent( + name="non_stream_agent", + system_prompt="Reply with exactly: OK", + model=model, + ) + + msg = await agent.reply(UserMsg(name="user", content="Say OK.")) + + assert msg.get_text_content() + _assert_agent_and_llm_spans(span_exporter.get_finished_spans()) + + +@pytest.mark.vcr() +async def test_v2_agent_streaming_e2e(instrument, span_exporter): + model = _make_model(stream=True) + agent = Agent( + name="stream_agent", + system_prompt="Reply with a short sentence.", + model=model, + ) + + events = [ + event + async for event in agent.reply_stream( + UserMsg(name="user", content="Say hello in one sentence.") + ) + ] + + assert events + assert any( + event.__class__.__name__ == "TextBlockDeltaEvent" for event in events + ) + _assert_agent_and_llm_spans(span_exporter.get_finished_spans()) + + +@pytest.mark.vcr() +async def test_v2_agent_concurrent_e2e(instrument, span_exporter): + async def call_agent(idx: int): + agent = Agent( + name=f"concurrent_agent_{idx}", + system_prompt="Reply with exactly one short sentence.", + model=_make_model(stream=False), + ) + return await agent.reply( + UserMsg(name="user", content=f"Say OK for request {idx}.") + ) + + results = await asyncio.gather(call_agent(1), call_agent(2)) + + assert all(result.get_text_content() for result in results) + spans = span_exporter.get_finished_spans() + agent_spans = _spans_by_operation(spans, "invoke_agent") + llm_spans = _spans_by_operation(spans, "chat") + assert len(agent_spans) == 2 + assert len(llm_spans) == 2 + agent_span_ids = {span.context.span_id for span in agent_spans} + assert {span.parent.span_id for span in llm_spans} == agent_span_ids + + +def _make_model(stream: bool): + return DashScopeChatModel( + credential=DashScopeCredential( + api_key=os.environ["DASHSCOPE_API_KEY"] + ), + model="qwen-plus", + parameters=DashScopeChatModel.Parameters( + max_tokens=16, + thinking_enable=False, + ), + stream=stream, + max_retries=0, + ) + + +def _assert_agent_and_llm_spans(spans): + assert _spans_by_operation(spans, "invoke_agent") + assert _spans_by_operation(spans, "chat") + + +def _spans_by_operation(spans, operation_name): + return [ + span + for span in spans + if span.attributes.get("gen_ai.operation.name") == operation_name + ] + + +def _middleware(middlewares): + return next( + middleware + for middleware in middlewares + if isinstance(middleware, AgentScopeV2Middleware) + ) diff --git a/loongsuite-distro/src/loongsuite/distro/bootstrap_gen.py b/loongsuite-distro/src/loongsuite/distro/bootstrap_gen.py index 784affc5d..f43ae0744 100644 --- a/loongsuite-distro/src/loongsuite/distro/bootstrap_gen.py +++ b/loongsuite-distro/src/loongsuite/distro/bootstrap_gen.py @@ -217,7 +217,7 @@ "instrumentation": "opentelemetry-instrumentation-urllib3==0.62b0.dev", }, { - "library": "agentscope >= 1.0.0", + "library": "agentscope >= 1.0.0, < 3.0.0", "instrumentation": "loongsuite-instrumentation-agentscope==0.6.0.dev", }, { diff --git a/tox-loongsuite.ini b/tox-loongsuite.ini index 0eda9c140..283085bb9 100644 --- a/tox-loongsuite.ini +++ b/tox-loongsuite.ini @@ -13,7 +13,8 @@ envlist = ; FIXME(cirilla-zmh): refactor test of original instrumentation module ; loongsuite-instrumentation-agentscope - py3{10,11,12,13}-test-loongsuite-instrumentation-agentscope-{oldest,latest} + py3{10,11,12,13}-test-loongsuite-instrumentation-agentscope-oldest + py3{11,12,13}-test-loongsuite-instrumentation-agentscope-latest lint-loongsuite-instrumentation-agentscope ; loongsuite-instrumentation-dashscope @@ -138,7 +139,6 @@ deps = coverage: pytest-cov agentscope-oldest: -r {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/requirements.oldest.txt - agentscope-latest: {[testenv]test_deps} agentscope-latest: -r {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/requirements.latest.txt lint-loongsuite-instrumentation-agentscope: -r {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/tests/requirements.oldest.txt