From e8644c16b3278490440994144a1e84a166ae7f22 Mon Sep 17 00:00:00 2001 From: "mehul.sonowal" Date: Tue, 15 Sep 2026 12:44:28 -0400 Subject: [PATCH] docs: reorganize Agent Observability experiments docs --- hugo/config/_default/menus/main.en.yaml | 25 - .../en/llm_observability/guide/_index.md | 4 + .../advanced_experiment_runs.md} | 3 +- .../analyzing_experiment_results.md} | 1 + .../en/llm_observability/guide/experiments.md | 716 ++++++++++++++++++ .../api.md => guide/experiments_api.md} | 3 +- .../prompt_optimization.md | 3 +- .../improve/experiments/_index.md | 2 +- .../improve/experiments/setup.md | 356 --------- .../llm_observability/improve/playground.md | 2 +- .../instrument/otel_instrumentation.md | 2 +- .../external_evaluations/deepeval.md | 4 +- .../external_evaluations/pydantic.md | 4 +- 13 files changed, 734 insertions(+), 391 deletions(-) rename hugo/content/en/llm_observability/{improve/experiments/advanced_runs.md => guide/advanced_experiment_runs.md} (99%) rename hugo/content/en/llm_observability/{improve/experiments/analyzing_results.md => guide/analyzing_experiment_results.md} (99%) create mode 100644 hugo/content/en/llm_observability/guide/experiments.md rename hugo/content/en/llm_observability/{improve/experiments/api.md => guide/experiments_api.md} (99%) rename hugo/content/en/llm_observability/{improve/experiments => guide}/prompt_optimization.md (99%) delete mode 100644 hugo/content/en/llm_observability/improve/experiments/setup.md diff --git a/hugo/config/_default/menus/main.en.yaml b/hugo/config/_default/menus/main.en.yaml index 81e75bbc93a..b0b2e3f4bcc 100644 --- a/hugo/config/_default/menus/main.en.yaml +++ b/hugo/config/_default/menus/main.en.yaml @@ -5749,31 +5749,6 @@ menu: parent: llm_obs_improve identifier: llm_obs_experiments weight: 503 - - name: Setup and Usage - url: llm_observability/improve/experiments/setup - parent: llm_obs_experiments - identifier: llm_obs_experiments_setup - weight: 50301 - - name: Analyzing Results - url: llm_observability/improve/experiments/analyzing_results - parent: llm_obs_experiments - identifier: llm_obs_experiments_analyzing_results - weight: 50302 - - name: Advanced Experiment Runs - url: llm_observability/improve/experiments/advanced_runs - parent: llm_obs_experiments - identifier: llm_obs_experiments_advanced_runs - weight: 50303 - - name: Prompt Optimization - url: llm_observability/improve/experiments/prompt_optimization - parent: llm_obs_experiments - identifier: llm_obs_experiments_prompt_optimization - weight: 50304 - - name: Experiments API - url: llm_observability/improve/experiments/api - parent: llm_obs_experiments - identifier: llm_obs_experiments_api - weight: 50305 - name: Configure url: llm_observability/configure/ parent: llm_obs diff --git a/hugo/content/en/llm_observability/guide/_index.md b/hugo/content/en/llm_observability/guide/_index.md index 0987cb5b77b..0871d0c2a34 100644 --- a/hugo/content/en/llm_observability/guide/_index.md +++ b/hugo/content/en/llm_observability/guide/_index.md @@ -12,6 +12,10 @@ cascade: {{< nextlink href="/llm_observability/quickstart/" >}}Trace an LLM Application{{< /nextlink >}} {{< nextlink href="/llm_observability/guide/trace-an-llm-application-in-aws-lambda" >}}Trace an LLM Application in AWS Lambda{{< /nextlink >}} {{< nextlink href="/llm_observability/guide/monitor_proxy_services" >}}Trace Proxy and Gateway Services{{< /nextlink >}} + {{< nextlink href="/llm_observability/guide/experiments" >}}Set up and use Agent Observability Experiments{{< /nextlink >}} + {{< nextlink href="/llm_observability/guide/advanced_experiment_runs" >}}Advanced experiment runs{{< /nextlink >}} + {{< nextlink href="/llm_observability/guide/analyzing_experiment_results" >}}Analyze experiment results{{< /nextlink >}} + {{< nextlink href="/llm_observability/guide/experiments_api" >}}Experiments API{{< /nextlink >}} {{< nextlink href="/llm_observability/instrument/agent_observability_and_apm" >}}Using Agent Observability and APM{{< /nextlink >}} {{< nextlink href="/llm_observability/guide/monitor_mcp_client" >}}Monitor MCP Clients{{< /nextlink >}} {{< nextlink href="/llm_observability/guide/crewai_guide" >}}Using the Datadog-CrewAI integration for Agent Observability{{< /nextlink >}} diff --git a/hugo/content/en/llm_observability/improve/experiments/advanced_runs.md b/hugo/content/en/llm_observability/guide/advanced_experiment_runs.md similarity index 99% rename from hugo/content/en/llm_observability/improve/experiments/advanced_runs.md rename to hugo/content/en/llm_observability/guide/advanced_experiment_runs.md index 5aa92e7207d..8707e6b5d68 100644 --- a/hugo/content/en/llm_observability/improve/experiments/advanced_runs.md +++ b/hugo/content/en/llm_observability/guide/advanced_experiment_runs.md @@ -1,6 +1,7 @@ --- title: Advanced Experiment Runs aliases: +- /llm_observability/improve/experiments/advanced_runs/ - /llm_observability/experiments/advanced_runs/ description: Run experiments multiple times to account for model variability on a subset of your dataset, and automate experiment execution in CI/CD pipelines. --- @@ -271,5 +272,5 @@ GET /api/v2/llm-obs/v1/experiments?filter[experiment]=my-pipeline&filter[metadat ``` [1]: /llm_observability/instrument/sdk?tab=python -[2]: /llm_observability/improve/experiments/api +[2]: /llm_observability/guide/experiments_api [3]: https://app.datadoghq.com/llm/experiments diff --git a/hugo/content/en/llm_observability/improve/experiments/analyzing_results.md b/hugo/content/en/llm_observability/guide/analyzing_experiment_results.md similarity index 99% rename from hugo/content/en/llm_observability/improve/experiments/analyzing_results.md rename to hugo/content/en/llm_observability/guide/analyzing_experiment_results.md index 3fe75ce9d25..9082f6ec23f 100644 --- a/hugo/content/en/llm_observability/improve/experiments/analyzing_results.md +++ b/hugo/content/en/llm_observability/guide/analyzing_experiment_results.md @@ -1,6 +1,7 @@ --- title: Analyze Your Experiments Results aliases: +- /llm_observability/improve/experiments/analyzing_results/ - /llm_observability/experiments/analyzing_results/ description: How to analyze Agent Observability Experiments results. --- diff --git a/hugo/content/en/llm_observability/guide/experiments.md b/hugo/content/en/llm_observability/guide/experiments.md new file mode 100644 index 00000000000..4ad76df2580 --- /dev/null +++ b/hugo/content/en/llm_observability/guide/experiments.md @@ -0,0 +1,716 @@ +--- +title: Set up and use Agent Observability Experiments +aliases: +- /llm_observability/improve/experiments/setup/ +- /llm_observability/experiments/setup/ +description: How to set up Agent Observability Experiments and start running experiments. +further_reading: +- link: "https://www.datadoghq.com/blog/debug-and-evaluate-your-ai-app-from-your-coding-agent/" + tag: "Blog" + text: "Debug and evaluate your AI app from your coding agent with Datadog Agent Observability" +--- + +This guide describes how to set up and use Agent Observability experiments with the Python or Node.js SDK. For complete runnable Node.js examples, see the [Node.js experiments examples](https://github.com/DataDog/llm-observability/tree/main/experiments/nodejs). + +## Set up Agent Observability + +If you have not already set up Agent Observability, use one of the following SDKs. Experiments require both a Datadog API key and application key. + +Your Datadog site is {{< region-param key="dd_site" code="true" >}}. Replace the `site` value in the examples with your site as necessary. + +1. Install the Agent Observability SDK: + +{{< tabs >}} +{{% tab "Python" %}} +```shell +pip install ddtrace>=4.3.0 +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```shell +npm install "dd-trace@>=6.13.0" +``` +{{% /tab %}} +{{< /tabs >}} + +2. Enable Agent Observability: + +{{< tabs >}} +{{% tab "Python" %}} +```python +from ddtrace.llmobs import LLMObs + +LLMObs.enable( + api_key="", # defaults to DD_API_KEY environment variable + app_key="", # defaults to DD_APP_KEY environment variable + site="datadoghq.com", # defaults to DD_SITE environment variable + project_name="" # defaults to DD_LLMOBS_PROJECT_NAME environment variable, or "default-project" if the environment variable is not set +) +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +Set `DD_API_KEY` and `DD_APP_KEY` in your environment, then initialize `dd-trace` in your application entrypoint. The Experiments client uses these environment variables for authentication: + +```javascript +const tracer = require('dd-trace').init({ + service: '', + site: 'datadoghq.com', + llmobs: { + mlApp: '', + projectName: '', + agentlessEnabled: true, + }, +}) + +const { experiments } = tracer.llmobs +``` + +The `projectName` value identifies the Experiments project that contains your datasets and experiments. The `mlApp` value identifies the LLM application used for Agent Observability traces. You can configure these values independently. If `projectName` is not configured, Experiments uses `default-project`; `mlApp` and `service` are not used as Experiments project-name fallbacks. If you use command-line setup, also provide the application key because experiments use the Experiments API: + +```shell +DD_SITE= DD_API_KEY= DD_APP_KEY= \ +DD_LLMOBS_ENABLED=1 DD_LLMOBS_ML_APP= DD_LLMOBS_PROJECT_NAME= \ +NODE_OPTIONS="--import dd-trace/initialize.mjs" node +``` + +For more information, see the [Node.js tracer command-line setup](/llm_observability/instrument/sdk?tab=nodejs#command-line-setup). +{{% /tab %}} +{{< /tabs >}} + +
You must supply both an api_key and app_key for Python, or set both DD_API_KEY and DD_APP_KEY for Node.js.
+ +### APM Trace correlation + +To correlate your Experiment spans with [APM Traces](/llm_observability/instrument/agent_observability_and_apm/), run Agent Observability through a Datadog Agent and keep the default `agentless_enabled=False` (for Python) or `agentlessEnabled: false` (for Node.js). The Agent forwards trace data to APM, which enables Experiment ↔ APM Trace correlation. + +{{< tabs >}} +{{% tab "Python" %}} +```python +LLMObs.enable( + api_key="", + app_key="", + site="datadoghq.com", + agentless_enabled=False, # default — required for APM trace correlation + project_name="", +) +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```javascript +const tracer = require('dd-trace').init({ + service: '', + site: 'datadoghq.com', + llmobs: { + mlApp: '', + projectName: '', + agentlessEnabled: false, // default — required for APM trace correlation + }, +}) +``` +{{% /tab %}} +{{< /tabs >}} + +If you are running without an Agent (for example, in a notebook or CI environment), you can set `agentless_enabled=True` or `agentlessEnabled: true`, but corresponding APM spans are not generated for experiment spans from agentless runs. + +## Create a project + +_Projects_ are the core organizational layer for LLM experiments. All datasets and experiments live in a project. You can create a project manually in the Datadog console, API, or SDK by specifying a project name that does not already exist. + +{{< tabs >}} +{{% tab "Python" %}} +```python +from ddtrace.llmobs import LLMObs + +LLMObs.enable( + ..., + project_name="" # defaults to DD_LLMOBS_PROJECT_NAME environment variable, or "default-project" if the environment variable is not set +) +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```javascript +const tracer = require('dd-trace').init({ + llmobs: { + mlApp: '', + projectName: 'experiments-project', + }, +}) + +const { experiments } = tracer.llmobs +``` +{{% /tab %}} +{{< /tabs >}} + +For Node.js, set `llmobs.projectName` to configure the default Experiments project. The equivalent environment variable is `DD_LLMOBS_PROJECT_NAME`. If you do not configure a project name, Experiments uses `default-project`. The `mlApp` and `service` settings are not used as Experiments project-name fallbacks. The project is created when the Experiments client first accesses it. + +To use a different project for a specific operation, pass `projectName` to the operation. Use the same project name for a dataset and its experiment: + +```javascript +async function main () { + const dataset = experiments.createDataset('capital-cities', { + projectName: 'other-project', + records: [ + { inputData: { country: 'France' }, expectedOutput: 'Paris' }, + { inputData: { country: 'Japan' }, expectedOutput: 'Tokyo' }, + ], + }) + + const experiment = experiments.experiment({ + name: 'capital-cities-test', + projectName: 'other-project', + dataset, + task: (inputData) => ({ France: 'Paris', Japan: 'Tokyo' })[inputData.country], + evaluators: { + exact_match: (_inputData, outputData, expectedOutput) => outputData === expectedOutput, + }, + }) + + const result = await experiment.run() + console.log(`View experiment: ${result.url}`) +} + +main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) +``` + +You can also pass `projectName` to `pullDataset()` to pull a dataset from another project. + +## Create a dataset + +A _dataset_ is a collection of _inputs_, _expected outputs_, and _metadata_ that represent scenarios you want to test your agent on. Each dataset is associated with a _project_. + +- **Input** (required): Represents all the information that the agent can access in a task. +- **Expected output** (optional): Also called _ground truth_, represents the ideal answer that the agent should output. You can use _expected output_ to store the actual output of the app, as well as any intermediary results you want to assess. +- **Metadata** (optional): Contains useful information to categorize the record and use for further analysis. For example: topics, tags, descriptions, and notes. + +To create a dataset from a CSV file, use `LLMObs.create_dataset_from_csv()`: + +{{< tabs >}} +{{% tab "Python" %}} +```python +# Create dataset from CSV +dataset = LLMObs.create_dataset_from_csv( + csv_path="questions.csv", + dataset_name="capitals-of-the-world", + project_name="capitals-project", # Optional: defaults to the project name from LLMObs.enable + description="Geography quiz dataset", # Optional: Dataset description + input_data_columns=["question", "category"], # Columns to use as input + expected_output_columns=["answer"], # Optional: Columns to use as expected output + metadata_columns=["difficulty"], # Optional: Additional columns as metadata + csv_delimiter="," # Optional: Defaults to comma +) + +# Example "questions.csv": +# question,category,answer,difficulty +# What is the capital of Japan?,geography,Tokyo,medium +# What is the capital of Brazil?,geography,Brasília,medium +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +The Node.js SDK does not include a CSV-specific helper. Parse the CSV with the library of your choice, then pass the records to `createDataset()`: + +```javascript +const dataset = experiments.createDataset('capitals-of-the-world', { + description: 'Geography quiz dataset', + records: [ + { + inputData: { question: 'What is the capital of Japan?', category: 'geography' }, + expectedOutput: 'Tokyo', + metadata: { difficulty: 'medium' }, + }, + { + inputData: { question: 'What is the capital of Brazil?', category: 'geography' }, + expectedOutput: 'Brasília', + metadata: { difficulty: 'medium' }, + }, + ], +}) + +// The dataset is pushed automatically when the experiment runs. +// To push it earlier, use: await dataset.push() +``` +{{% /tab %}} +{{< /tabs >}} + +See [Datasets][1] for more information about datasets, including how to manually create datasets, retrieve and manage datasets, and how Datadog retains dataset versions. + +## Create an experiment + +An _experiment_ lets you systematically test your LLM application by running your agent across a set of scenarios from your dataset and measuring performance against the expected outputs using evaluators. You can then compare how different app configurations perform, side by side. + +- **Task**: Defines the core workflow you want to evaluate. It can range from a single LLM call to a more complex flow involving multiple LLM calls and RAG steps. The task is executed sequentially across all records in the dataset. +- **Evaluator**: A function, executed on each record, that measures how well the model or agent performs. Evaluators allow you to compare the output to either the expected output or the original input. +- **Summary evaluators**: Optional functions executed against all the data of the experiment (input, output, expected output, and evaluator results). Summary evaluators allow you to compute more advanced metrics like precision, recall, and accuracy across your dataset. + +To create an experiment: + +### 1. Load a dataset + +{{< tabs >}} +{{% tab "Python" %}} +```python +from ddtrace.llmobs import LLMObs +from typing import Dict, Any, Optional, List + +dataset = LLMObs.pull_dataset("capitals-of-the-world") +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```javascript +const tracer = require('dd-trace').init() +const { experiments } = tracer.llmobs + +const dataset = await experiments.pullDataset('capitals-of-the-world') +``` +{{% /tab %}} +{{< /tabs >}} + +### 2. Define a task function that processes a single dataset record + +{{< tabs >}} +{{% tab "Python" %}} +```python +def task(input_data: Dict[str, Any], config: Optional[Dict[str, Any]] = None) -> str: + question = input_data["question"] + # Your LLM or processing logic here + return "Beijing" if "China" in question else "Unknown" +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```javascript +async function task (inputData, config, metadata) { + const question = inputData.question + // Your LLM or processing logic here + return question.includes('China') ? 'Beijing' : 'Unknown' +} +``` +{{% /tab %}} +{{< /tabs >}} + +A task can take any non-null type as `input_data` or `inputData` (string, number, Boolean, object, or array). The output that will be used in evaluators can be any type. This example generates a string, but a dictionary or object can be generated as output to store intermediary information and compare in evaluators. + +Optionally, your task function can accept metadata from the dataset record: + +{{< tabs >}} +{{% tab "Python" %}} +```python +def task(input_data: Dict[str, Any], config: Optional[Dict[str, Any]] = None, metadata: Optional[Dict[str, Any]] = None) -> str: + difficulty = metadata.get("difficulty", "unknown") if metadata else "unknown" + question = input_data["question"] + return "Beijing" if "China" in question else "Unknown" +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```javascript +function task (inputData, config, metadata) { + const difficulty = metadata?.difficulty ?? 'unknown' + const question = inputData.question + // Use difficulty in the task when it is part of your evaluation logic. + return question.includes('China') ? 'Beijing' : 'Unknown' +} +``` +{{% /tab %}} +{{< /tabs >}} + +You can trace the different parts of your experiment task (workflow, tool calls, and so on) using the same tracing APIs you use in production. For Python, see the [custom tracing decorators][2]. For Node.js, use the `llmobs.trace()` API. If you use a [supported Python framework][3] or [supported Node.js framework][7] (OpenAI, Amazon Bedrock, and so on), Agent Observability automatically traces and annotates calls to LLM frameworks and libraries, giving you out-of-the-box observability for calls that your LLM application makes. + +{{< tabs >}} +{{% tab "Python" %}} +```python +# Example: trace a workflow or tool call with the same decorators used in production. +from ddtrace.llmobs.decorators import workflow + +@workflow(name="retrieve_context") +def retrieve_context(question): + return call_retriever(question) +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```javascript +// Example: trace a workflow or tool call with the same LLMObs APIs used in production. +const context = await tracer.llmobs.trace( + { kind: 'task', name: 'retrieve_context' }, + async () => callRetriever(question), +) +``` +{{% /tab %}} +{{< /tabs >}} + +#### Using OpenTelemetry spans inside experiments + +If your application uses [OpenTelemetry instrumentation][6], you can create OTel spans inside your experiment task. With `DD_TRACE_OTEL_ENABLED=1`, the Datadog tracer acts as the OpenTelemetry TracerProvider, so OTel spans appear as children of the experiment span automatically. + +{{< tabs >}} +{{% tab "Python" %}} +```python +import json +from opentelemetry import trace + +tracer = trace.get_tracer(__name__) + +def task(input_data: Dict[str, Any], config: Optional[Dict[str, Any]] = None) -> str: + question = input_data["question"] + + # OTel gen_ai span — automatically becomes a child of the experiment span + with tracer.start_as_current_span("my-llm-call") as span: + span.set_attribute("gen_ai.operation.name", "chat") + span.set_attribute("gen_ai.system", "openai") + span.set_attribute("gen_ai.request.model", "gpt-4o") + span.set_attribute("gen_ai.usage.input_tokens", 25) + span.set_attribute("gen_ai.usage.output_tokens", 8) + span.set_attribute( + "gen_ai.input.messages", + json.dumps([{"role": "user", "parts": [{"type": "text", "content": question}]}]), + ) + + result = call_my_llm(question) + + span.set_attribute( + "gen_ai.output.messages", + json.dumps([{"role": "assistant", "parts": [{"type": "text", "content": result}]}]), + ) + + return result +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +Install the OpenTelemetry API if it is not already a dependency: + +```shell +npm install @opentelemetry/api +``` + +```javascript +const { trace } = require('@opentelemetry/api') + +const otelTracer = trace.getTracer('my-experiment') + +async function task (inputData, config, metadata) { + const question = inputData.question + + return otelTracer.startActiveSpan('my-llm-call', async (span) => { + span.setAttribute('gen_ai.operation.name', 'chat') + span.setAttribute('gen_ai.system', 'openai') + span.setAttribute('gen_ai.request.model', 'gpt-4o') + span.setAttribute('gen_ai.usage.input_tokens', 25) + span.setAttribute('gen_ai.usage.output_tokens', 8) + span.setAttribute('gen_ai.input.messages', JSON.stringify([ + { role: 'user', parts: [{ type: 'text', content: question }] }, + ])) + + const result = await callMyLlm(question) + + span.setAttribute('gen_ai.output.messages', JSON.stringify([ + { role: 'assistant', parts: [{ type: 'text', content: result }] }, + ])) + span.end() + return result + }) +} +``` +{{% /tab %}} +{{< /tabs >}} + +To enable this, set the `DD_TRACE_OTEL_ENABLED` environment variable: + +{{< tabs >}} +{{% tab "Python" %}} +```shell +DD_TRACE_OTEL_ENABLED=1 python my_experiment.py +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```shell +DD_TRACE_OTEL_ENABLED=1 node my_experiment.js +``` +{{% /tab %}} +{{< /tabs >}} + +### 3. Define evaluators + +Evaluators measure how well your model or agent performs on each record. You can define evaluators using function-based evaluators. Python also supports reusable class-based evaluators. + +For detailed information on building evaluators, including the full data model reference and best practices, see the [Evaluation Developer Guide][4]. + +Datadog supports the following evaluator return types: + +- **Boolean**: returns true or false +- **Score**: Returns a numeric value (float) +- **Categorical**: Returns a labeled category (string) +- **JSON**: Returns structured data (dict or object) + +{{< tabs >}} +{{% tab "Python" %}} +#### Function-based evaluators + +```python +def exact_match(input_data: Dict[str, Any], output_data: str, expected_output: str) -> bool: + return output_data == expected_output + +def overlap(input_data: Dict[str, Any], output_data: str, expected_output: str) -> float: + expected_output_set = set(expected_output) + output_set = set(output_data) + + intersection = len(output_set.intersection(expected_output_set)) + union = len(output_set.union(expected_output_set)) + + return intersection / union + +def fake_llm_as_a_judge(input_data: Dict[str, Any], output_data: str, expected_output: str) -> EvaluatorResult: + fake_llm_call = "excellent" + return EvaluatorResult( + value=fake_llm_call, + reasoning="the model explains itself", + assessment="pass", # or fail + tags={"task": "judge_llm_call"}, + ) + +# Return multiple metrics from one evaluator call +from ddtrace.llmobs import EvaluatorResult, MultiEvaluatorResult + +def multi_metric_evaluator(input_data, output_data, expected_output): + correct = output_data == expected_output + return MultiEvaluatorResult( + { + "correct": EvaluatorResult(value=correct, assessment="pass" if correct else "fail"), + "length": len(str(output_data)), + } + ) + # Emitted as: multi_metric_evaluator-correct, multi_metric_evaluator-length +``` + +#### Class-based evaluators + +```python +from ddtrace.llmobs import BaseEvaluator, EvaluatorContext, EvaluatorResult + +class SemanticSimilarityEvaluator(BaseEvaluator): + def __init__(self, threshold: float = 0.8): + super().__init__(name="semantic_similarity") + self.threshold = threshold + + def evaluate(self, context: EvaluatorContext) -> EvaluatorResult: + score = compute_similarity(context.output_data, context.expected_output) + return EvaluatorResult( + value=score, + reasoning=f"Similarity score: {score:.2f}", + assessment="pass" if score >= self.threshold else "fail", + ) +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +The Node.js SDK supports function evaluators. Return a Boolean, number, string, or JSON-serializable object. Use an object map when you want to assign evaluator names explicitly. + +```javascript +function exact_match (inputData, outputData, expectedOutput) { + return outputData === expectedOutput +} + +function overlap (inputData, outputData, expectedOutput) { + const expected = new Set(String(expectedOutput)) + const output = new Set(String(outputData)) + const intersection = [...output].filter(value => expected.has(value)).length + const union = new Set([...expected, ...output]).size + return intersection / union +} + +function fake_llm_as_a_judge (inputData, outputData, expectedOutput) { + // Return a categorical metric. Use a separate evaluator for each metric. + return 'excellent' +} +``` + +Class-based evaluators and `MultiEvaluatorResult` are not part of the Node.js experiments API. Define multiple named functions when you need multiple metrics. +{{% /tab %}} +{{< /tabs >}} + +### 4. (Optional) Define summary evaluators + +Summary evaluators run after all record-level evaluators have finished, and receive the aggregated results to compute dataset-level statistics like averages or pass rates. Like record-level evaluators, you can define summary evaluators as functions or classes. + +For the class-based approach using `BaseSummaryEvaluator`, see the [Evaluation Developer Guide][4]. + +{{< tabs >}} +{{% tab "Python" %}} +#### Function-based summary evaluators + +```python +def num_exact_matches(inputs, outputs, expected_outputs, evaluators_results): + return evaluators_results["exact_match"].count(True) +``` + +Summary evaluator functions can take a list of any non-null type as `inputs` (string, number, Boolean, object, or array); `outputs` and `expected_outputs` can be lists of any type. `evaluators_results` is a dictionary of lists of results from evaluators, keyed by the name of the evaluator function. + +#### Class-based summary evaluators + +```python +from ddtrace.llmobs import BaseSummaryEvaluator, SummaryEvaluatorContext + +class AverageScoreEvaluator(BaseSummaryEvaluator): + def __init__(self, target_evaluator: str): + super().__init__(name="average_score") + self.target_evaluator = target_evaluator + + def evaluate(self, context: SummaryEvaluatorContext): + scores = context.evaluation_results.get(self.target_evaluator, []) + if not scores: + return None + return sum(scores) / len(scores) +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +Summary evaluators are functions that receive the inputs, outputs, expected outputs, and an object containing the results from each evaluator. The optional fifth argument contains record metadata. + +```javascript +function num_exact_matches (inputs, outputs, expectedOutputs, evaluatorResults, metadata) { + return (evaluatorResults.exact_match || []).filter(Boolean).length +} +``` + +The Node.js SDK does not use class-based summary evaluators. Define multiple named summary evaluator functions when you need multiple dataset-level metrics. +{{% /tab %}} +{{< /tabs >}} + +### 5. Create and run the experiment. + +{{< tabs >}} +{{% tab "Python" %}} +```python +experiment = LLMObs.experiment( + name="capital-cities-test", + task=task, + dataset=dataset, + evaluators=[exact_match, overlap, fake_llm_as_a_judge], + summary_evaluators=[num_exact_matches], # optional + description="Testing capital cities knowledge", + config={ + "model_name": "gpt-4", + "version": "1.0" + }, +) + +# Run the experiment +results = experiment.run() # Run on all dataset records + +# Process results +for result in results.get("rows", []): + print(f"Record {result['idx']}") + print(f"Input: {result['input']}") + print(f"Output: {result['output']}") + print(f"Score: {result['evaluations']['evaluator']['value']}") + if result['error']['message']: + print(f"Error: {result['error']['message']}") +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```javascript +const experiment = experiments.experiment({ + name: 'capital-cities-test', + task, + dataset, + evaluators: { + exact_match, + overlap, + fake_llm_as_a_judge, + }, + summaryEvaluators: { + num_exact_matches, + }, + description: 'Testing capital cities knowledge', + config: { + model: 'gpt-4', + version: '1.0', + }, +}) + +// Run the experiment. The dataset is pushed automatically if needed. +const results = await experiment.run() + +// Process results +for (const row of results.rows) { + console.log(`Record ${row.index}`) + console.log(`Input: ${JSON.stringify(row.input)}`) + console.log(`Output: ${JSON.stringify(row.output)}`) + console.log(`Score: ${row.evaluations.exact_match}`) + if (row.isError) { + console.log(`Error: ${row.errorMessage}`) + } +} +``` +{{% /tab %}} +{{< /tabs >}} + +To increase execution speed or limit the data used by the experiment: + +{{< tabs >}} +{{% tab "Python" %}} +```python +# Increase execution speed +results = experiment.run(jobs=4) + +# Test your pipeline on a subset of the data +results = experiment.run(sample_size=10) + +# Stop the experiment if an error occurs +results = experiment.run(raise_errors=True) +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```javascript +// The Node.js SDK runs dataset records sequentially. +// Use your task implementation to manage concurrency if needed. +const results = await experiment.run({ + throwOnErrors: true, +}) +``` +{{% /tab %}} +{{< /tabs >}} + +### 6. Review your experiment results in Datadog. + +{{< tabs >}} +{{% tab "Python" %}} +```python +print(f"View experiment: {experiment.url}") +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```javascript +console.log(`View experiment: ${results.url}`) +// The same URL is available from the experiment after it has run. +console.log(experiment.url()) +``` +{{% /tab %}} +{{< /tabs >}} + +**Note:** LLM Experiments traces are retained for 90 days. + +[1]: /llm_observability/improve/datasets +[2]: /llm_observability/instrument/custom_instrumentation?tab=decorators#trace-an-llm-application +[3]: /llm_observability/instrument/auto_instrumentation?tab=python +[4]: /llm_observability/investigate/evaluations/evaluation_developer_guide +[5]: /llm_observability/instrument/agent_observability_and_apm/ +[6]: /llm_observability/instrument/otel_instrumentation +[7]: /llm_observability/instrument/auto_instrumentation?tab=nodejs + +## Further reading + +{{< partial name="whats-next/whats-next.html" >}} diff --git a/hugo/content/en/llm_observability/improve/experiments/api.md b/hugo/content/en/llm_observability/guide/experiments_api.md similarity index 99% rename from hugo/content/en/llm_observability/improve/experiments/api.md rename to hugo/content/en/llm_observability/guide/experiments_api.md index cf99ea1320b..dfff9719533 100644 --- a/hugo/content/en/llm_observability/improve/experiments/api.md +++ b/hugo/content/en/llm_observability/guide/experiments_api.md @@ -1,6 +1,7 @@ --- title: Experiments API aliases: +- /llm_observability/improve/experiments/api/ - /llm_observability/experiments/api/ description: Reference the Agent Observability Experiments HTTP API. --- @@ -558,6 +559,6 @@ Empty body on success. {{% /collapse-content %}} -[1]: /llm_observability/improve/experiments/setup +[1]: /llm_observability/guide/experiments [2]: https://github.com/DataDog/llm-observability/tree/main/experiments [3]: https://www.postman.com/ diff --git a/hugo/content/en/llm_observability/improve/experiments/prompt_optimization.md b/hugo/content/en/llm_observability/guide/prompt_optimization.md similarity index 99% rename from hugo/content/en/llm_observability/improve/experiments/prompt_optimization.md rename to hugo/content/en/llm_observability/guide/prompt_optimization.md index 59e58904473..54c0af953b1 100644 --- a/hugo/content/en/llm_observability/improve/experiments/prompt_optimization.md +++ b/hugo/content/en/llm_observability/guide/prompt_optimization.md @@ -1,11 +1,12 @@ --- title: Prompt Optimization aliases: +- /llm_observability/improve/experiments/prompt_optimization/ - /llm_observability/experiments/prompt_optimization/ description: Automatically improve LLM prompts through iterative refinement and AI-powered evaluation. private: true further_reading: - - link: /llm_observability/improve/experiments/setup + - link: /llm_observability/guide/experiments tag: "Documentation" text: Set up and use Agent Observability Experiments - link: /llm_observability/improve/datasets diff --git a/hugo/content/en/llm_observability/improve/experiments/_index.md b/hugo/content/en/llm_observability/improve/experiments/_index.md index 89c3e29aac6..492329d553e 100644 --- a/hugo/content/en/llm_observability/improve/experiments/_index.md +++ b/hugo/content/en/llm_observability/improve/experiments/_index.md @@ -5,7 +5,7 @@ aliases: - /llm_observability/experiments_preview - /llm_observability/experiments/ further_reading: - - link: /llm_observability/improve/experiments/setup + - link: /llm_observability/guide/experiments tag: "Documentation" text: Set up and use Agent Observability Experiments - link: "https://www.datadoghq.com/blog/llm-experiments/" diff --git a/hugo/content/en/llm_observability/improve/experiments/setup.md b/hugo/content/en/llm_observability/improve/experiments/setup.md deleted file mode 100644 index b59a77754ec..00000000000 --- a/hugo/content/en/llm_observability/improve/experiments/setup.md +++ /dev/null @@ -1,356 +0,0 @@ ---- -title: Setup and Usage -aliases: -- /llm_observability/experiments/setup/ -description: How to set up Agent Observability Experiments and start running experiments. -further_reading: -- link: "https://www.datadoghq.com/blog/debug-and-evaluate-your-ai-app-from-your-coding-agent/" - tag: "Blog" - text: "Debug and evaluate your AI app from your coding agent with Datadog Agent Observability" ---- - -This page describes how to set up and use Agent Observability Experiments with the Python SDK. - -## Set up Agent Observability - -If you have not already set up Agent Observability: - -1. Install Agent Observability Python SDK: - - ```shell - pip install ddtrace>=4.3.0 - ``` - -2. Enable Agent Observability: - - ```python - from ddtrace.llmobs import LLMObs - - LLMObs.enable( - api_key="", # defaults to DD_API_KEY environment variable - app_key="", # defaults to DD_APP_KEY environment variable - site="datadoghq.com", # defaults to DD_SITE environment variable - project_name="" # defaults to DD_LLMOBS_PROJECT_NAME environment variable, or "default-project" if the environment variable is not set - ) - ``` - -
You must supply both an api_key and app_key.
- -### APM Trace correlation - -To correlate your Experiment spans with [APM Traces][5], run Agent Observability through a Datadog Agent and keep `agentless_enabled` set to `False` (the default). The Agent forwards trace data to APM, which is what enables Experiment ↔ APM Trace correlation. - - ```python - LLMObs.enable( - api_key="", - app_key="", - site="datadoghq.com", - agentless_enabled=False, # default — required for APM Trace correlation - project_name="", - ) - ``` - -If you are running without an Agent (for example, in a notebook or CI environment), you can set `agentless_enabled=True`, but corresponding APM spans are not generated for Experiment spans from agentless runs. - -## Create a project -_Projects_ are the core organizational layer for LLM Experiments. All datasets and experiments live in a project. -You can create a project manually in the Datadog console, API, or SDK by specifying a project name that does not already exist in `LLMObs.enable`. - -```python -LLMObs.enable( - ... - project_name="" # defaults to DD_LLMOBS_PROJECT_NAME environment variable, or "default-project" if the environment variable is not set -) -``` - -## Create a dataset - -A _dataset_ is a collection of _inputs_, and _expected outputs_ and _metadata_ that represent scenarios you want to tests your agent on. Each dataset is associated with a _project_. - -- **input** (required): Represents all the information that the agent can access in a task. -- **expected output** (optional): Also called _ground truth_, represents the ideal answer that the agent should output. You can use _expected output_ to store the actual output of the app, as well as any intermediary results you want to assesss. -- **metadata** (optional): Contains any useful information to categorize the record and use for further analysis. For example: topics, tags, descriptions, notes. - -To create a dataset from a CSV file, use `LLMObs.create_dataset_from_csv()`: - -```python -# Create dataset from CSV -dataset = LLMObs.create_dataset_from_csv( - csv_path="questions.csv", - dataset_name="capitals-of-the-world", - project_name="capitals-project", # Optional: defaults to the project name from LLMObs.enable - description="Geography quiz dataset", # Optional: Dataset description - input_data_columns=["question", "category"], # Columns to use as input - expected_output_columns=["answer"], # Optional: Columns to use as expected output - metadata_columns=["difficulty"], # Optional: Additional columns as metadata - csv_delimiter="," # Optional: Defaults to comma -) - -# Example "questions.csv": -# question,category,answer,difficulty -# What is the capital of Japan?,geography,Tokyo,medium -# What is the capital of Brazil?,geography,Brasília,medium - -``` - -See [Datasets][1] for more information about datasets, including: how to manually create datasets, how to retrieve and manage datasets, and how Datadog retains dataset versions. - -## Create an experiment -An _experiment_ lets you systematically test your LLM application by running your agent across a set of scenarios from your dataset and measuring performance against the expected outputs using evaluators. You can then compare how different app configurations perform, side by side. - -- **task**: Defines the core workflow you want to evaluate. It can range from a single LLM call to a more complex flow involving multiple LLM calls and RAG steps. The task is executed sequentially across all records in the dataset. -- **evaluator**: A function, executed on each record, that measures how well the model or agent performs. Evaluators allow you to compare the output to either the expected output or the original input. - -- **summary evaluators**: Optional functions executed against all the data of the Experiment (input, output, expected, evaluators' results). Summary evaluators allow you to compute more advanced metrics like precision, recall, and accuracy across your dataset. - - -To create an experiment: - - -### 1. Load a dataset - ```python - from ddtrace.llmobs import LLMObs - from typing import Dict, Any, Optional, List - - dataset = LLMObs.pull_dataset("capitals-of-the-world") - ``` - -### 2. Define a task function that processes a single dataset record - - ```python - def task(input_data: Dict[str, Any], config: Optional[Dict[str, Any]] = None) -> str: - question = input_data["question"] - # Your LLM or processing logic here - return "Beijing" if "China" in question else "Unknown" - ``` - A task can take any non-null type as `input_data` (string, number, Boolean, object, array). The output that will be used in the Evaluators can be of any type. - This example generates a string, but a dict can be generated as output to store any intermediary information and compare in the Evaluators. - - Optionally, your task function can accept a third `metadata` parameter to receive the dataset record's metadata: - ```python - def task(input_data: Dict[str, Any], config: Optional[Dict[str, Any]] = None, metadata: Optional[Dict[str, Any]] = None) -> str: - difficulty = metadata.get("difficulty", "unknown") if metadata else "unknown" - question = input_data["question"] - return "Beijing" if "China" in question else "Unknown" - ``` - - You can trace the different parts of your Experiment task (workflow, tool calls, etc.) using the [same tracing decorators][2] you use in production. - If you use a [supported framework][3] (OpenAI, Amazon Bedrock, etc.), Agent Observability automatically traces and annotates calls to LLM frameworks and libraries, giving you out-of-the-box observability for calls that your LLM application makes. - - #### Using OpenTelemetry spans inside experiments - - If your application uses [OpenTelemetry instrumentation][6], you can create OTel spans inside your experiment task. With `DD_TRACE_OTEL_ENABLED=1`, ddtrace acts as the OpenTelemetry TracerProvider, so OTel spans appear as children of the experiment span automatically. - - ```python - import json - from opentelemetry import trace - - tracer = trace.get_tracer(__name__) - - def task(input_data: Dict[str, Any], config: Optional[Dict[str, Any]] = None) -> str: - question = input_data["question"] - - # OTel gen_ai span — automatically becomes a child of the experiment span - with tracer.start_as_current_span("my-llm-call") as span: - span.set_attribute("gen_ai.operation.name", "chat") - span.set_attribute("gen_ai.system", "openai") - span.set_attribute("gen_ai.request.model", "gpt-4o") - span.set_attribute("gen_ai.usage.input_tokens", 25) - span.set_attribute("gen_ai.usage.output_tokens", 8) - span.set_attribute( - "gen_ai.input.messages", - json.dumps([{"role": "user", "parts": [{"type": "text", "content": question}]}]), - ) - - result = call_my_llm(question) - - span.set_attribute( - "gen_ai.output.messages", - json.dumps([{"role": "assistant", "parts": [{"type": "text", "content": result}]}]), - ) - - return result - ``` - - To enable this, set the `DD_TRACE_OTEL_ENABLED` environment variable: - - ```shell - DD_TRACE_OTEL_ENABLED=1 python my_experiment.py - ``` - - -### 3. Define evaluators - - Evaluators measure how well your model or agent performs on each record. You can define evaluators using two approaches: - - - **Function-based**: Define a function that receives `input_data`, `output_data`, and `expected_output` as separate arguments. Best for one-off evaluators with straightforward logic. - - **Class-based**: Subclass `BaseEvaluator` for reusable evaluators with custom configuration. Class-based evaluators receive an `EvaluatorContext` object with full span context. - - For detailed information on building evaluators, including the full data model reference and best practices, see the [Evaluation Developer Guide][4]. - - Datadog supports the following evaluator return types: - - **Boolean**: returns true or false - - **score**: returns a numeric value (float) - - **categorical**: returns a labeled category (string) - - **json**: returns structured data (dict) - - You can also return: - - An `EvaluatorResult` to capture richer evaluation data, such as `reasoning`, `assessment` (`"pass"` or `"fail"`), `metadata`, and `tags`. - - A `MultiEvaluatorResult` to emit multiple named metrics from a single evaluator call. For details and examples, see the [Evaluation Developer Guide][4]. - - #### Function-based evaluators - - ```python - def exact_match(input_data: Dict[str, Any], output_data: str, expected_output: str) -> bool: - return output_data == expected_output - - def overlap(input_data: Dict[str, Any], output_data: str, expected_output: str) -> float: - expected_output_set = set(expected_output) - output_set = set(output_data) - - intersection = len(output_set.intersection(expected_output_set)) - union = len(output_set.union(expected_output_set)) - - return intersection / union - - def fake_llm_as_a_judge(input_data: Dict[str, Any], output_data: str, expected_output: str) -> EvaluatorResult: - fake_llm_call = "excellent" - return EvaluatorResult( - value=fake_llm_call, - reasoning="the model explains itself", - assessment="pass", # or fail - tags={"task": "judge_llm_call"}, - ) - - # Return multiple metrics from one evaluator call - from ddtrace.llmobs import MultiEvaluatorResult - - def multi_metric_evaluator(input_data, output_data, expected_output): - correct = output_data == expected_output - return MultiEvaluatorResult( - { - "correct": EvaluatorResult(value=correct, assessment="pass" if correct else "fail"), - "length": len(str(output_data)), - } - ) - # Emitted as: multi_metric_evaluator-correct, multi_metric_evaluator-length - ``` - - #### Class-based evaluators - - ```python - from ddtrace.llmobs import BaseEvaluator, EvaluatorContext, EvaluatorResult - - class SemanticSimilarityEvaluator(BaseEvaluator): - def __init__(self, threshold: float = 0.8): - super().__init__(name="semantic_similarity") - self.threshold = threshold - - def evaluate(self, context: EvaluatorContext) -> EvaluatorResult: - score = compute_similarity(context.output_data, context.expected_output) - return EvaluatorResult( - value=score, - reasoning=f"Similarity score: {score:.2f}", - assessment="pass" if score >= self.threshold else "fail", - ) - ``` - -### 4. (Optional) Define summary evaluators - - Summary evaluators run after all record-level evaluators have finished, and receive the aggregated results to compute dataset-level statistics like averages or pass rates. Like record-level evaluators, you can define summary evaluators as functions or classes. - - For the class-based approach using `BaseSummaryEvaluator`, see the [Evaluation Developer Guide][4]. - - #### Function-based summary evaluators - - ```python - def num_exact_matches(inputs, outputs, expected_outputs, evaluators_results): - return evaluators_results["exact_match"].count(True) - - ``` - - Summary evaluator functions can take a list of any non-null type as `inputs` (string, number, Boolean, object, array); `outputs` and `expected_outputs` can be lists of any type. `evaluators_results` is a dictionary of list of results from evaluators, keyed by the name of the evaluator function. For example, in the above code snippet the summary evaluator `num_exact_matches` uses the results (a list of Booleans) from the `exact_match` evaluator to provide a count of number of exact matches. - - #### Class-based summary evaluators - - ```python - from ddtrace.llmobs import BaseSummaryEvaluator, SummaryEvaluatorContext - - class AverageScoreEvaluator(BaseSummaryEvaluator): - def __init__(self, target_evaluator: str): - super().__init__(name="average_score") - self.target_evaluator = target_evaluator - - def evaluate(self, context: SummaryEvaluatorContext): - scores = context.evaluation_results.get(self.target_evaluator, []) - if not scores: - return None - return sum(scores) / len(scores) - ``` - - Datadog supports the following Summary Evaluator return types: - - **Boolean**: returns true or false - - **score**: returns a numeric value (float) - - **categorical**: returns a labeled category (string) - - **json**: returns structured data (dict) - -### 5. Create and run the experiment. - ```python - experiment = LLMObs.experiment( - name="capital-cities-test", - task=task, - dataset=dataset, - evaluators=[exact_match, overlap, fake_llm_as_a_judge], - summary_evaluators=[num_exact_matches], # optional - description="Testing capital cities knowledge", - config={ - "model_name": "gpt-4", - "version": "1.0" - }, - ) - - # Run the experiment - results = experiment.run() # Run on all dataset records - - # Process results - for result in results.get("rows", []): - print(f"Record {result['idx']}") - print(f"Input: {result['input']}") - print(f"Output: {result['output']}") - print(f"Score: {result['evaluations']['evaluator']['value']}") - if result['error']['message']: - print(f"Error: {result['error']['message']}") - ``` - - To increase the execution speed of the experiment, you can enable parallel processing: - ``` - results = experiment.run(jobs=4) - ``` - - To test your pipeline on a subset of the data, use: - ``` - results = experiment.run(sample_size=10) - ``` - - To stop the execution of the Experiment if an error occurs, use: - ``` - results = experiment.run(raise_errors=True) - ``` - -### 6. Review your experiment results in Datadog. - ``` - print(f"View experiment: {experiment.url}") - ``` - -Note: LLM Experiments traces are retained for 90 days. - -[1]: /llm_observability/improve/datasets -[2]: /llm_observability/instrument/custom_instrumentation?tab=decorators#trace-an-llm-application -[3]: /llm_observability/instrument/auto_instrumentation?tab=python -[4]: /llm_observability/investigate/evaluations/evaluation_developer_guide -[5]: /llm_observability/instrument/agent_observability_and_apm/ -[6]: /llm_observability/instrument/otel_instrumentation - -## Further reading - -{{< partial name="whats-next/whats-next.html" >}} diff --git a/hugo/content/en/llm_observability/improve/playground.md b/hugo/content/en/llm_observability/improve/playground.md index b13bf76c12c..0b3856e7ff1 100644 --- a/hugo/content/en/llm_observability/improve/playground.md +++ b/hugo/content/en/llm_observability/improve/playground.md @@ -10,7 +10,7 @@ further_reading: - link: "/llm_observability/improve/datasets" tag: "Documentation" text: "Datasets" - - link: "/llm_observability/improve/experiments/analyzing_results" + - link: "/llm_observability/guide/analyzing_experiment_results" tag: "Documentation" text: "Analyzing Experiment Results" --- diff --git a/hugo/content/en/llm_observability/instrument/otel_instrumentation.md b/hugo/content/en/llm_observability/instrument/otel_instrumentation.md index a413617e62d..27e49cb84b0 100644 --- a/hugo/content/en/llm_observability/instrument/otel_instrumentation.md +++ b/hugo/content/en/llm_observability/instrument/otel_instrumentation.md @@ -27,7 +27,7 @@ For information on using Prompt Tracking with OpenTelemetry spans, see [Prompt T ### Experiments -You can use OpenTelemetry spans inside [Agent Observability Experiments](/llm_observability/improve/experiments/setup#using-opentelemetry-spans-inside-experiments). By setting `DD_TRACE_OTEL_ENABLED=1`, OTel spans created inside an experiment task automatically appear as children of the experiment span. +You can use OpenTelemetry spans inside [Agent Observability Experiments](/llm_observability/guide/experiments#using-opentelemetry-spans-inside-experiments). By setting `DD_TRACE_OTEL_ENABLED=1`, OTel spans created inside an experiment task automatically appear as children of the experiment span. ### Multimodal support diff --git a/hugo/content/en/llm_observability/investigate/evaluations/external_evaluations/deepeval.md b/hugo/content/en/llm_observability/investigate/evaluations/external_evaluations/deepeval.md index d09b272d7ed..d145f7b594a 100644 --- a/hugo/content/en/llm_observability/investigate/evaluations/external_evaluations/deepeval.md +++ b/hugo/content/en/llm_observability/investigate/evaluations/external_evaluations/deepeval.md @@ -109,8 +109,8 @@ After you run an experiment with a DeepEval evaluation, you can view the DeepEva {{< partial name="whats-next/whats-next.html" >}} [1]: /llm_observability/improve/experiments -[2]: /llm_observability/improve/experiments/setup#create-an-experiment +[2]: /llm_observability/guide/experiments#create-an-experiment [3]: https://deepeval.com/docs/metrics-introduction -[4]: /llm_observability/improve/experiments/setup#create-a-dataset +[4]: /llm_observability/guide/experiments#create-a-dataset [5]: /llm_observability/improve/datasets [6]: https://github.com/DataDog/llm-observability/blob/main/experiments/eval-integrations/1-deepeval-demo.py diff --git a/hugo/content/en/llm_observability/investigate/evaluations/external_evaluations/pydantic.md b/hugo/content/en/llm_observability/investigate/evaluations/external_evaluations/pydantic.md index a1b08d1a65e..4e973dfd7d1 100644 --- a/hugo/content/en/llm_observability/investigate/evaluations/external_evaluations/pydantic.md +++ b/hugo/content/en/llm_observability/investigate/evaluations/external_evaluations/pydantic.md @@ -141,8 +141,8 @@ After you run an experiment with a Pydantic evaluation, you can view the Pydanti {{< partial name="whats-next/whats-next.html" >}} [1]: /llm_observability/improve/experiments -[2]: /llm_observability/improve/experiments/setup#create-an-experiment +[2]: /llm_observability/guide/experiments#create-an-experiment [3]: https://ai.pydantic.dev/evals/ -[4]: /llm_observability/improve/experiments/setup#create-a-dataset +[4]: /llm_observability/guide/experiments#create-a-dataset [5]: /llm_observability/improve/datasets [6]: https://github.com/DataDog/llm-observability/blob/main/experiments/eval-integrations/2-pydantic-demo.py