PR feedback from Harry

Author: Andrei Bratu

src/humanloop/client.py

@@ -1,44 +1,55 @@
-import logging
+from contextvars import ContextVar
 import os
 import typing
-from typing import Any, Callable, List, Optional, Sequence, Union
+from typing import Any, List, Optional, Sequence
 
 import httpx
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.trace import Tracer
 
+from humanloop.core.client_wrapper import SyncClientWrapper
+from humanloop.eval_utils.context import EVALUATION_CONTEXT_VARIABLE_NAME, EvaluationContext
 from humanloop.types.model_endpoints import ModelEndpoints
 from humanloop.types.model_providers import ModelProviders
 from humanloop.types.prompt_kernel_request_stop import PromptKernelRequestStop
 from humanloop.types.prompt_kernel_request_template import PromptKernelRequestTemplate
 from humanloop.types.response_format import ResponseFormat
 
 if typing.TYPE_CHECKING:
-    from . import ToolFunctionParams
+    from humanloop import ToolFunctionParams
 
 from humanloop.eval_utils import log_with_evaluation_context, run_eval
 from humanloop.eval_utils.types import Dataset, Evaluator, EvaluatorCheck, File
 
-from .base_client import AsyncBaseHumanloop, BaseHumanloop
-from .decorators.flow import flow as flow_decorator_factory
-from .decorators.prompt import prompt as prompt_decorator_factory
-from .decorators.tool import tool as tool_decorator_factory
-from .environment import HumanloopEnvironment
-from .evaluations.client import EvaluationsClient
-from .otel import instrument_provider
-from .otel.exporter import HumanloopSpanExporter
-from .otel.processor import HumanloopSpanProcessor
-from .prompt_utils import populate_template
-from .prompts.client import PromptsClient
+from humanloop.base_client import AsyncBaseHumanloop, BaseHumanloop
+from humanloop.decorators.flow import flow as flow_decorator_factory
+from humanloop.decorators.prompt import prompt as prompt_decorator_factory
+from humanloop.decorators.tool import tool as tool_decorator_factory
+from humanloop.environment import HumanloopEnvironment
+from humanloop.evaluations.client import EvaluationsClient
+from humanloop.otel import instrument_provider
+from humanloop.otel.exporter import HumanloopSpanExporter
+from humanloop.otel.processor import HumanloopSpanProcessor
+from humanloop.prompt_utils import populate_template
+from humanloop.prompts.client import PromptsClient
 
 
 class ExtendedEvalsClient(EvaluationsClient):
     client: BaseHumanloop
 
+    def __init__(
+        self,
+        *,
+        client_wrapper: SyncClientWrapper,
+        evaluation_context_variable: ContextVar[Optional[EvaluationContext]],
+    ):
+        super().__init__(client_wrapper=client_wrapper)
+        self._evaluation_context_variable = evaluation_context_variable
+
     def run(
         self,
-        file: Union[File, Callable],
+        file: File,
         name: Optional[str],
         dataset: Dataset,
         evaluators: Optional[Sequence[Evaluator]] = None,
@@ -64,6 +75,7 @@ def run(
             dataset=dataset,
             evaluators=evaluators,
             workers=workers,
+            evaluation_context_variable=self._evaluation_context_variable,
         )
 
 
@@ -111,18 +123,31 @@ def __init__(
             httpx_client=httpx_client,
         )
 
-        eval_client = ExtendedEvalsClient(client_wrapper=self._client_wrapper)
+        self.evaluation_context_variable: ContextVar[Optional[EvaluationContext]] = ContextVar(
+            EVALUATION_CONTEXT_VARIABLE_NAME
+        )
+
+        eval_client = ExtendedEvalsClient(
+            client_wrapper=self._client_wrapper,
+            evaluation_context_variable=self.evaluation_context_variable,
+        )
         eval_client.client = self
         self.evaluations = eval_client
         self.prompts = ExtendedPromptsClient(client_wrapper=self._client_wrapper)
 
         # Overload the .log method of the clients to be aware of Evaluation Context
         # TODO: Overload the log for Evaluators and Tools once run_id is added
         # to them.
-        self.prompts = log_with_evaluation_context(client=self.prompts)
+        self.prompts = log_with_evaluation_context(
+            client=self.prompts,
+            evaluation_context_variable=self.evaluation_context_variable,
+        )
         # self.evaluators = log_with_evaluation_context(client=self.evaluators)
         # self.tools = log_with_evaluation_context(client=self.tools)
-        self.flows = log_with_evaluation_context(client=self.flows)
+        self.flows = log_with_evaluation_context(
+            client=self.flows,
+            evaluation_context_variable=self.evaluation_context_variable,
+        )
 
         if opentelemetry_tracer_provider is not None:
             self._tracer_provider = opentelemetry_tracer_provider

src/humanloop/decorators/flow.py

@@ -1,7 +1,7 @@
 import logging
-import traceback
-import uuid
 from functools import wraps
+import os
+import sys
 from typing import Any, Callable, Mapping, Optional, Sequence
 
 from opentelemetry.sdk.trace import Span
@@ -11,8 +11,8 @@
 from humanloop.decorators.helpers import args_to_inputs
 from humanloop.eval_utils.types import File
 from humanloop.otel import TRACE_FLOW_CONTEXT, FlowContext
-from humanloop.otel.constants import HL_FILE_KEY, HL_FILE_TYPE_KEY, HL_LOG_KEY, HL_PATH_KEY
-from humanloop.otel.helpers import write_to_opentelemetry_span
+from humanloop.otel.constants import HUMANLOOP_FILE_KEY, HUMANLOOP_FILE_TYPE_KEY, HUMANLOOP_LOG_KEY, HUMANLOOP_PATH_KEY
+from humanloop.otel.helpers import generate_span_id, write_to_opentelemetry_span
 from humanloop.requests import FlowKernelRequestParams as FlowDict
 
 logger = logging.getLogger("humanloop.sdk")
@@ -31,7 +31,7 @@ def decorator(func: Callable):
         @wraps(func)
         def wrapper(*args: Sequence[Any], **kwargs: Mapping[str, Any]) -> Any:
             span: Span
-            with opentelemetry_tracer.start_as_current_span(str(uuid.uuid4())) as span:
+            with opentelemetry_tracer.start_as_current_span(generate_span_id()) as span:
                 span_id = span.get_span_context().span_id
                 if span.parent:
                     span_parent_id = span.parent.span_id
@@ -52,12 +52,12 @@ def wrapper(*args: Sequence[Any], **kwargs: Mapping[str, Any]) -> Any:
                         is_flow_log=True,
                     )
 
-                span.set_attribute(HL_PATH_KEY, path if path else func.__name__)
-                span.set_attribute(HL_FILE_TYPE_KEY, "flow")
+                span.set_attribute(HUMANLOOP_PATH_KEY, path if path else func.__name__)
+                span.set_attribute(HUMANLOOP_FILE_TYPE_KEY, "flow")
                 if attributes:
                     write_to_opentelemetry_span(
                         span=span,
-                        key=f"{HL_FILE_KEY}.flow.attributes",
+                        key=f"{HUMANLOOP_FILE_KEY}.flow.attributes",
                         value=attributes,  # type: ignore
                     )
 
@@ -68,7 +68,6 @@ def wrapper(*args: Sequence[Any], **kwargs: Mapping[str, Any]) -> Any:
                     output = func(*args, **kwargs)
                     error = None
                 except Exception as e:
-                    # print error, line of code, and function name
                     logger.error(f"Error calling {func.__name__}: {e}")
                     output = None
                     error = str(e)
@@ -87,18 +86,17 @@ def wrapper(*args: Sequence[Any], **kwargs: Mapping[str, Any]) -> Any:
                 if flow_log:
                     write_to_opentelemetry_span(
                         span=span,
-                        key=HL_LOG_KEY,
+                        key=HUMANLOOP_LOG_KEY,
                         value=flow_log,  # type: ignore
                     )
 
             # Return the output of the decorated function
             return output
 
-        func.file = File(  # type: ignore
+        wrapper.file = File(  # type: ignore
             path=path if path else func.__name__,
             type="flow",
             version=FlowDict(attributes=attributes),  # type: ignore
-            is_decorated=True,
             callable=wrapper,
         )
 

src/humanloop/decorators/prompt.py

@@ -1,4 +1,6 @@
 import logging
+import os
+import sys
 import typing
 import uuid
 from functools import wraps
@@ -7,13 +9,15 @@
 from opentelemetry.sdk.trace import Span
 from opentelemetry.trace import Tracer
 
+from humanloop.types.prompt_kernel_request import PromptKernelRequest
+
 if typing.TYPE_CHECKING:
     from humanloop import ToolFunctionParams
 from humanloop.decorators.helpers import args_to_inputs
 from humanloop.eval_utils import File
 from humanloop.otel import TRACE_FLOW_CONTEXT, FlowContext
-from humanloop.otel.constants import HL_FILE_KEY, HL_FILE_TYPE_KEY, HL_LOG_KEY, HL_PATH_KEY
-from humanloop.otel.helpers import write_to_opentelemetry_span
+from humanloop.otel.constants import HUMANLOOP_FILE_KEY, HUMANLOOP_FILE_TYPE_KEY, HUMANLOOP_LOG_KEY, HUMANLOOP_PATH_KEY
+from humanloop.otel.helpers import generate_span_id, write_to_opentelemetry_span
 from humanloop.types.model_endpoints import ModelEndpoints
 from humanloop.types.model_providers import ModelProviders
 from humanloop.types.prompt_kernel_request_stop import PromptKernelRequestStop
@@ -45,27 +49,6 @@ def prompt(
 ):
     def decorator(func: Callable):
         prompt_kernel = {}
-
-        if temperature is not None:
-            if not 0 <= temperature < 1:
-                raise ValueError(f"{func.__name__}: Temperature parameter must be between 0 and 1")
-            prompt_kernel["temperature"] = temperature
-
-        if top_p is not None:
-            if not 0 <= top_p <= 1:
-                raise ValueError(f"{func.__name__}: Top-p parameter must be between 0 and 1")
-            prompt_kernel["top_p"] = top_p
-
-        if presence_penalty is not None:
-            if not -2 <= presence_penalty <= 2:
-                raise ValueError(f"{func.__name__}: Presence penalty parameter must be between -2 and 2")
-            prompt_kernel["presence_penalty"] = presence_penalty
-
-        if frequency_penalty is not None:
-            if not -2 <= frequency_penalty <= 2:
-                raise ValueError(f"{func.__name__}: Frequency penalty parameter must be between -2 and 2")
-            prompt_kernel["frequency_penalty"] = frequency_penalty
-
         for attr_name, attr_value in {
             "model": model,
             "endpoint": endpoint,
@@ -76,16 +59,19 @@ def decorator(func: Callable):
             "other": other,
             "seed": seed,
             "response_format": response_format,
-            # {} -> None
             "attributes": attributes or None,
             "tools": tools,
+            "temperature": temperature,
+            "top_p": top_p,
+            "presence_penalty": presence_penalty,
+            "frequency_penalty": frequency_penalty,
         }.items():
             prompt_kernel[attr_name] = attr_value  # type: ignore
 
         @wraps(func)
         def wrapper(*args: Sequence[Any], **kwargs: Mapping[str, Any]) -> Any:
             span: Span
-            with opentelemetry_tracer.start_as_current_span(str(uuid.uuid4())) as span:
+            with opentelemetry_tracer.start_as_current_span(generate_span_id()) as span:
                 span_id = span.get_span_context().span_id
                 if span.parent:
                     span_parent_id = span.parent.span_id
@@ -97,13 +83,13 @@ def wrapper(*args: Sequence[Any], **kwargs: Mapping[str, Any]) -> Any:
                             is_flow_log=False,
                         )
 
-                span.set_attribute(HL_PATH_KEY, path if path else func.__name__)
-                span.set_attribute(HL_FILE_TYPE_KEY, "prompt")
+                span.set_attribute(HUMANLOOP_PATH_KEY, path if path else func.__name__)
+                span.set_attribute(HUMANLOOP_FILE_TYPE_KEY, "prompt")
 
                 if prompt_kernel:
                     write_to_opentelemetry_span(
                         span=span,
-                        key=f"{HL_FILE_KEY}.prompt",
+                        key=f"{HUMANLOOP_FILE_KEY}.prompt",
                         value=prompt_kernel,  # type: ignore
                     )
 
@@ -123,7 +109,7 @@ def wrapper(*args: Sequence[Any], **kwargs: Mapping[str, Any]) -> Any:
                 }
                 write_to_opentelemetry_span(
                     span=span,
-                    key=HL_LOG_KEY,
+                    key=HUMANLOOP_LOG_KEY,
                     value=prompt_log,
                 )
 
@@ -140,7 +126,6 @@ def wrapper(*args: Sequence[Any], **kwargs: Mapping[str, Any]) -> Any:
             path=path if path else func.__name__,
             type="prompt",
             version={**prompt_kernel_file},  # type: ignore
-            is_decorated=True,
             callable=wrapper,
         )