refactor: use callable function as fixture instead of passing in the client directly

Author: AlePouroullis

src/humanloop/overload.py

@@ -206,6 +206,7 @@ def _overload(self, function_name: str, **kwargs) -> PromptCallResponse:
             normalized_path = sync_client._normalize_path(kwargs["path"])
 
             if use_remote:
+                # Don't allow user to specify path + version_id/environment because it's ambiguous
                 raise HumanloopRuntimeError(
                     f"Cannot use local file for `{normalized_path}` as version_id or environment was specified. "
                     "Please either remove version_id/environment to use local files, or set use_local_files=False to use remote files."

tests/custom/conftest.py

@@ -1,10 +1,10 @@
 from typing import Generator
-import typing
 import os
 from dotenv import load_dotenv
 from unittest.mock import MagicMock
 
 import pytest
+from humanloop.client import Humanloop
 from humanloop.otel.exporter import HumanloopSpanExporter
 from humanloop.otel.processor import HumanloopSpanProcessor
 from openai.types.chat.chat_completion_message_param import ChatCompletionMessageParam
@@ -21,9 +21,6 @@
 from opentelemetry.trace import Tracer
 from tests.custom.types import GetHumanloopClientFn
 
-if typing.TYPE_CHECKING:
-    from humanloop.client import Humanloop
-
 
 @pytest.fixture(scope="function")
 def opentelemetry_test_provider() -> TracerProvider:
@@ -82,7 +79,6 @@ def opentelemetry_test_configuration(
         instrumentor.uninstrument()
 
 
-
 @pytest.fixture(scope="session")
 def get_humanloop_client() -> GetHumanloopClientFn:
     load_dotenv()
@@ -92,6 +88,7 @@ def get_humanloop_client() -> GetHumanloopClientFn:
     def _get_humanloop_test_client(use_local_files: bool = False) -> Humanloop:
         return Humanloop(
             api_key=os.getenv("HUMANLOOP_API_KEY"),
+            base_url="http://localhost:80/v5",
             use_local_files=use_local_files,
         )
 

tests/custom/integration/conftest.py

@@ -1,4 +1,3 @@
-from typing import Callable
 from contextlib import contextmanager, redirect_stdout
 from dataclasses import dataclass
 import os
@@ -10,6 +9,7 @@
 import pytest
 import dotenv
 from humanloop.client import Humanloop
+from tests.custom.types import GetHumanloopClientFn
 
 
 @dataclass
@@ -18,7 +18,6 @@ class TestIdentifiers:
     file_path: str
 
 
-
 @pytest.fixture()
 def capture_stdout() -> ContextManager[TextIO]:
     @contextmanager
@@ -39,31 +38,32 @@ def openai_key() -> str:
 
 
 @pytest.fixture(scope="function")
-def sdk_test_dir(humanloop_test_client: Humanloop) -> Generator[str, None, None]:
+def sdk_test_dir(get_humanloop_client: GetHumanloopClientFn) -> Generator[str, None, None]:
+    humanloop_client = get_humanloop_client()
     def cleanup_directory(directory_id: str):
-        directory_response = humanloop_test_client.directories.get(id=directory_id)
+        directory_response = humanloop_client.directories.get(id=directory_id)
         for subdirectory in directory_response.subdirectories:
             cleanup_directory(subdirectory.id)
         for file in directory_response.files:
             match file.type:
                 case "prompt":
-                    humanloop_test_client.prompts.delete(id=file.id)
+                    humanloop_client.prompts.delete(id=file.id)
                 case "agent":
-                    humanloop_test_client.agents.delete(id=file.id)
+                    humanloop_client.agents.delete(id=file.id)
                 case "dataset":
-                    humanloop_test_client.datasets.delete(id=file.id)
+                    humanloop_client.datasets.delete(id=file.id)
                 case "evaluator":
-                    humanloop_test_client.evaluators.delete(id=file.id)
+                    humanloop_client.evaluators.delete(id=file.id)
                 case "flow":
-                    humanloop_test_client.flows.delete(id=file.id)
+                    humanloop_client.flows.delete(id=file.id)
                 case _:
                     raise ValueError(f"Unknown file type: {file.type}")
-        humanloop_test_client.directories.delete(id=directory_response.id)
+        humanloop_client.directories.delete(id=directory_response.id)
 
     path = f"SDK_INTEGRATION_TEST_{uuid.uuid4()}"
     response = None
     try:
-        response = humanloop_test_client.directories.create(path=path)
+        response = humanloop_client.directories.create(path=path)
         yield response.path
     except Exception as e:
         pytest.fail(f"Failed to create directory {path}: {e}")
@@ -93,10 +93,11 @@ def test_prompt_config() -> dict[str, Any]:
 
 
 @pytest.fixture(scope="function")
-def eval_dataset(humanloop_test_client: Humanloop, sdk_test_dir: str) -> Generator[TestIdentifiers, None, None]:
+def eval_dataset(get_humanloop_client: GetHumanloopClientFn, sdk_test_dir: str) -> Generator[TestIdentifiers, None, None]:
+    humanloop_client = get_humanloop_client()
     dataset_path = f"{sdk_test_dir}/eval_dataset"
     try:
-        response = humanloop_test_client.datasets.upsert(
+        response = humanloop_client.datasets.upsert(
             path=dataset_path,
             datapoints=[
                 {
@@ -117,34 +118,36 @@ def eval_dataset(humanloop_test_client: Humanloop, sdk_test_dir: str) -> Generat
             ],
         )
         yield TestIdentifiers(file_id=response.id, file_path=response.path)
-        humanloop_test_client.datasets.delete(id=response.id)
+        humanloop_client.datasets.delete(id=response.id)
     except Exception as e:
         pytest.fail(f"Failed to create dataset {dataset_path}: {e}")
 
 
 @pytest.fixture(scope="function")
 def eval_prompt(
-    humanloop_test_client: Humanloop, sdk_test_dir: str, openai_key: str, test_prompt_config: dict[str, Any]
+    get_humanloop_client: GetHumanloopClientFn, sdk_test_dir: str, openai_key: str, test_prompt_config: dict[str, Any]
 ) -> Generator[TestIdentifiers, None, None]:
+    humanloop_client = get_humanloop_client()
     prompt_path = f"{sdk_test_dir}/eval_prompt"
     try:
-        response = humanloop_test_client.prompts.upsert(
+        response = humanloop_client.prompts.upsert(
             path=prompt_path,
             **test_prompt_config,
         )
         yield TestIdentifiers(file_id=response.id, file_path=response.path)
-        humanloop_test_client.prompts.delete(id=response.id)
+        humanloop_client.prompts.delete(id=response.id)
     except Exception as e:
         pytest.fail(f"Failed to create prompt {prompt_path}: {e}")
 
 
 @pytest.fixture(scope="function")
 def output_not_null_evaluator(
-    humanloop_test_client: Humanloop, sdk_test_dir: str
+    get_humanloop_client: GetHumanloopClientFn, sdk_test_dir: str
 ) -> Generator[TestIdentifiers, None, None]:
+    humanloop_client = get_humanloop_client()
     evaluator_path = f"{sdk_test_dir}/output_not_null_evaluator"
     try:
-        response = humanloop_test_client.evaluators.upsert(
+        response = humanloop_client.evaluators.upsert(
             path=evaluator_path,
             spec={
                 "arguments_type": "target_required",
@@ -157,14 +160,15 @@ def output_not_null(log: dict) -> bool:
             },
         )
         yield TestIdentifiers(file_id=response.id, file_path=response.path)
-        humanloop_test_client.evaluators.delete(id=response.id)
+        humanloop_client.evaluators.delete(id=response.id)
     except Exception as e:
         pytest.fail(f"Failed to create evaluator {evaluator_path}: {e}")
 
 
 @pytest.fixture(scope="function")
-def id_for_staging_environment(humanloop_test_client: Humanloop, eval_prompt: TestIdentifiers) -> str:
-    response = humanloop_test_client.prompts.list_environments(id=eval_prompt.file_id)
+def id_for_staging_environment(get_humanloop_client: GetHumanloopClientFn, eval_prompt: TestIdentifiers) -> str:
+    humanloop_client = get_humanloop_client()
+    response = humanloop_client.prompts.list_environments(id=eval_prompt.file_id)
     for environment in response:
         if environment.name == "staging":
             return environment.id

tests/custom/integration/test_decorators.py

@@ -2,26 +2,27 @@
 from typing import Any
 
 from openai import OpenAI
-from humanloop.client import Humanloop
+from tests.custom.integration.conftest import GetHumanloopClientFn
 
 
 def test_prompt_decorator(
-    humanloop_test_client: Humanloop,
+    get_humanloop_client: GetHumanloopClientFn,
     sdk_test_dir: str,
     test_prompt_config: dict[str, Any],
     openai_key: str,
 ):
     try:
+        humanloop_client = get_humanloop_client()
         prompt_path = f"{sdk_test_dir}/test_prompt"
-        prompt_response = humanloop_test_client.prompts.upsert(
+        prompt_response = humanloop_client.prompts.upsert(
             path=prompt_path,
             **test_prompt_config,
         )
 
-        prompt_versions_response = humanloop_test_client.prompts.list_versions(id=prompt_response.id)
+        prompt_versions_response = humanloop_client.prompts.list_versions(id=prompt_response.id)
         assert len(prompt_versions_response.records) == 1
 
-        @humanloop_test_client.prompt(path=prompt_path)
+        @humanloop_client.prompt(path=prompt_path)
         def my_prompt(question: str) -> str:
             openai_client = OpenAI(api_key=openai_key)
 
@@ -36,26 +37,27 @@ def my_prompt(question: str) -> str:
         assert "paris" in my_prompt("What is the capital of the France?").lower()
 
         time.sleep(5)
-        prompt_versions_response = humanloop_test_client.prompts.list_versions(id=prompt_response.id)
+        prompt_versions_response = humanloop_client.prompts.list_versions(id=prompt_response.id)
         assert len(prompt_versions_response.records) == 2
 
-        logs_response = humanloop_test_client.logs.list(file_id=prompt_response.id, page=1, size=50)
+        logs_response = humanloop_client.logs.list(file_id=prompt_response.id, page=1, size=50)
 
         assert logs_response.items is not None and len(logs_response.items) == 1
     finally:
-        humanloop_test_client.prompts.delete(id=prompt_response.id)
+        humanloop_client.prompts.delete(id=prompt_response.id)
 
 
 def test_call_prompt_in_flow_decorator(
-    humanloop_test_client: Humanloop,
+    get_humanloop_client: GetHumanloopClientFn,
     sdk_test_dir: str,
     openai_key: str,
 ):
     try:
+        humanloop_client = get_humanloop_client()
 
-        @humanloop_test_client.flow(path=f"{sdk_test_dir}/test_flow")
+        @humanloop_client.flow(path=f"{sdk_test_dir}/test_flow")
         def my_flow(question: str) -> str:
-            response = humanloop_test_client.prompts.call(
+            response = humanloop_client.prompts.call(
                 path=f"{sdk_test_dir}/test_prompt",
                 prompt={
                     "provider": "openai",
@@ -71,83 +73,81 @@ def my_flow(question: str) -> str:
 
         assert "paris" in my_flow("What is the capital of the France?").lower()
         time.sleep(5)
-        prompt_response = humanloop_test_client.files.retrieve_by_path(path=f"{sdk_test_dir}/test_prompt")
+        prompt_response = humanloop_client.files.retrieve_by_path(path=f"{sdk_test_dir}/test_prompt")
         assert prompt_response is not None
-        prompt_logs_response = humanloop_test_client.logs.list(file_id=prompt_response.id, page=1, size=50)
+        prompt_logs_response = humanloop_client.logs.list(file_id=prompt_response.id, page=1, size=50)
         assert prompt_logs_response.items is not None and len(prompt_logs_response.items) == 1
         prompt_log = prompt_logs_response.items[0]
 
-        flow_response = humanloop_test_client.files.retrieve_by_path(path=f"{sdk_test_dir}/test_flow")
+        flow_response = humanloop_client.files.retrieve_by_path(path=f"{sdk_test_dir}/test_flow")
         assert flow_response is not None
-        flow_logs_response = humanloop_test_client.logs.list(file_id=flow_response.id, page=1, size=50)
+        flow_logs_response = humanloop_client.logs.list(file_id=flow_response.id, page=1, size=50)
         assert flow_logs_response.items is not None and len(flow_logs_response.items) == 1
         flow_log = flow_logs_response.items[0]
         assert prompt_log.trace_parent_id == flow_log.id
     finally:
-        flow_response = humanloop_test_client.files.retrieve_by_path(path=f"{sdk_test_dir}/test_flow")
+        flow_response = humanloop_client.files.retrieve_by_path(path=f"{sdk_test_dir}/test_flow")
         if flow_response is not None:
-            humanloop_test_client.flows.delete(id=flow_response.id)
-        prompt_response = humanloop_test_client.files.retrieve_by_path(path=f"{sdk_test_dir}/test_prompt")
+            humanloop_client.flows.delete(id=flow_response.id)
+        prompt_response = humanloop_client.files.retrieve_by_path(path=f"{sdk_test_dir}/test_prompt")
         if prompt_response is not None:
-            humanloop_test_client.prompts.delete(id=prompt_response.id)
+            humanloop_client.prompts.delete(id=prompt_response.id)
 
 
 def test_flow_decorator_logs_exceptions(
-    humanloop_test_client: Humanloop,
+    get_humanloop_client: GetHumanloopClientFn,
     sdk_test_dir: str,
 ):
     try:
+        humanloop_client = get_humanloop_client()
 
-        @humanloop_test_client.flow(path=f"{sdk_test_dir}/test_flow_log_error")
+        @humanloop_client.flow(path=f"{sdk_test_dir}/test_flow_log_error")
         def my_flow(question: str) -> str:
             raise ValueError("This is a test exception")
 
         my_flow("test")
 
         time.sleep(5)
 
-        flow_response = humanloop_test_client.files.retrieve_by_path(path=f"{sdk_test_dir}/test_flow_log_error")
+        flow_response = humanloop_client.files.retrieve_by_path(path=f"{sdk_test_dir}/test_flow_log_error")
         assert flow_response is not None
-        flow_logs_response = humanloop_test_client.logs.list(file_id=flow_response.id, page=1, size=50)
+        flow_logs_response = humanloop_client.logs.list(file_id=flow_response.id, page=1, size=50)
         assert flow_logs_response.items is not None and len(flow_logs_response.items) == 1
         flow_log = flow_logs_response.items[0]
         assert flow_log.error is not None
         assert flow_log.output is None
 
     finally:
-        flow_response = humanloop_test_client.files.retrieve_by_path(path=f"{sdk_test_dir}/test_flow_log_error")
+        flow_response = humanloop_client.files.retrieve_by_path(path=f"{sdk_test_dir}/test_flow_log_error")
         if flow_response is not None:
-            humanloop_test_client.flows.delete(id=flow_response.id)
+            humanloop_client.flows.delete(id=flow_response.id)
 
 
 def test_flow_decorator_populates_output_message(
-    humanloop_test_client: Humanloop,
+    get_humanloop_client: GetHumanloopClientFn,
     sdk_test_dir: str,
 ):
     try:
+        humanloop_client = get_humanloop_client()
 
-        @humanloop_test_client.flow(path=f"{sdk_test_dir}/test_flow_log_output_message")
+        @humanloop_client.flow(path=f"{sdk_test_dir}/test_flow_log_output_message")
         def my_flow(question: str) -> dict[str, Any]:
             return {"role": "user", "content": question}
 
         assert "france" in my_flow("What is the capital of the France?")["content"].lower()
 
         time.sleep(5)
 
-        flow_response = humanloop_test_client.files.retrieve_by_path(
-            path=f"{sdk_test_dir}/test_flow_log_output_message"
-        )
+        flow_response = humanloop_client.files.retrieve_by_path(path=f"{sdk_test_dir}/test_flow_log_output_message")
         assert flow_response is not None
-        flow_logs_response = humanloop_test_client.logs.list(file_id=flow_response.id, page=1, size=50)
+        flow_logs_response = humanloop_client.logs.list(file_id=flow_response.id, page=1, size=50)
         assert flow_logs_response.items is not None and len(flow_logs_response.items) == 1
         flow_log = flow_logs_response.items[0]
         assert flow_log.output_message is not None
         assert flow_log.output is None
         assert flow_log.error is None
 
     finally:
-        flow_response = humanloop_test_client.files.retrieve_by_path(
-            path=f"{sdk_test_dir}/test_flow_log_output_message"
-        )
+        flow_response = humanloop_client.files.retrieve_by_path(path=f"{sdk_test_dir}/test_flow_log_output_message")
         if flow_response is not None:
-            humanloop_test_client.flows.delete(id=flow_response.id)
+            humanloop_client.flows.delete(id=flow_response.id)