eval run quality of life changes

Author: Andrei Bratu

src/humanloop/evals/run.py

@@ -152,7 +152,7 @@ def run_eval(
     """
     evaluators_worker_pool = ThreadPoolExecutor(max_workers=workers)
 
-    hl_file, function_ = _get_hl_file(client=client, file=file)
+    hl_file, function_ = _get_hl_file(client=client, file_config=file)
     type_ = hl_file.type
     try:
         hl_dataset = _upsert_dataset(dataset=dataset, client=client)
@@ -415,48 +415,52 @@ def _safe_get_default_file_version(client: "BaseHumanloop", file_config: FileEva
         raise HumanloopRuntimeError("You must provide a path or id in your `file`.")
 
     if path is not None:
-        try:
-            hl_file = client.files.retrieve_by_path(path=path)
-            if hl_file.type != type:
-                raise HumanloopRuntimeError(
-                    f"File in Humanloop workspace at {path} is not of type {type}, but {hl_file.type}."
-                )
-            return hl_file
-        except ApiError as e:
-            if e.status_code == 404:
-                raise HumanloopRuntimeError(f"File with path {path} not found. Please check your path and try again.")
-            raise e
+        hl_file = client.files.retrieve_by_path(path=path)
+        if hl_file.type != type:
+            raise HumanloopRuntimeError(
+                f"File in Humanloop workspace at {path} is not of type {type}, but {hl_file.type}."
+            )
+        return hl_file
     else:
-        subclient = _get_subclient(client, file_config)
-        try:
-            return subclient.get(id=file_id)
-        except ApiError as e:
-            if e.status_code == 404:
-                raise HumanloopRuntimeError(f"File with id {file_id} not found. Please check your id and try again.")
-            raise e
+        subclient = _get_subclient(client=client, file_config=file_config)
+        return subclient.get(id=file_id)
 
 
 def _resolve_file(client: "BaseHumanloop", file_config: FileEvalConfig) -> tuple[EvaluatedFile, Optional[Callable]]:
     """Resolve the File to be evaluated. Will return a FileResponse and an optional callable.
 
     If the callable is null, the File will be evaluated on Humanloop. Otherwise, the File will be evaluated locally.
     """
-    if file_config.get("version") is not None and file_config.get("path") is None:
-        raise HumanloopRuntimeError(
-            "You are trying to create a new version of the File by passing the `version` argument. You must also pass the `path` argument."
-        )
     hl_file = _safe_get_default_file_version(client=client, file_config=file_config)
-    if "version" in file_config:
-        # User wants to upsert a version
-        return (
-            _upsert_file(file_config=file_config, client=client),
-            _get_file_callable(file_config),
-        )
+    file_id = file_config.get("id")
+    path = file_config.get("path")
     version_id = file_config.get("version_id")
     environment = file_config.get("environment")
+    callable = _get_file_callable(file_config=file_config)
+    version = file_config.get("version")
+
+    if version is not None and path is None and file_id:
+        raise HumanloopRuntimeError(
+            "You are trying to create a new version of the File by passing the `version` argument. You must pass either the `file.path` or `file.id` argument."
+        )
+
+    if version:
+        # User wants to upsert a version
+        return (_upsert_file(file_config=file_config, client=client), callable)
+
     if version_id is None and environment is None:
         # Return default version of the File
-        return hl_file, None
+        return hl_file, callable
+
+    if callable:
+        raise HumanloopRuntimeError(
+            "You cannot request local evaluation while requesting a specific File version by version ID or environment"
+        )
+
+    if file_id is None and (version_id or environment):
+        raise HumanloopRuntimeError(
+            "You must provide the `file.id` when addressing a file by version ID or environment"
+        )
     # Use version_id or environment to retrieve specific version of the File
     subclient = _get_subclient(client=client, file_config=file_config)
     # Let backend handle case where both or none of version_id and environment are provided
@@ -466,21 +470,21 @@ def _resolve_file(client: "BaseHumanloop", file_config: FileEvalConfig) -> tuple
     ), None
 
 
-def _get_hl_file(client: "BaseHumanloop", file: FileEvalConfig) -> tuple[EvaluatedFile, Optional[Callable]]:
+def _get_hl_file(client: "BaseHumanloop", file_config: FileEvalConfig) -> tuple[EvaluatedFile, Optional[Callable]]:
     """Check if the config object is valid, and resolve the File to be evaluated
 
     The callable will be null if the evaluation will happen on Humanloop runtime.
     Otherwise, the evaluation will happen locally.
     """
-    file_ = _file_or_file_inside_hl_decorator(file)
+    file_ = _file_or_file_inside_hl_decorator(file_config)
     file_ = _check_file_type(file_)
 
     return _resolve_file(client=client, file_config=file_)
 
 
-def _callable_is_hl_utility(file: FileEvalConfig) -> bool:
+def _callable_is_hl_utility(file_config: FileEvalConfig) -> bool:
     """Check if a File is a decorated function."""
-    return hasattr(file.get("callable", {}), "file")
+    return hasattr(file_config.get("callable", {}), "file")
 
 
 def _wait_for_evaluation_to_complete(
@@ -557,37 +561,37 @@ def _get_checks(
     return checks
 
 
-def _file_or_file_inside_hl_decorator(file: FileEvalConfig) -> FileEvalConfig:
-    if _callable_is_hl_utility(file):
-        inner_file: FileEvalConfig = file["callable"].file  # type: ignore [misc, attr-defined]
-        function_ = file["callable"]
-        type_ = file["type"]
+def _file_or_file_inside_hl_decorator(file_config: FileEvalConfig) -> FileEvalConfig:
+    if _callable_is_hl_utility(file_config):
+        inner_file: FileEvalConfig = file_config["callable"].file  # type: ignore [misc, attr-defined]
+        function_ = file_config["callable"]
+        type_ = file_config["type"]
         decorator_type = function_.file["type"]  # type: ignore [attr-defined, union-attr]
         if decorator_type != type_:
             raise HumanloopRuntimeError(
                 "The type of the decorated function does not match the type of the file. Expected `%s`, got `%s`."
                 % (type_.capitalize(), decorator_type.capitalize())
             )
-        if "path" in file and inner_file["path"] != file["path"]:
+        if "path" in file_config and inner_file["path"] != file_config["path"]:
             raise HumanloopRuntimeError(
                 "`path` attribute specified in the `file` does not match the path of the decorated function. "
-                f"Expected `{inner_file['path']}`, got `{file['path']}`."
+                f"Expected `{inner_file['path']}`, got `{file_config['path']}`."
             )
-        if "id" in file:
+        if "id" in file_config:
             raise HumanloopRuntimeError(
                 "Do not specify an `id` attribute in `file` argument when using a decorated function."
             )
-        if "version" in file:
+        if "version" in file_config:
             if inner_file["type"] != "prompt":
                 raise HumanloopRuntimeError(
                     f"Do not specify a `version` attribute in `file` argument when using a {inner_file['type'].capitalize()} decorated function."
                 )
-        if "type" in file and inner_file["type"] != file["type"]:
+        if "type" in file_config and inner_file["type"] != file_config["type"]:
             raise HumanloopRuntimeError(
                 "Attribute `type` of `file` argument does not match the file type of the decorated function. "
-                f"Expected `{inner_file['type']}`, got `{file['type']}`."
+                f"Expected `{inner_file['type']}`, got `{file_config['type']}`."
             )
-        if "id" in file:
+        if "id" in file_config:
             raise HumanloopRuntimeError(
                 "Do not specify an `id` attribute in `file` argument when using a decorated function."
             )
@@ -602,9 +606,9 @@ def _file_or_file_inside_hl_decorator(file: FileEvalConfig) -> FileEvalConfig:
                 f"{RESET}"
             )
             # TODO: document this
-            file_["version"] = file["version"]
+            file_["version"] = file_config["version"]
     else:
-        file_ = copy.deepcopy(file)
+        file_ = copy.deepcopy(file_config)
 
     # Raise error if neither path nor id is provided
     if not file_.get("path") and not file_.get("id"):
@@ -807,7 +811,7 @@ def _call_function(
         try:
             output = json.dumps(output)
         except Exception:
-            # throw error if it fails to serialize
+            # throw error if output fails to serialize
             raise ValueError(f"Your {type}'s `callable` must return a string or a JSON serializable object.")
     return output