diff --git a/docs/api/benchmarks.rst b/docs/api/benchmarks.rst
new file mode 100644
index 000000000..a388d2383
--- /dev/null
+++ b/docs/api/benchmarks.rst
@@ -0,0 +1,21 @@
+Benchmarks
+==========
+
+:mod:`pyhealth.benchmarks` provides the base abstractions for benchmarks. A
+benchmark defines and executes evaluations. It knows nothing about runners,
+storage or leaderboards.
+
+An :class:`~pyhealth.benchmarks.Evaluation` is one unit of work, identified by
+a hash of its parameters. A :class:`~pyhealth.benchmarks.Benchmark` runs a
+batch of evaluations and returns one :class:`~pyhealth.benchmarks.Result` per
+evaluation.
+
+.. autoclass:: pyhealth.benchmarks.Evaluation
+    :members:
+
+.. autoclass:: pyhealth.benchmarks.Result
+    :members:
+
+.. autoclass:: pyhealth.benchmarks.Benchmark
+    :members:
+    :private-members: _evaluate
diff --git a/docs/index.rst b/docs/index.rst
index cddf07488..9c7b9e9e4 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -238,6 +238,7 @@ Quick Navigation
    api/interpret
    api/trainer
    api/autoimprove
+   api/benchmarks
    api/tokenizer
    api/metrics
    api/medcode
diff --git a/pyhealth/benchmarks/__init__.py b/pyhealth/benchmarks/__init__.py
new file mode 100644
index 000000000..d8703c5bf
--- /dev/null
+++ b/pyhealth/benchmarks/__init__.py
@@ -0,0 +1,5 @@
+"""Base abstractions for defining and executing PyHealth benchmarks."""
+
+from .base import Benchmark, Evaluation, Result
+
+__all__ = ["Benchmark", "Evaluation", "Result"]
diff --git a/pyhealth/benchmarks/_serialization.py b/pyhealth/benchmarks/_serialization.py
new file mode 100644
index 000000000..489ae2144
--- /dev/null
+++ b/pyhealth/benchmarks/_serialization.py
@@ -0,0 +1,178 @@
+"""Canonical JSON serialization of Evaluation params.
+
+Classes and functions are encoded by import path as
+``{"__type__": "<module>.<qualname>"}`` and resolved back on decode.
+"""
+
+import importlib
+import json
+from collections.abc import Mapping
+from types import BuiltinFunctionType, FunctionType
+from typing import Any
+
+
+def _encode(obj: Any) -> Any:
+    """Encodes a non-JSON value for ``json.dumps``.
+
+    Module-level classes and functions become
+    ``{"__type__": "<module>.<qualname>"}``. Mappings that are not dicts
+    become dicts.
+
+    The path must resolve back to ``obj`` in a fresh process, so that
+    ``from_dict`` can rebuild it. That rules out classes and functions
+    defined in ``__main__`` (a script run directly or a notebook), classes
+    made with ``type()`` under a name they aren't bound to, objects whose
+    name was later rebound to something else, and built-in types whose
+    ``__module__`` and ``__qualname__`` don't name them.
+
+    Args:
+        obj: The value ``json.dumps`` could not serialize.
+
+    Returns:
+        A JSON-compatible stand-in for ``obj``.
+
+    Raises:
+        TypeError: If ``obj`` is not a mapping or a module-level class or
+            function whose path resolves back to it.
+    """
+    if _is_type_ref(obj):
+        qualname = obj.__qualname__
+        module = obj.__module__
+        if not module or "<" in qualname:
+            raise TypeError(
+                f"{obj!r} is not a module-level class or function and cannot "
+                "be serialized by import path."
+            )
+        if module == "__main__":
+            raise TypeError(
+                f"{qualname} is defined in __main__ (a script run directly or "
+                "a notebook), so it can't be imported from another process. "
+                "Move it into an importable module and import it from there."
+            )
+        path = f"{module}.{qualname}"
+        try:
+            resolved = _resolve(path)
+        except ValueError:
+            resolved = None
+        if resolved is not obj:
+            raise TypeError(
+                f"{obj!r} is serialized as {path!r}, but that path does not "
+                "resolve back to it. Bind it to a module-level name matching "
+                "its __qualname__."
+            )
+        return {"__type__": path}
+    if isinstance(obj, Mapping):
+        return dict(obj)
+    raise TypeError(
+        f"Object of type {type(obj).__name__} is not allowed in Evaluation "
+        "params. Use JSON types or module-level classes and functions."
+    )
+
+
+def _is_type_ref(obj: Any) -> bool:
+    """Returns whether ``obj`` is a class or function."""
+    return isinstance(obj, (type, FunctionType, BuiltinFunctionType))
+
+
+def _check_keys(obj: Any) -> None:
+    """Checks every mapping key anywhere in ``obj``.
+
+    Keys must be strings, because ``json.dumps`` silently turns ``1`` into
+    ``"1"``, which would give ``{1: x}`` and ``{"1": x}`` the same id. The
+    key ``"__type__"`` is reserved for class and function references, so
+    allowing it in user data would make ``to_dict`` output ambiguous.
+
+    Raises:
+        TypeError: If a key is not a string.
+        ValueError: If a key is ``"__type__"``.
+    """
+    if isinstance(obj, Mapping):
+        for key in obj:
+            if not isinstance(key, str):
+                raise TypeError(
+                    f"Evaluation params keys must be strings, got {key!r} of "
+                    f"type {type(key).__name__}."
+                )
+        if "__type__" in obj:
+            raise ValueError(
+                '"__type__" is reserved for class and function references and '
+                "cannot be used as a key in Evaluation params."
+            )
+        for value in obj.values():
+            _check_keys(value)
+    elif isinstance(obj, (list, tuple)):
+        for value in obj:
+            _check_keys(value)
+
+
+def _resolve(path: str) -> Any:
+    """Resolves a dotted path to the class or function it names.
+
+    Imports the longest importable module prefix of ``path``, then resolves
+    the remaining parts with ``getattr``, so nested classes such as
+    ``pkg.mod.Outer.Inner`` work.
+
+    Args:
+        path: Dotted import path, e.g. ``"json.decoder.JSONDecoder"``.
+
+    Returns:
+        The class or function at ``path``.
+
+    Raises:
+        ValueError: If ``path`` can't be resolved, or names something that is
+            not a class or function.
+    """
+    parts = path.split(".")
+    for i in range(len(parts), 0, -1):
+        prefix = ".".join(parts[:i])
+        try:
+            obj = importlib.import_module(prefix)
+        except ModuleNotFoundError as exc:
+            # Only skip prefixes that are themselves missing. A missing
+            # dependency inside an existing module is a real error.
+            if exc.name is None or not (
+                prefix == exc.name or prefix.startswith(exc.name + ".")
+            ):
+                raise
+            continue
+        except ValueError:  # Empty or malformed module name.
+            break
+        try:
+            for attr in parts[i:]:
+                obj = getattr(obj, attr)
+        except AttributeError:
+            break
+        if not _is_type_ref(obj):
+            raise ValueError(f"{path!r} does not name a class or function.")
+        return obj
+    raise ValueError(f"Cannot resolve {path!r} to a class or function.")
+
+
+def _decode(obj: Any) -> Any:
+    """Rebuilds params from ``to_dict`` output, resolving ``__type__`` refs."""
+    if isinstance(obj, Mapping):
+        if "__type__" in obj:
+            path = obj["__type__"]
+            if len(obj) != 1:
+                raise ValueError(
+                    f"Type reference {path!r} must be the only key in its "
+                    f"mapping, got keys {sorted(obj)}."
+                )
+            if not isinstance(path, str):
+                raise ValueError(f"Type reference {path!r} must be a string.")
+            return _resolve(path)
+        return {key: _decode(value) for key, value in obj.items()}
+    if isinstance(obj, (list, tuple)):
+        return [_decode(value) for value in obj]
+    return obj
+
+
+def _dumps(params: Mapping[str, Any]) -> str:
+    """Serializes params canonically: sorted keys, no whitespace, no NaN."""
+    return json.dumps(
+        params,
+        sort_keys=True,
+        separators=(",", ":"),
+        allow_nan=False,
+        default=_encode,
+    )
diff --git a/pyhealth/benchmarks/base.py b/pyhealth/benchmarks/base.py
new file mode 100644
index 000000000..a6903698e
--- /dev/null
+++ b/pyhealth/benchmarks/base.py
@@ -0,0 +1,285 @@
+import hashlib
+import json
+from abc import ABC, abstractmethod
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal, final
+
+from ._serialization import _check_keys, _decode, _dumps
+
+
+@dataclass(frozen=True, eq=False)
+class Evaluation:
+    """One unit of work a benchmark runs, identified by its params.
+
+    ``params`` may hold JSON types (``None``, ``bool``, ``int``, finite
+    ``float``, ``str``, lists/tuples and str-keyed mappings) plus
+    module-level classes and functions. Classes and functions are serialized
+    by import path as ``{"__type__": "<module>.<qualname>"}``, and that path
+    must import back to the same object. Classes defined in ``__main__``
+    (a script run directly or a notebook) are rejected: move them into an
+    importable module.
+
+    The ``id`` is the hex SHA-256 of the canonical JSON of ``params``. It is
+    computed once at construction, so bad params fail immediately.
+
+    Notes:
+        - ``params`` must not be mutated after construction, because the
+          id is cached.
+        - The path uses the *defining* module: ``pyhealth.models.RNN``
+          serializes as ``pyhealth.models.rnn.RNN``. Moving a class to
+          another file changes the id.
+        - ``1`` and ``1.0`` give different ids.
+        - The id is unique only within one benchmark. Storage keys on
+          ``(benchmark.name, evaluation.id)``.
+
+    Attributes:
+        params: The evaluation's parameters.
+        id: Hex SHA-256 of the canonical JSON of ``params``.
+
+    Raises:
+        TypeError: If ``params`` holds a value that is not allowed, such as
+            an arbitrary object, a lambda, a locally defined class, a class
+            defined in ``__main__`` or a class whose path doesn't import
+            back to it, or a mapping key that is not a string.
+        ValueError: If ``params`` holds NaN or infinity, or uses
+            ``"__type__"`` as a key anywhere.
+
+    Examples:
+        >>> from collections import OrderedDict
+        >>> ev = Evaluation({"model": OrderedDict, "lr": 0.001})
+        >>> ev.to_dict()
+        {'lr': 0.001, 'model': {'__type__': 'collections.OrderedDict'}}
+        >>> ev == Evaluation({"lr": 0.001, "model": OrderedDict})
+        True
+        >>> Evaluation.from_dict(ev.to_dict()) == ev
+        True
+    """
+
+    params: Mapping[str, Any]
+    id: str = field(init=False, repr=False)
+
+    def __post_init__(self) -> None:
+        _check_keys(self.params)
+        digest = hashlib.sha256(_dumps(self.params).encode()).hexdigest()
+        object.__setattr__(self, "id", digest)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Returns ``params`` as JSON-compatible data.
+
+        Returns:
+            The params, with classes and functions replaced by their
+            ``{"__type__": ...}`` form.
+        """
+        return json.loads(_dumps(self.params))
+
+    @classmethod
+    def from_dict(cls, data: Mapping[str, Any]) -> "Evaluation":
+        """Builds an evaluation from ``to_dict`` output.
+
+        This is the inverse of :meth:`to_dict`:
+        ``Evaluation.from_dict(ev.to_dict()) == ev``. Each
+        ``{"__type__": "<dotted.path>"}`` mapping is resolved to the class or
+        function at that path. Public re-export paths are accepted, but the
+        id always uses the defining-module path.
+
+        Args:
+            data: JSON-compatible params, as returned by :meth:`to_dict`.
+
+        Returns:
+            The rebuilt evaluation.
+
+        Raises:
+            ValueError: If a path can't be resolved, names something that is
+                not a class or function, or shares its mapping with other
+                keys.
+
+        Examples:
+            >>> ev = Evaluation.from_dict({"c": {"__type__": "json.JSONDecoder"}})
+            >>> ev.to_dict()
+            {'c': {'__type__': 'json.decoder.JSONDecoder'}}
+            >>> Evaluation.from_dict(ev.to_dict()) == ev
+            True
+        """
+        return cls(_decode(data))
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Evaluation):
+            return NotImplemented
+        return self.id == other.id
+
+    def __hash__(self) -> int:
+        return hash(self.id)
+
+
+@dataclass(frozen=True)
+class Result:
+    """The outcome of one evaluation.
+
+    By convention, a failed evaluation is returned as
+    ``Result("failed", {}, {"error": str(exc), "error_type":
+    type(exc).__name__})``, so one failure doesn't discard the batch.
+
+    ``metadata["error"]`` may contain patient data and must never be
+    published. ``error_type`` is safe to display.
+
+    ``artifact`` points to the evaluation's reusable output, such as a
+    trained model checkpoint, so other benchmarks can load it instead of
+    retraining. It may be a file or a directory, and is typically written
+    under ``output_dir``. Like ``metadata["error"]``, it may contain patient
+    data (models can memorize training records) and must never be
+    published.
+
+    Attributes:
+        status: ``"success"`` or ``"failed"``.
+        metrics: Metric names mapped to values.
+        metadata: Any other information about the run.
+        artifact: Path to the reusable output, or ``None`` if the
+            evaluation produced none.
+
+    Examples:
+        >>> Result("success", {"accuracy": 0.9}).metrics
+        {'accuracy': 0.9}
+        >>> from pathlib import Path
+        >>> Result("success", {}, artifact=Path("runs/1/model.pt")).artifact.name
+        'model.pt'
+        >>> exc = ValueError("bad input")
+        >>> Result(
+        ...     "failed", {}, {"error": str(exc), "error_type": type(exc).__name__}
+        ... ).status
+        'failed'
+    """
+
+    status: Literal["success", "failed"]
+    metrics: Mapping[str, Any]
+    metadata: Mapping[str, Any] = field(default_factory=dict)
+    artifact: Path | None = None
+
+
+class Benchmark(ABC):
+    """A benchmark defines and executes evaluations.
+
+    Subclasses implement :attr:`name` and :meth:`_evaluate`. Callers use
+    :meth:`evaluate`, which validates the evaluations, runs them and
+    validates the results.
+
+    The ``_evaluate`` contract:
+        - Any order, grouping, or internal parallelism is allowed.
+        - It returns one ``Result`` per evaluation, and catches
+          per-evaluation failures as failed Results.
+        - ``cache_dir`` may be shared by concurrent jobs, so writes are
+          atomic (temporary file, then ``os.replace``).
+        - ``output_dir`` holds per-run artifacts. Results are always
+          returned, never only written to disk.
+
+    Examples:
+        >>> class Squares(Benchmark):
+        ...     @property
+        ...     def name(self):
+        ...         return "squares"
+        ...
+        ...     def _evaluate(self, evaluations, cache_dir, output_dir):
+        ...         return {
+        ...             ev: Result("success", {"square": ev.params["x"] ** 2})
+        ...             for ev in evaluations
+        ...         }
+        >>> ev = Evaluation({"x": 3})
+        >>> Squares().evaluate([ev])[ev].metrics
+        {'square': 9}
+    """
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Lowercase identifier for the benchmark, e.g. ``"cpbench"``."""
+
+    def validate_evaluations(self, evaluations: Sequence[Evaluation]) -> None:
+        """Checks the evaluations before they run.
+
+        Subclasses extend this via ``super()``.
+
+        Args:
+            evaluations: The evaluations to run.
+
+        Raises:
+            ValueError: If two evaluations share an id. Every duplicate is
+                listed, one per line.
+        """
+        seen: set[str] = set()
+        problems = []
+        for i, ev in enumerate(evaluations):
+            if ev.id in seen:
+                problems.append(f"evaluations[{i}]: duplicate id {ev.id}")
+            seen.add(ev.id)
+        if problems:
+            raise ValueError("\n".join(problems))
+
+    def validate_results(
+        self,
+        evaluations: Sequence[Evaluation],
+        results: Mapping[Evaluation, Result],
+    ) -> None:
+        """Checks that the results match the evaluations.
+
+        Subclasses extend this via ``super()``.
+
+        Args:
+            evaluations: The evaluations that were run.
+            results: The results returned by :meth:`_evaluate`.
+
+        Raises:
+            ValueError: Unless ``results`` has exactly one entry per
+                evaluation. Every missing or extra entry is listed, one per
+                line.
+        """
+        expected = set(evaluations)
+        problems = [
+            f"missing result for evaluation {ev.id}"
+            for ev in evaluations
+            if ev not in results
+        ]
+        problems += [
+            f"unexpected result for evaluation {ev.id}"
+            for ev in results
+            if ev not in expected
+        ]
+        if problems:
+            raise ValueError("\n".join(problems))
+
+    @final
+    def evaluate(
+        self,
+        evaluations: Sequence[Evaluation],
+        cache_dir: Path | None = None,
+        output_dir: Path | None = None,
+    ) -> dict[Evaluation, Result]:
+        """Validates, runs and returns the results of the evaluations.
+
+        Exceptions escaping :meth:`_evaluate` propagate unchanged.
+
+        Args:
+            evaluations: The evaluations to run.
+            cache_dir: Directory for reusable intermediate data, possibly
+                shared by concurrent jobs.
+            output_dir: Directory for per-run artifacts.
+
+        Returns:
+            One ``Result`` per evaluation.
+
+        Raises:
+            ValueError: If the evaluations or the results fail validation.
+        """
+        self.validate_evaluations(evaluations)
+        results = self._evaluate(evaluations, cache_dir, output_dir)
+        self.validate_results(evaluations, results)
+        return results
+
+    @abstractmethod
+    def _evaluate(
+        self,
+        evaluations: Sequence[Evaluation],
+        cache_dir: Path | None,
+        output_dir: Path | None,
+    ) -> dict[Evaluation, Result]:
+        """Runs the evaluations. See the class docstring for the contract."""
diff --git a/tests/core/test_benchmark_base.py b/tests/core/test_benchmark_base.py
new file mode 100644
index 000000000..42e10cf50
--- /dev/null
+++ b/tests/core/test_benchmark_base.py
@@ -0,0 +1,304 @@
+# Author: Felipe Amaral Bonchristiano
+# NetID: felipea5
+# Description: Tests for the pyhealth.benchmarks base abstractions.
+
+import json
+import tempfile
+import types
+import unittest
+from collections import OrderedDict
+from pathlib import Path
+
+from pyhealth.benchmarks import Benchmark, Evaluation, Result
+
+
+def _module_function():
+    return None
+
+
+class _Outer:
+    class Inner:
+        pass
+
+
+# Named differently from the variable, so its path doesn't resolve.
+_Alias = type("_NotBound", (), {})
+
+# Its name is rebound below, so its path resolves to a different class.
+_Shadowed = type("_Shadowed", (), {})
+_OldShadowed = _Shadowed
+_Shadowed = type("_Shadowed", (), {})
+
+
+class _CountingBenchmark(Benchmark):
+    """Returns one success per evaluation and counts ``_evaluate`` calls."""
+
+    def __init__(self, results=None):
+        self.calls = 0
+        self.results = results
+
+    @property
+    def name(self):
+        return "counting"
+
+    def _evaluate(self, evaluations, cache_dir, output_dir):
+        self.calls += 1
+        if self.results is not None:
+            return self.results
+        return {ev: Result("success", {"x": ev.params["x"]}) for ev in evaluations}
+
+
+class TestEvaluation(unittest.TestCase):
+    def test_id_independent_of_key_order(self):
+        a = Evaluation({"a": 1, "b": {"c": 2, "d": 3}})
+        b = Evaluation({"b": {"d": 3, "c": 2}, "a": 1})
+        self.assertEqual(a.id, b.id)
+        self.assertEqual(a, b)
+
+    def test_id_changes_with_any_value(self):
+        base = {"a": 1, "b": [1, 2], "c": {"d": "x"}, "e": json.JSONDecoder}
+        variants = [
+            {**base, "a": 2},
+            {**base, "a": 1.0},
+            {**base, "b": [1, 3]},
+            {**base, "c": {"d": "y"}},
+            {**base, "e": json.JSONEncoder},
+        ]
+        ids = {Evaluation(base).id} | {Evaluation(v).id for v in variants}
+        self.assertEqual(len(ids), len(variants) + 1)
+
+    def test_class_serializes_to_defining_module(self):
+        ev = Evaluation({"cls": json.JSONDecoder})
+        self.assertEqual(
+            ev.to_dict(), {"cls": {"__type__": "json.decoder.JSONDecoder"}}
+        )
+
+    def test_function_serializes_by_path(self):
+        ev = Evaluation({"fn": json.dumps})
+        self.assertEqual(ev.to_dict(), {"fn": {"__type__": "json.dumps"}})
+
+    def test_rejects_arbitrary_object(self):
+        with self.assertRaises(TypeError):
+            Evaluation({"x": object()})
+
+    def test_rejects_lambda(self):
+        with self.assertRaises(TypeError):
+            Evaluation({"fn": lambda: None})
+
+    def test_rejects_local_class(self):
+        class Local:
+            pass
+
+        with self.assertRaises(TypeError):
+            Evaluation({"cls": Local})
+
+    def test_rejects_main_class(self):
+        cls = type("Script", (), {"__module__": "__main__"})
+        with self.assertRaisesRegex(TypeError, "importable module"):
+            Evaluation({"cls": cls})
+
+    def test_rejects_type_alias(self):
+        with self.assertRaises(TypeError):
+            Evaluation({"cls": _Alias})
+
+    def test_rejects_rebound_class(self):
+        with self.assertRaises(TypeError):
+            Evaluation({"cls": _OldShadowed})
+        Evaluation({"cls": _Shadowed})
+
+    def test_rejects_unresolvable_builtin_type(self):
+        for cls in [type(None), types.FunctionType]:
+            with self.subTest(cls=cls), self.assertRaises(TypeError):
+                Evaluation({"cls": cls})
+
+    def test_rejects_nan(self):
+        with self.assertRaises(ValueError):
+            Evaluation({"x": float("nan")})
+
+    def test_golden_id(self):
+        ev = Evaluation(
+            {
+                "cls": OrderedDict,
+                "n": 3,
+                "lr": 0.5,
+                "flag": True,
+                "none": None,
+                "name": "x",
+                "seq": [1, 2.5, "a"],
+                "nested": {"b": 1, "a": (True, None)},
+            }
+        )
+        self.assertEqual(
+            ev.id,
+            "d3322d1ed849f5b773ab44fc80198a64a0d6533ef515c7006577abd3a31fd613",
+        )
+
+    def test_dedupes_in_set_and_dict(self):
+        a = Evaluation({"x": 1, "fn": _module_function})
+        b = Evaluation({"fn": _module_function, "x": 1})
+        self.assertIsNot(a, b)
+        self.assertEqual(len({a, b}), 1)
+        self.assertEqual(len({a: 1, b: 2}), 1)
+
+
+class TestEvaluationFromDict(unittest.TestCase):
+    def test_round_trip_plain_params(self):
+        ev = Evaluation({"a": 1, "b": [1.5, "x", None], "c": {"d": True}})
+        self.assertEqual(Evaluation.from_dict(ev.to_dict()), ev)
+
+    def test_round_trip_types(self):
+        ev = Evaluation(
+            {
+                "cls": json.JSONDecoder,
+                "fn": _module_function,
+                "nested": _Outer.Inner,
+                "seq": [OrderedDict, {"fn": json.dumps}],
+            }
+        )
+        rebuilt = Evaluation.from_dict(ev.to_dict())
+        self.assertEqual(rebuilt, ev)
+        self.assertIs(rebuilt.params["nested"], _Outer.Inner)
+        self.assertIs(rebuilt.params["fn"], _module_function)
+
+    def test_reexport_path_uses_defining_module(self):
+        ev = Evaluation.from_dict({"c": {"__type__": "json.JSONDecoder"}})
+        self.assertEqual(ev, Evaluation({"c": json.decoder.JSONDecoder}))
+        self.assertEqual(
+            ev.to_dict(), {"c": {"__type__": "json.decoder.JSONDecoder"}}
+        )
+
+    def test_unresolvable_path_raises(self):
+        for path in ["json.NoSuchThing", "no_such_module_xyz.Thing", ""]:
+            with self.subTest(path=path), self.assertRaises(ValueError):
+                Evaluation.from_dict({"c": {"__type__": path}})
+
+    def test_non_type_target_raises(self):
+        with self.assertRaisesRegex(ValueError, "math.pi"):
+            Evaluation.from_dict({"c": {"__type__": "math.pi"}})
+
+    def test_type_with_extra_keys_raises(self):
+        with self.assertRaisesRegex(ValueError, "json.JSONDecoder"):
+            Evaluation.from_dict({"c": {"__type__": "json.JSONDecoder", "extra": 1}})
+
+    def test_type_key_in_params_raises(self):
+        for params in [
+            {"__type__": "x"},
+            {"a": {"__type__": "json.JSONDecoder"}},
+            {"a": [{"b": {"__type__": 1}}]},
+        ]:
+            with self.subTest(params=params), self.assertRaises(ValueError):
+                Evaluation(params)
+
+    def test_non_str_key_raises(self):
+        for params in [
+            {"p": {1: "a"}},
+            {"p": [{None: "a"}]},
+            {"p": {"q": {True: "a"}}},
+            {"p": {1.5: "a"}},
+        ]:
+            with self.subTest(params=params), self.assertRaises(TypeError):
+                Evaluation(params)
+
+
+class TestBenchmark(unittest.TestCase):
+    def test_evaluate_returns_results(self):
+        evs = [Evaluation({"x": 1}), Evaluation({"x": 2})]
+        results = _CountingBenchmark().evaluate(evs)
+        self.assertEqual(results[evs[1]].metrics, {"x": 2})
+
+    def test_duplicates_raise_before_evaluate(self):
+        bench = _CountingBenchmark()
+        evs = [Evaluation({"x": 1}), Evaluation({"x": 2}), Evaluation({"x": 1})]
+        with self.assertRaises(ValueError):
+            bench.evaluate(evs)
+        self.assertEqual(bench.calls, 0)
+
+    def test_missing_results_raise(self):
+        evs = [Evaluation({"x": 1}), Evaluation({"x": 2})]
+        bench = _CountingBenchmark(results={evs[0]: Result("success", {})})
+        with self.assertRaisesRegex(ValueError, "missing"):
+            bench.evaluate(evs)
+
+    def test_extra_results_raise(self):
+        evs = [Evaluation({"x": 1})]
+        extra = Evaluation({"x": 2})
+        bench = _CountingBenchmark(
+            results={
+                evs[0]: Result("success", {}),
+                extra: Result("success", {}),
+            }
+        )
+        with self.assertRaisesRegex(ValueError, "unexpected"):
+            bench.evaluate(evs)
+
+    def test_artifact_defaults_to_none(self):
+        self.assertIsNone(Result("success", {}).artifact)
+
+    def test_artifact_returned_from_evaluate(self):
+        class Checkpointing(Benchmark):
+            @property
+            def name(self):
+                return "checkpointing"
+
+            def _evaluate(self, evaluations, cache_dir, output_dir):
+                results = {}
+                for ev in evaluations:
+                    path = output_dir / f"{ev.id}.pt"
+                    path.write_text("weights")
+                    results[ev] = Result("success", {}, artifact=path)
+                return results
+
+        ev = Evaluation({"x": 1})
+        with tempfile.TemporaryDirectory() as tmp:
+            result = Checkpointing().evaluate([ev], output_dir=Path(tmp))[ev]
+            self.assertEqual(result.artifact, Path(tmp) / f"{ev.id}.pt")
+            self.assertTrue(result.artifact.exists())
+
+    def test_mixed_success_and_failure_returned(self):
+        class Flaky(Benchmark):
+            @property
+            def name(self):
+                return "flaky"
+
+            def _evaluate(self, evaluations, cache_dir, output_dir):
+                results = {}
+                for ev in evaluations:
+                    try:
+                        if ev.params["x"] < 0:
+                            raise ValueError("negative")
+                        results[ev] = Result("success", {"x": ev.params["x"]})
+                    except ValueError as exc:
+                        results[ev] = Result(
+                            "failed",
+                            {},
+                            {"error": str(exc), "error_type": type(exc).__name__},
+                        )
+                return results
+
+        evs = [Evaluation({"x": 1}), Evaluation({"x": -1})]
+        results = Flaky().evaluate(evs)
+        self.assertEqual(len(results), 2)
+        self.assertEqual(results[evs[0]].status, "success")
+        self.assertEqual(results[evs[1]].status, "failed")
+        self.assertEqual(results[evs[1]].metadata["error_type"], "ValueError")
+
+    def test_missing_name_cannot_instantiate(self):
+        class NoName(Benchmark):
+            def _evaluate(self, evaluations, cache_dir, output_dir):
+                return {}
+
+        with self.assertRaises(TypeError):
+            NoName()
+
+    def test_missing_evaluate_cannot_instantiate(self):
+        class NoEvaluate(Benchmark):
+            @property
+            def name(self):
+                return "noevaluate"
+
+        with self.assertRaises(TypeError):
+            NoEvaluate()
+
+
+if __name__ == "__main__":
+    unittest.main()