tests: add API spec test suite

README.md

@@ -115,6 +115,10 @@ uv run pytest -m "not slow"
 #   --fork   Target fork (default: devnet)
 #   --output Optional directory for filled fixtures
 uv run fill --clean --fork=devnet
+
+# Run API conformance tests against an external client implementation
+# Usage: uv run apitest <server-url> [pytest-args]
+uv run apitest http://localhost:5052
 ```
 
 ### Code Quality
@@ -221,6 +225,7 @@ def test_withdrawal_amount_above_uint64_max():
 | Serve docs                                    | `uv run mkdocs serve`                                  |
 | Run everything (checks + tests + docs)        | `uvx tox`                                              |
 | Run all quality checks (no tests/docs)        | `uvx tox -e all-checks`                                |
+| Test external client API conformance          | `uv run apitest http://localhost:5052`                 |
 | Run consensus node                            | `uv run python -m lean_spec --genesis config.yaml`     |
 | Build Docker test image                       | `docker build -t lean-spec:test .`                     |
 | Build Docker node image                       | `docker build --target node -t lean-spec:node .`       |

packages/testing/pyproject.toml

@@ -35,6 +35,7 @@ Issues = "https://github.com/leanEthereum/lean-spec/issues"
 
 [project.scripts]
 fill = "framework.cli.fill:fill"
+apitest = "framework.cli.apitest:apitest"
 
 [tool.setuptools.packages.find]
 where = ["src"]

packages/testing/src/framework/cli/apitest.py

@@ -0,0 +1,71 @@
+"""CLI command for running API conformance tests against an external server."""
+
+import sys
+from pathlib import Path
+from typing import Sequence
+
+import click
+import pytest
+
+
+@click.command(
+    context_settings={
+        "ignore_unknown_options": True,
+        "allow_extra_args": True,
+    }
+)
+@click.argument("server_url")
+@click.argument("pytest_args", nargs=-1, type=click.UNPROCESSED)
+@click.pass_context
+def apitest(
+    ctx: click.Context,
+    server_url: str,
+    pytest_args: Sequence[str],
+) -> None:
+    """
+    Run API conformance tests against an external server, i.e. a client implementation.
+
+    SERVER_URL is the base URL of the API server (e.g., http://localhost:5052).
+
+    For testing the local leanSpec implementation, use `uv run pytest tests/api`
+    which automatically starts a local server.
+
+    Examples:
+        # Run against external server
+        apitest http://localhost:5052
+
+        # Run with verbose output
+        apitest http://localhost:5052 -v
+
+        # Run specific test
+        apitest http://localhost:5052 -k test_health
+    """
+    config_path = Path(__file__).parent / "pytest_ini_files" / "pytest-apitest.ini"
+
+    # Find project root
+    project_root = Path.cwd()
+    while project_root != project_root.parent:
+        if (project_root / "pyproject.toml").exists():
+            pyproject = project_root / "pyproject.toml"
+            if "[tool.uv.workspace]" in pyproject.read_text():
+                break
+        project_root = project_root.parent
+
+    # Build pytest arguments
+    args = [
+        "-c",
+        str(config_path),
+        f"--rootdir={project_root}",
+        f"--server-url={server_url}",
+        "tests/api",
+    ]
+
+    args.extend(pytest_args)
+    args.extend(ctx.args)
+
+    exit_code = pytest.main(args)
+    sys.exit(exit_code)
+
+
+if __name__ == "__main__":
+    apitest()

packages/testing/src/framework/cli/pytest_ini_files/pytest-apitest.ini

@@ -0,0 +1,20 @@
+[pytest]
+# Configuration for apitest command
+
+# Search for API conformance tests
+testpaths = tests/api
+
+# Options
+addopts =
+    # Show shorter tracebacks
+    --tb=short
+    # Disable coverage for API testing
+    --no-cov
+
+# Test discovery
+python_files = test_*.py
+python_classes = Test*
+python_functions = test_*
+
+# Minimal output
+console_output_style = classic

src/lean_spec/subspecs/api/server.py

@@ -28,21 +28,23 @@
 logger = logging.getLogger(__name__)
 
 
-def _no_store() -> Store | None:
-    """Default store getter that returns None."""
-    return None
-
-
 async def _handle_health(_request: web.Request) -> web.Response:
-    """Handle health check endpoint."""
-    return web.json_response({"status": "healthy", "service": "lean-spec-api"})
+    """
+    Handle health check endpoint.
+
+    Response format:
+    - status: The status of the API server. Always return "healthy" when the API endpoint is served.
+    - service: The API service name. Fixed to "lean-rpc-api".
+    """
+    return web.json_response({"status": "healthy", "service": "lean-rpc-api"})
 
 
 async def _handle_metrics(_request: web.Request) -> web.Response:
     """Handle Prometheus metrics endpoint."""
     return web.Response(
         body=generate_metrics(),
-        content_type="text/plain; version=0.0.4; charset=utf-8",
+        content_type="text/plain; version=0.0.4",
+        charset="utf-8",
     )
 
 
@@ -75,7 +77,7 @@ class ApiServer:
     config: ApiServerConfig
     """Server configuration."""
 
-    store_getter: Callable[[], Store | None] = _no_store
+    store_getter: Callable[[], Store | None] | None = None
     """Callable that returns the current Store instance."""
 
     _runner: web.AppRunner | None = field(default=None, init=False)
@@ -87,7 +89,7 @@ class ApiServer:
     @property
     def store(self) -> Store | None:
         """Get the current Store instance."""
-        return self.store_getter()
+        return self.store_getter() if self.store_getter else None
 
     async def start(self) -> None:
         """Start the API server in the background."""
@@ -188,6 +190,6 @@ async def _handle_justified_checkpoint(self, _request: web.Request) -> web.Respo
         return web.json_response(
             {
                 "slot": justified.slot,
-                "root": justified.root.hex(),
+                "root": "0x" + justified.root.hex(),
             }
         )

tests/api/__init__.py

@@ -0,0 +1 @@
+"""API conformance test suite for validating leanSpec client implementations."""

tests/api/conftest.py

@@ -0,0 +1,145 @@
+"""Pytest configuration for API conformance tests."""
+
+import asyncio
+import threading
+import time
+from typing import TYPE_CHECKING, Generator
+
+import httpx
+import pytest
+
+if TYPE_CHECKING:
+    from lean_spec.subspecs.api import ApiServer
+
+# Default port for auto-started local server
+DEFAULT_PORT = 15099
+
+
+class _ServerThread(threading.Thread):
+    """Thread that runs the API server in its own event loop."""
+
+    def __init__(self, port: int):
+        super().__init__(daemon=True)
+        self.port = port
+        self.server: ApiServer | None = None
+        self.loop: asyncio.AbstractEventLoop | None = None
+        self.ready = threading.Event()
+        self.error: Exception | None = None
+
+    def run(self) -> None:
+        """Run the server in a new event loop."""
+        try:
+            self.loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(self.loop)
+
+            self.server = self._create_server()
+            self.loop.run_until_complete(self.server.start())
+            self.ready.set()
+
+            self.loop.run_forever()
+
+        except Exception as e:
+            self.error = e
+            self.ready.set()
+        finally:
+            if self.loop:
+                self.loop.close()
+
+    def _create_server(self) -> "ApiServer":
+        """Create the API server with a test store."""
+        from lean_spec.subspecs.api import ApiServer, ApiServerConfig
+        from lean_spec.subspecs.containers import Block, BlockBody, State, Validator
+        from lean_spec.subspecs.containers.block.types import AggregatedAttestations
+        from lean_spec.subspecs.containers.slot import Slot
+        from lean_spec.subspecs.containers.state import Validators
+        from lean_spec.subspecs.containers.validator import ValidatorIndex
+        from lean_spec.subspecs.forkchoice import Store
+        from lean_spec.subspecs.ssz.hash import hash_tree_root
+        from lean_spec.types import Bytes32, Bytes52, Uint64
+
+        validators = Validators(
+            data=[
+                Validator(pubkey=Bytes52(b"\x00" * 52), index=ValidatorIndex(i)) for i in range(3)
+            ]
+        )
+
+        genesis_state = State.generate_genesis(
+            genesis_time=Uint64(int(time.time())),
+            validators=validators,
+        )
+
+        genesis_block = Block(
+            slot=Slot(0),
+            proposer_index=ValidatorIndex(0),
+            parent_root=Bytes32.zero(),
+            state_root=hash_tree_root(genesis_state),
+            body=BlockBody(attestations=AggregatedAttestations(data=[])),
+        )
+
+        store = Store.get_forkchoice_store(genesis_state, genesis_block)
+
+        config = ApiServerConfig(host="127.0.0.1", port=self.port)
+        return ApiServer(config=config, store_getter=lambda: store)
+
+    def stop(self) -> None:
+        """Stop the server and event loop."""
+        if self.server and self.loop:
+            self.loop.call_soon_threadsafe(self.server.stop)
+            self.loop.call_soon_threadsafe(self.loop.stop)
+
+
+def _wait_for_server(url: str, timeout: float = 5.0) -> bool:
+    """Wait for server to be ready by polling the health endpoint."""
+    start = time.time()
+    while time.time() - start < timeout:
+        try:
+            response = httpx.get(f"{url}/lean/v0/health", timeout=1.0)
+            if response.status_code == 200:
+                return True
+        except (httpx.ConnectError, httpx.ReadTimeout):
+            pass
+        time.sleep(0.1)
+    return False
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    """Add --server-url option for testing against external servers."""
+    parser.addoption(
+        "--server-url",
+        action="store",
+        default=None,
+        help="External server URL. If not provided, starts a local server.",
+    )
+
+
+@pytest.fixture(scope="session")
+def server_url(request: pytest.FixtureRequest) -> Generator[str, None, None]:
+    """
+    Provide the server URL for API tests.
+
+    If --server-url is provided, uses that external server.
+    Otherwise, starts a local leanSpec server for the test session.
+    """
+    external_url = request.config.getoption("--server-url")
+
+    if external_url:
+        # Use external server
+        yield external_url
+    else:
+        # Start local server
+        server_thread = _ServerThread(DEFAULT_PORT)
+        server_thread.start()
+        server_thread.ready.wait(timeout=10.0)
+
+        if server_thread.error:
+            pytest.fail(f"Failed to start local server: {server_thread.error}")
+
+        url = f"http://127.0.0.1:{DEFAULT_PORT}"
+
+        if not _wait_for_server(url):
+            server_thread.stop()
+            pytest.fail("Local server failed to become ready")
+
+        yield url
+
+        server_thread.stop()

tests/api/test_finalized_state.py

@@ -0,0 +1,54 @@
+"""Tests for the finalized state endpoint with SSZ validation."""
+
+import httpx
+
+from lean_spec.subspecs.containers import State
+
+
+def test_finalized_state_returns_200(server_url: str) -> None:
+    """Finalized state endpoint returns 200 status code."""
+    response = httpx.get(
+        f"{server_url}/lean/v0/states/finalized",
+        headers={"Accept": "application/octet-stream"},
+    )
+    assert response.status_code == 200
+
+
+def test_finalized_state_content_type_is_octet_stream(server_url: str) -> None:
+    """Finalized state endpoint returns octet-stream content type."""
+    response = httpx.get(
+        f"{server_url}/lean/v0/states/finalized",
+        headers={"Accept": "application/octet-stream"},
+    )
+    content_type = response.headers.get("content-type", "")
+    assert "application/octet-stream" in content_type
+
+
+def test_finalized_state_ssz_deserializes(server_url: str) -> None:
+    """Finalized state SSZ bytes deserialize to a valid State object."""
+    response = httpx.get(
+        f"{server_url}/lean/v0/states/finalized",
+        headers={"Accept": "application/octet-stream"},
+    )
+    state = State.decode_bytes(response.content)
+    assert state is not None
+
+
+def test_finalized_state_has_valid_slot(server_url: str) -> None:
+    """Finalized state has a non-negative slot."""
+    response = httpx.get(
+        f"{server_url}/lean/v0/states/finalized",
+        headers={"Accept": "application/octet-stream"},
+    )
+    state = State.decode_bytes(response.content)
+    assert int(state.slot) >= 0
+
+
+def test_finalized_state_has_validators(server_url: str) -> None:
+    """Finalized state has at least one validator."""
+    response = httpx.get(
+        f"{server_url}/lean/v0/states/finalized",
+        headers={"Accept": "application/octet-stream"},
+    )
+    state = State.decode_bytes(response.content)
+    assert len(state.validators) > 0