Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
name: Test

on:
push:
pull_request:

jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install uv
uses: astral-sh/setup-uv@v5
with:
enable-cache: true
- name: Install dependencies
run: uv sync
- name: Lint with ruff
run: uv run ruff check bootstraptools
- name: Check formatting with ruff
run: uv run ruff format --check bootstraptools
- name: Type check with ty
run: uv run ty check bootstraptools
- name: Test with pytest
run: uv run pytest --cov=bootstraptools
10 changes: 9 additions & 1 deletion bootstraptools/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,15 @@
from bootstraptools.resample import draw, draw_counts, out_of_bag, select
from bootstraptools.seeds import derive_seeds, replicate_seeds
from bootstraptools.store import Run, init
from bootstraptools.uq import basic, bayesian_bootstrap, bca, ci, normal, percentile, studentized
from bootstraptools.uq import (
basic,
bayesian_bootstrap,
bca,
ci,
normal,
percentile,
studentized,
)

__all__ = [
"draw",
Expand Down
27 changes: 15 additions & 12 deletions bootstraptools/optimism.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@

from __future__ import annotations

from collections.abc import Callable
from pathlib import Path
from typing import Any, Callable
from typing import Any

import numpy as np
from scipy.special import ndtri
Expand Down Expand Up @@ -188,7 +189,9 @@ def _load_run_arrays(
return y, apparent_p, replicate_ps, counts


def optimism_from_run(store: str | Path, run_id: str, metric: MetricFn) -> dict[str, Any]:
def optimism_from_run(
store: str | Path, run_id: str, metric: MetricFn
) -> dict[str, Any]:
"""Load an `optimism_bootstrap` run's artifacts and apply `optimism_correction`.

Requires a run produced by the `optimism_bootstrap` procedure with a
Expand Down Expand Up @@ -219,17 +222,17 @@ def optimism_location_shifted_ci(
higher-better or lower-better, since both `corrected` and the percentile
endpoints are computed in `metric`'s native direction.

This interval is `(q_lo, q_hi) - optimism`, NOT `corrected ± (something)`.
Because the apparent statistic's bootstrap distribution (`p_boot`) carries
its own bias relative to the full-data apparent value, the interval is not
guaranteed to be centered on or contain the `corrected` point estimate.
This interval is `(q_lo, q_hi) - optimism`, NOT `corrected ± (something)`.
Because the apparent statistic's bootstrap distribution (`p_boot`) carries
its own bias relative to the full-data apparent value, the interval is not
guaranteed to be centered on or contain the `corrected` point estimate.

COVERAGE CAVEAT: this interval covers well in large samples but
under-covers in small samples (empirically ~70-80% actual coverage at
a nominal 95% level), because it ignores the sampling variability of
the optimism term `O` itself. Only the spread of `theta_boot` is
used, and `O` is treated as a fixed shift. The double bootstrap
(Noma et al. 2021 "method 2") corrects this but is much more expensive.
(Noma et al. 2021 "method 2") corrects this but is much more expensive.

Parameters
----------
Expand Down Expand Up @@ -506,7 +509,7 @@ def error_632(
"""The `.632` bootstrap estimate of out-of-sample per-sample loss.

For per-sample-loss metrics (e.g., 0/1 error rate, squared error, log-loss
the discontinuous / improper-scoring-rule family), combining the apparent
the discontinuous / improper-scoring-rule family), combining the apparent
(in-sample) error with the leave-one-out bootstrap (out-of-bag) error:

err_632 = 0.368 * err_app + 0.632 * eps0
Expand Down Expand Up @@ -615,7 +618,9 @@ def error_632_plus(
}


def error_632_from_run(store: str | Path, run_id: str, loss_fn: LossFn) -> dict[str, Any]:
def error_632_from_run(
store: str | Path, run_id: str, loss_fn: LossFn
) -> dict[str, Any]:
"""Load an `optimism_bootstrap` run's artifacts and apply `error_632`."""
y, apparent_p, replicate_ps, counts = _load_run_arrays(store, run_id)
return error_632(y, apparent_p, replicate_ps, counts, loss_fn)
Expand All @@ -629,9 +634,7 @@ def error_632_plus_from_run(
return error_632_plus(y, apparent_p, replicate_ps, counts, loss_fn)


def double_bootstrap_ci(
theta_corr: np.ndarray, alpha: float = 0.05
) -> dict[str, Any]:
def double_bootstrap_ci(theta_corr: np.ndarray, alpha: float = 0.05) -> dict[str, Any]:
"""CI on the Efron-Gong optimism-corrected estimate via Noma (2021) "method 2".

`theta_corr` holds the R per-OUTER-replicate optimism-corrected estimates
Expand Down
2 changes: 1 addition & 1 deletion bootstraptools/procedures.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ def optimism_bootstrap(

@dataclass(frozen=True)
class DoubleResamplePlan:
"""One outer replicate of Noma method 2 (double/two-stage bootstrap),
"""One outer replicate of Noma method 2 (double/two-stage bootstrap),
paired with its nested inner resample plans."""

outer_idx: int
Expand Down
4 changes: 1 addition & 3 deletions bootstraptools/query.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,9 +132,7 @@ def query_apparent(store: str | Path, run_id: str) -> pl.DataFrame:
"""
path = Path(store) / "runs" / run_id / "apparent.parquet"
if not path.is_file():
raise FileNotFoundError(
f"No apparent table found for run {run_id!r}: {path}"
)
raise FileNotFoundError(f"No apparent table found for run {run_id!r}: {path}")
return pl.read_parquet(path)


Expand Down
1 change: 0 additions & 1 deletion bootstraptools/seeds.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

from __future__ import annotations

import numpy as np
Expand Down
19 changes: 13 additions & 6 deletions bootstraptools/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

import json
import re
from datetime import datetime, timezone
from datetime import UTC, datetime
from importlib.metadata import PackageNotFoundError, version
from pathlib import Path
from typing import Any
Expand Down Expand Up @@ -99,7 +99,7 @@ def init(
"config": config or {},
"tags": tags or [],
"store_model_state": store_model_state,
"created": datetime.now(timezone.utc).isoformat(),
"created": datetime.now(UTC).isoformat(),
"status": "running",
"n_replicates": 0,
"summary": {},
Expand Down Expand Up @@ -243,7 +243,8 @@ def log_replicate(
arrays_dir = self.run_dir / "arrays"
arrays_dir.mkdir(exist_ok=True)
np.savez(
arrays_dir / f"replicate_{plan.replicate_idx:04d}.npz", **arrays
arrays_dir / f"replicate_{plan.replicate_idx:04d}.npz",
**arrays, # ty: ignore[invalid-argument-type]
)

if model_state:
Expand All @@ -256,7 +257,7 @@ def log_replicate(
model_state_dir.mkdir(exist_ok=True)
np.savez(
model_state_dir / f"replicate_{plan.replicate_idx:04d}.npz",
**model_state,
**model_state, # ty: ignore[invalid-argument-type]
)

def log_apparent(
Expand Down Expand Up @@ -317,7 +318,10 @@ def log_apparent(
if arrays:
apparent_dir = self.run_dir / "apparent"
apparent_dir.mkdir(exist_ok=True)
np.savez(apparent_dir / "arrays.npz", **arrays)
np.savez(
apparent_dir / "arrays.npz",
**arrays, # ty: ignore[invalid-argument-type]
)

if model_state:
if not self.store_model_state:
Expand All @@ -327,7 +331,10 @@ def log_apparent(
)
apparent_dir = self.run_dir / "apparent"
apparent_dir.mkdir(exist_ok=True)
np.savez(apparent_dir / "model_state.npz", **model_state)
np.savez(
apparent_dir / "model_state.npz",
**model_state, # ty: ignore[invalid-argument-type]
)

self._apparent_logged = True

Expand Down
18 changes: 7 additions & 11 deletions bootstraptools/uq.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,7 @@ def _z0(theta_boot: np.ndarray, theta_hat: float) -> float:
"""Bias-correction z0 (SciPy convention, kept finite at the edges)."""
theta_boot = np.asarray(theta_boot)
B = len(theta_boot)
prop = (
np.sum(theta_boot < theta_hat) + np.sum(theta_boot <= theta_hat)
) / (2 * B)
prop = (np.sum(theta_boot < theta_hat) + np.sum(theta_boot <= theta_hat)) / (2 * B)
return float(ndtri(prop))


Expand Down Expand Up @@ -113,9 +111,7 @@ def bca(
elif membership is not None:
a = _acceleration_from_jack(_jab_theta_tilde(theta_boot, membership))
else:
raise ValueError(
"BCa requires one of: acceleration, jackknife, or membership"
)
raise ValueError("BCa requires one of: acceleration, jackknife, or membership")

zL = ndtri(alpha / 2)
zU = ndtri(1 - alpha / 2)
Expand Down Expand Up @@ -193,23 +189,23 @@ def ci(
`theta_boot`, so it is called directly instead.
"""
if method == "percentile":
return percentile(theta_boot, **kwargs) # type: ignore[arg-type]
return percentile(theta_boot, **kwargs) # ty: ignore[invalid-argument-type]
if method == "basic":
if theta_hat is None:
raise ValueError("basic requires theta_hat")
return basic(theta_boot, theta_hat, **kwargs) # type: ignore[arg-type]
return basic(theta_boot, theta_hat, **kwargs) # ty: ignore[invalid-argument-type]
if method == "normal":
if theta_hat is None:
raise ValueError("normal requires theta_hat")
return normal(theta_boot, theta_hat, **kwargs) # type: ignore[arg-type]
return normal(theta_boot, theta_hat, **kwargs) # ty: ignore[invalid-argument-type]
if method == "bca":
if theta_hat is None:
raise ValueError("bca requires theta_hat")
return bca(theta_boot, theta_hat, **kwargs) # type: ignore[arg-type]
return bca(theta_boot, theta_hat, **kwargs) # ty: ignore[invalid-argument-type]
if method == "studentized":
if theta_hat is None:
raise ValueError("studentized requires theta_hat")
if "se_boot" not in kwargs or "se_hat" not in kwargs:
raise ValueError("studentized requires se_boot and se_hat")
return studentized(theta_boot, theta_hat, **kwargs) # type: ignore[arg-type]
return studentized(theta_boot, theta_hat, **kwargs) # ty: ignore[invalid-argument-type]
raise ValueError(f"Unknown method {method!r}")
17 changes: 17 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,20 @@ dependencies = [
"scikit-learn>=1.8.0",
"scipy>=1.17.1",
]

[dependency-groups]
dev = [
"pytest-cov>=6.0,<7.0",
"ruff>=0.15",
"ty>=0.0.1a0",
]

[tool.ruff]
target-version = "py313"

[tool.ruff.lint]
select = ["E", "F", "I", "UP", "B"]
ignore = ["E501"]

[tool.ty.environment]
python-version = "3.13"
Loading
Loading