ENH: Add tF confidence intervals at multiple levels for IV2SLS

MANIFEST.in

@@ -4,6 +4,7 @@ recursive-include doc/source *
 recursive-include linearmodels *.pyx
 recursive-include linearmodels/tests *.txt *.csv *.dta
 recursive-include linearmodels/datasets *
+recursive-include linearmodels/iv/_tf_data *.npz *.json *.rst LICENSE
 recursive-include examples *
 include linearmodels/_version.py
 include LICENSE.md

doc/source/iv/index.rst

@@ -26,6 +26,7 @@ tests.
    examples/advanced-examples.ipynb
    examples/using-formulas.ipynb
    examples/absorbing-regression.ipynb
+   weak-instruments
    reference
    mathematical-formula
 

doc/source/iv/weak-instruments.rst

@@ -0,0 +1,93 @@
+Confidence intervals with a weak instrument
+===========================================
+
+:meth:`~linearmodels.iv.results.IVResults.tf_conf_int` constructs a tF interval
+for one endogenous regressor and one excluded instrument. It is
+an explicit alternative to the conventional :meth:`~linearmodels.iv.results.IVResults.conf_int`.
+It supports IV2SLS with unadjusted or heteroskedasticity-robust covariance,
+``debiased=False``, and constant observation weights. Other configurations raise
+``ValueError``. Included exogenous controls are allowed.
+
+For example, use proximity to a four-year college as an instrument for education:
+
+.. doctest::
+
+    >>> from linearmodels.datasets import card
+    >>> from linearmodels.iv import IV2SLS
+    >>> data = card.load()
+    >>> columns = ["lwage", "educ", "exper", "expersq", "black", "south", "smsa", "nearc4"]
+    >>> data = data[columns].dropna()
+    >>> formula = "lwage ~ 1 + exper + expersq + black + south + smsa + [educ ~ nearc4]"
+    >>> result = IV2SLS.from_formula(formula, data).fit(cov_type="robust")
+    >>> interval = result.tf_conf_int()
+    >>> list(interval.index), list(interval.columns)
+    (['educ'], ['lower', 'upper'])
+    >>> conventional = result.conf_int().loc["educ"]
+    >>> bool(interval.loc["educ", "lower"] <= conventional["lower"])
+    True
+    >>> bool(interval.loc["educ", "upper"] >= conventional["upper"])
+    True
+    >>> interval_99 = result.tf_conf_int(level=0.99)
+    >>> bool(interval_99.loc["educ", "lower"] < interval.loc["educ", "lower"])
+    True
+    >>> bool(interval_99.loc["educ", "upper"] > interval.loc["educ", "upper"])
+    True
+
+The first-stage statistic is the squared t statistic for the excluded instrument,
+using the fitted model's covariance convention. The coefficient and SE are reused.
+Neither the original SE nor the conventional p-values and intervals are changed.
+
+The default ``method="table"`` uses bundled tables generated by
+`tf-inference <https://github.com/finite-sample/tf-inference>`_ from Appendix B.3
+of Lee, McCrary, Moreira and Porter's
+`Valid t-Ratio Inference for IV <https://doi.org/10.1257/aer.20211063>`_.
+The `author manuscript <https://eml.berkeley.edu/~jmccrary/LMMP_2_15_22.pdf>`_
+contains the construction. The generated tables contain at least 16,385 rows
+per level and are cached after the first load. They require no additional
+dependency, network access or numerical calibration at runtime.
+
+``level`` is a confidence level, matching ``conf_int(level=...)``; the default
+is 0.95. Bundled levels are 0.75, 0.8, 0.85, 0.875, 0.9, 0.925, 0.94, 0.95,
+0.96, 0.97, 0.975, 0.98, 0.99, 0.995, 0.998, 0.999, 0.9995 and 0.9999.
+No interpolation across confidence levels or automatic numerical computation
+is performed. An unavailable level raises an error.
+
+For an unconventional level, explicitly request numerical calibration::
+
+    interval = result.tf_conf_int(level=0.963, method="compute")
+
+This requires Python 3.11 or later and the optional tf-inference package.
+The validated source revision can be installed with::
+
+    python -m pip install git+https://github.com/finite-sample/tf-inference.git@ebaa0f8ff0286203b0a1488abdf3330ace84c4fe
+
+The first computation can take several seconds; the calibration is cached by
+level for later calls. The numerical implementation accepts levels between
+0.75 and 0.9999. For custom numerical settings, inspection of calibration
+diagnostics, or export of reusable constants, use tf-inference directly with
+``alpha=1-level``. The finite calibration search should be checked beyond its
+search settings before relying on an unconventional level for research.
+
+The unbounded-interval threshold is the squared normal critical value for the
+requested level. At 95% it is about 3.84145882. F strictly above this boundary
+gives finite bounds, including the interval between this boundary and 4.
+For F=10 the 95% critical value is about 3.4303537; an estimate of 1 with SE=0.4
+therefore gives bounds of approximately [-0.37214, 2.37214]. Each confidence
+level has its own calibrated plateau. At 99%, for example, its critical value
+is about 2.72549, above the normal 99% critical value of 2.57583.
+
+The code interpolates the regularized squared critical value
+``(F - boundary) * critical(F)**2``, which has a finite limit at the boundary.
+The tables' source revision, license, checksums, construction settings and
+validation provenance are included in ``linearmodels/iv/_tf_data``.
+
+Calibration uses deterministic integration, a finite nuisance-parameter grid
+and local searches. Held-out checks and grid refinement assess the numerical
+approximation; they do not prove a uniform size bound. This implementation does
+not claim the conservative rounding guarantee of the paper's printed-table
+interpolation. It is not a p-value calculation or a guarantee of finite-sample
+coverage. Instrument validity and the
+model's other identifying and sampling assumptions still matter. The newer VtF
+procedure uses additional information and can yield shorter intervals; this
+method implements tF, not VtF. The existing ``anderson_rubin`` property is a test
+of overidentification, not a weak-instrument confidence set for this coefficient.

linearmodels/iv/_tf.py

@@ -0,0 +1,92 @@
+"""Generated tF critical values for single-instrument IV inference."""
+
+from functools import lru_cache
+import hashlib
+from importlib import import_module
+from importlib.resources import files
+import io
+import json
+from math import inf, isfinite, sqrt
+from typing import Any
+
+from numpy import load
+from scipy.interpolate import PchipInterpolator
+
+_TF_ALPHAS = (
+    0.0001,
+    0.0005,
+    0.001,
+    0.002,
+    0.005,
+    0.01,
+    0.02,
+    0.025,
+    0.03,
+    0.04,
+    0.05,
+    0.06,
+    0.075,
+    0.1,
+    0.125,
+    0.15,
+    0.2,
+    0.25,
+)
+_TF_LEVELS = {1 - alpha: alpha for alpha in _TF_ALPHAS}
+
+
+@lru_cache(maxsize=32)
+def _tf_table(alpha: float) -> tuple[float, float, PchipInterpolator]:
+    root = files("linearmodels.iv").joinpath("_tf_data")
+    prefix = f"alpha-{alpha!r}"
+    metadata = json.loads(root.joinpath(prefix + ".json").read_text())
+    if metadata["schema_version"] != 1 or metadata["alpha"] != alpha:
+        raise ValueError("Unsupported or inconsistent tF table metadata.")
+    contents = root.joinpath(prefix + ".npz").read_bytes()
+    if hashlib.sha256(contents).hexdigest() != metadata["npz_sha256"]:
+        raise ValueError("tF table checksum does not match its metadata.")
+    with load(io.BytesIO(contents), allow_pickle=False) as data:
+        spline = PchipInterpolator(data["f"], data["regularized"], extrapolate=False)
+    return metadata["boundary"], metadata["plateau_f"], spline
+
+
+@lru_cache(maxsize=32)
+def _tf_calibration(alpha: float) -> Any:
+    try:
+        package = import_module("tf_inference")
+    except ImportError as exc:
+        raise ImportError(
+            "method='compute' requires the optional tf-inference package. "
+            "Install it from https://github.com/finite-sample/tf-inference."
+        ) from exc
+    return package.calibrate(alpha)
+
+
+def _tf_critical_value(
+    f_stat: float, level: float = 0.95, *, method: str = "table"
+) -> float:
+    """Evaluate a generated tF table or explicitly request numerical calibration."""
+    if not isfinite(f_stat) or f_stat < 0:
+        raise ValueError("The first-stage F statistic must be finite and nonnegative.")
+    if not isfinite(level) or not 0.75 <= level <= 0.9999:
+        raise ValueError("level must be finite and between 0.75 and 0.9999.")
+    # Match confidence levels before subtraction to retain the exact bundled alpha.
+    alpha = _TF_LEVELS.get(level)
+    if method == "compute":
+        return float(
+            _tf_calibration(alpha if alpha is not None else 1 - level).critical_value(
+                f_stat
+            )
+        )
+    if method != "table":
+        raise ValueError("method must be 'table' or 'compute'.")
+    if alpha is None:
+        raise ValueError(
+            f"No bundled tF table for level={level}. "
+            "Use method='compute' with the optional tf-inference package."
+        )
+    boundary, plateau_f, spline = _tf_table(alpha)
+    if f_stat <= boundary:
+        return inf
+    value = min(f_stat, plateau_f)
+    return sqrt(float(spline(value)) / (value - boundary))

linearmodels/iv/_tf_data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Gaurav Sood
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

linearmodels/iv/_tf_data/README.rst

@@ -0,0 +1,32 @@
+Generated tF calibration tables
+===============================
+
+The NPZ and JSON files are copied without modification from
+https://github.com/finite-sample/tf-inference at commit
+``ebaa0f8ff0286203b0a1488abdf3330ace84c4fe`` (version 0.1.0).
+The accompanying LICENSE is that project's MIT license.
+
+Each NPZ stores the F grid and the regularized squared critical value
+``Z(F) = (F - boundary) * critical(F)**2``. The matching JSON records the
+boundary, calibrated plateau, construction settings, numerical size-search
+diagnostics, midpoint interpolation error and SHA-256 checksum. Runtime lookup
+verifies the checksum once, constructs a PCHIP interpolator for Z, and caches it.
+F at or below the boundary returns infinity. Above it, cap F at ``plateau_f``
+and evaluate ``sqrt(Z(F) / (F - boundary))``.
+
+These are generated numerical approximations to Appendix B.3 of Lee, McCrary,
+Moreira and Porter (2022), https://doi.org/10.1257/aer.20211063. They do not use
+the rounded printed tables. Calibration uses a finite nuisance-parameter grid
+and local optimization, not a certified supremum over the unbounded domain.
+Interpolation is not claimed to preserve the printed table's conservative bound.
+
+The source package's ``validation/results.json`` records 4,032 held-out
+rejection-probability checks and comparisons against refined curves. The largest
+observed relative critical-value difference across the 18 levels was 1.76e-7.
+These are numerical checks, not uniform error or coverage guarantees.
+
+To regenerate, check out that source commit, install it, run ``make tables``
+and ``make validate``, then copy its ``src/tf_inference/data/*.npz`` and
+``src/tf_inference/data/*.json`` into this directory. Retain its license and
+update this provenance record. The source package also exports CSV files for
+other software; linearmodels needs only NPZ and JSON.

linearmodels/iv/_tf_data/alpha-0.0001.json

@@ -0,0 +1,82 @@
+{
+  "schema_version": 1,
+  "alpha": 0.0001,
+  "boundary": 15.136705226623398,
+  "epsilon": 0.0001,
+  "seeds": 256,
+  "points": 16385,
+  "plateau_f": 644.9077041789053,
+  "minimum_plateau_f": 644.9077041789053,
+  "property_c_endpoint": 644.9077041789053,
+  "table_relative_error_on_midpoints": 3.709627693915252e-08,
+  "npz_sha256": "8b4bcc9025761afc38e28a8ccccdb90f1a46cf783aab4fff0b610d25dce5b46b",
+  "csv_sha256": "647a218049bc7173324bf7b6f30e5044b718a5b3c2ded8edb928eef77fc0b528",
+  "diagnostics": {
+    "alpha": 0.0001,
+    "plateau_f": 644.9077041789053,
+    "plateau_critical": 4.357098832839683,
+    "property_c_endpoint": 644.9077041789053,
+    "maximum_rejection": 0.00010000000363025216,
+    "worst_strength": 2.0274492775642177,
+    "worst_rho": 1.0,
+    "integration_error": 2.328172135753514e-10,
+    "size_tolerance": 2e-07,
+    "strength_limit": 80,
+    "evaluations": 583,
+    "adjustments": 0,
+    "correlations": [
+      0.0,
+      0.25,
+      0.5,
+      0.75,
+      0.9,
+      0.95,
+      0.99,
+      0.995,
+      0.999,
+      0.9999,
+      1.0
+    ],
+    "strengths": [
+      0.0,
+      0.05,
+      0.1,
+      0.25,
+      0.5,
+      1.0,
+      1.1518342801539727,
+      1.3267222089378203,
+      1.5281641204961829,
+      1.7601918196888497,
+      2.0274492775642177,
+      2.3352855791718725,
+      2.689861984039387,
+      3.0982752420995436,
+      3.5686996332026037,
+      4.110550573095668,
+      4.734673060398148,
+      5.453558736288107,
+      6.281595901289821,
+      7.235357493180306,
+      8.333932789813991,
+      9.599309475806987,
+      11.056813720041351,
+      12.7356170720204,
+      14.66932032246726,
+      16.89662601397712,
+      19.462113061840224,
+      22.417128988859965,
+      25.82081763200227,
+      29.74130289014434,
+      34.25705220531067,
+      39.45844706710108,
+      45.44959197352801,
+      52.350398054120404,
+      60.298983058441706,
+      69.45443574513679,
+      80.0
+    ]
+  },
+  "reference": "https://doi.org/10.1257/aer.20211063",
+  "algorithm": "Appendix B.3; regularized inverse map; Property C; finite size search"
+}