diff --git a/scripts/fit_attention.py b/scripts/fit_attention.py index 62c4cec2..077a4bae 100755 --- a/scripts/fit_attention.py +++ b/scripts/fit_attention.py @@ -46,6 +46,15 @@ history_max, and zero when nothing in the list has any), killer, tt_move, tt_score_only, eval_beta, alpha_gap, generated, searched. +The split is by fen by default. `--group-by` takes a file naming, for each +ledger on the command line, the group its rows belong to, and splits by group +instead, so a ledger recorded one root at a time can hold every row of a root +on one side. Rows from one root are not exchangeable: they share a parent node +and most of a feature vector, and a split that does not respect them reads +back a holdout the fit has half seen. `--drop` leaves a feature out of the fit +and gives the engine a zero for its constant, which is how a weight that is +only identified on a thin slice of the rows is priced. + numpy and nothing else. The fit is Newton's method with an L2 penalty, written out, which is what the tuner beside this does for the same reason. """ @@ -175,11 +184,44 @@ def attention(row): return 1 if row["label"] == "harmful" else 0 +def key_parity(key): + """Which half a key belongs to. The first bit of its sha256, so the same + key lands on the same side every run and no order of the rows moves it.""" + return hashlib.sha256(key.encode("utf-8")).digest()[0] & 1 + + def fen_parity(fen): """Which half a position belongs to. By the fen and not by the row, so that two rows from one position cannot land on opposite sides and let the holdout score a position the fit has already seen.""" - return hashlib.sha256(fen.encode("utf-8")).digest()[0] & 1 + return key_parity(fen) + + +def read_groups(path): + """The group each ledger's rows belong to, as a name to key mapping. + + Two whitespace separated columns, the ledger's file name and its group + key, and anything after a hash is a comment. The ledger row carries the + fen and nothing about the root it was recorded from, so the attribution + has to come from outside: one ledger per root, or per small chunk of + roots, and this file saying which. + """ + groups = {} + for number, line in enumerate( + Path(path).read_text(encoding="utf-8").splitlines(), 1 + ): + text = line.split("#", 1)[0].strip() + if not text: + continue + words = text.split() + if len(words) != 2: + sys.exit(f"{path}:{number}: not a ledger name and a group key: {line}") + name, key = words + if groups.setdefault(name, key) != key: + sys.exit(f"{path}:{number}: {name} is already in group {groups[name]}") + if not groups: + sys.exit(f"{path}: names no group") + return groups def features_of(row): @@ -266,6 +308,70 @@ def auc_of(scores, y): return (ranks[positive].sum() - ones * (ones + 1) / 2) / (ones * zeros) +def bootstrap_auc(scores, y, units, draws=200, seed=20260915): + """The standard error of an auc, by resampling the units the rows came in. + + A row is not the unit when the split is by group: rows from one root move + together, so resampling rows alone would report an error bar for a sample + that was never drawn. `units` is the group each row belongs to, as integer + codes, or a distinct code per row when the rows really are the unit. + + Returns nan when the rows come from fewer than two units, since every draw + is then the same sample and its spread of zero would read as precision, + and when fewer than two draws produce an auc at all, which a holdout with + almost no attention in it can do. + """ + order = np.argsort(units, kind="mergesort") + sorted_units = units[order] + starts = np.searchsorted(sorted_units, np.arange(sorted_units[-1] + 1), "left") + ends = np.searchsorted(sorted_units, np.arange(sorted_units[-1] + 1), "right") + present = np.where(ends > starts)[0] + if len(present) < 2: + return float("nan") + rng = np.random.default_rng(seed) + areas = [] + for _ in range(draws): + drawn = rng.choice(present, size=len(present), replace=True) + rows = np.concatenate([order[starts[one] : ends[one]] for one in drawn]) + area = auc_of(scores[rows], y[rows]) + if not np.isnan(area): + areas.append(area) + if len(areas) < 2: + return float("nan") + return float(np.std(areas, ddof=1)) + + +def print_auc(name, scores, y, units, draws): + """One held-out auc with its bootstrap standard error, or why there is + none: a stratum of one class has no area under its curve.""" + area = auc_of(scores, y) + if np.isnan(area): + print(f" {name:<28} {len(y):>9} rows {int(y.sum()):>7} attention auc none") + return + error = bootstrap_auc(scores, y, units, draws) + print( + f" {name:<28} {len(y):>9} rows {int(y.sum()):>7} attention" + f" auc {area:.4f} +/- {error:.4f}" + ) + + +def print_holdout_aucs(scores, y, depth, units, draws): + """The held-out quality, over every row, over the rows the gate can act on + and one depth at a time. + + The engine's gate is reached at depth four and up, so an auc over every row + is carried by the depth three rows that outnumber them and is not the + number the gate would be chosen on. + """ + print(f"\nholdout auc, bootstrap standard error over {draws} draws:") + print_auc("all rows", scores, y, units, draws) + deep = depth >= 4 + print_auc("gate reachable (depth 4+)", scores[deep], y[deep], units[deep], draws) + for one in sorted(set(depth.tolist())): + at = depth == one + print_auc(f"depth {one}", scores[at], y[at], units[at], draws) + + def rate(values): """The share of `values` that are true, as a percentage, or nan when there are none of them. A rate over no rows is not a zero.""" @@ -352,16 +458,23 @@ def print_trivial_gate(X, y, train, hold): ) -def write_csvs(out_dir, X, y, depth, rows, train, hold, weights, intercept): - """The two halves and the fitted weights, for reading back by hand.""" +def write_csvs(out_dir, X, y, depth, rows, groups, train, hold, weights, intercept): + """The two halves and the fitted weights, for reading back by hand. + + The group goes in the row beside the fen, so a half can be checked + against the map that produced it rather than taken on the script's + word. + """ out_dir.mkdir(parents=True, exist_ok=True) - header = ",".join([*FEATURES, "attention", "depth", "fen"]) + header = ",".join([*FEATURES, "attention", "depth", "group", "fen"]) for mask, name in ((train, "train.csv"), (hold, "holdout.csv")): with (out_dir / name).open("w", encoding="utf-8") as out: out.write(header + "\n") for at in np.where(mask)[0]: cells = ",".join(str(int(value)) for value in X[at]) - out.write(f'{cells},{int(y[at])},{depth[at]},"{rows[at]["fen"]}"\n') + out.write( + f'{cells},{int(y[at])},{depth[at]},{groups[at]},"{rows[at]["fen"]}"\n' + ) with (out_dir / "weights.txt").open("w", encoding="utf-8") as out: out.write("# logistic regression on the reduction ledger\n") out.write(f"# fixed point scale 2^{SHIFT} = {1 << SHIFT}\n") @@ -385,6 +498,27 @@ def main(argv=None): nargs="+", help="what `arche reductions` printed, one file or several", ) + parser.add_argument( + "--group-by", + type=Path, + help="a file naming each ledger and the group its rows belong to," + " which is then what train and holdout are split by", + ) + parser.add_argument( + "--drop", + action="append", + default=[], + choices=FEATURES, + help="leave a feature out of the fit, and out of the engine's" + " constants, which can be given more than once", + ) + parser.add_argument( + "--bootstrap", + type=int, + default=200, + help="how many resamples the held-out auc's standard error is taken" + " over, resampling groups where there are groups", + ) parser.add_argument( "--out-dir", type=Path, @@ -392,12 +526,18 @@ def main(argv=None): ) args = parser.parse_args(argv) + named = read_groups(args.group_by) if args.group_by else None rows = [] + groups = [] for path in args.ledger: header, read, skipped = parse_file(path) print(f"{path.name}: {header}") print(f" {len(read)} scouts, {skipped} skipped rows left out") rows.extend(read) + if named is not None: + if path.name not in named: + sys.exit(f"{args.group_by} names no group for {path.name}") + groups.extend([named[path.name]] * len(read)) if not rows: sys.exit("no scouts in the ledger, nothing to fit") @@ -408,38 +548,60 @@ def main(argv=None): seen[0] += 1 seen[1] += row["scout"] == "high" seen[2] += attention(row) - for depth in sorted(depths): - rows_at, high, attention_at = depths[depth] - print(f" depth {depth}: {rows_at} rows, {high} high, {attention_at} attention") + for depth_at in sorted(depths): + rows_at, high, attention_at = depths[depth_at] + print( + f" depth {depth_at}: {rows_at} rows, {high} high, {attention_at} attention" + ) X = np.array([features_of(row) for row in rows], dtype=float) y = np.array([attention(row) for row in rows], dtype=float) depth = np.array([row["depth"] for row in rows]) - parity = np.array([fen_parity(row["fen"]) for row in rows]) + if named is not None: + keys = groups + parity = np.array([key_parity(key) for key in groups]) + print(f"\nsplit by group: {len(set(groups))} groups over {len(rows)} rows") + else: + keys = [row["fen"] for row in rows] + parity = np.array([fen_parity(row["fen"]) for row in rows]) + print("\nsplit by fen") train = parity == 0 hold = parity == 1 print( - f"\nsplit by fen: train {train.sum()} rows" - f" ({int(y[train].sum())} attention)," + f"train {train.sum()} rows ({int(y[train].sum())} attention)," f" holdout {hold.sum()} rows ({int(y[hold].sum())} attention)" ) if not train.any() or not hold.any(): sys.exit("one half of the split is empty, the ledger is too small to fit") - weights, intercept = fit_logistic(X[train], y[train]) + # the unit the bootstrap resamples is the unit the split was made by: the + # group when there is one and the fen otherwise, since rows of one fen + # were kept on one side and have to be drawn together too + codes = {} + units = np.array( + [codes.setdefault(keys[at], len(codes)) for at in np.where(hold)[0]] + ) + + kept = [at for at, name in enumerate(FEATURES) if name not in args.drop] + if not kept: + sys.exit("every feature was dropped, there is nothing to fit") + if args.drop: + print(f"dropped from the fit: {', '.join(sorted(set(args.drop)))}") + fitted, intercept = fit_logistic(X[train][:, kept], y[train]) + weights = np.zeros(len(FEATURES)) + weights[kept] = fitted fixed = quantize(weights) fixed_intercept = quantize([intercept])[0] print(f"\nweights, fixed point at a scale of 2^{SHIFT} = {1 << SHIFT}:") - for name, raw, one in zip(FEATURES, weights, fixed): - print(f" {CONSTANTS[name]:<26} {one:>8d} ({raw:.6f})") + for at, (name, raw, one) in enumerate(zip(FEATURES, weights, fixed)): + note = "dropped" if at not in kept else f"{raw:.6f}" + print(f" {CONSTANTS[name]:<26} {one:>8d} ({note})") print(f" {CONSTANTS['intercept']:<26} {fixed_intercept:>8d} ({intercept:.6f})") scores = (X.astype(np.int64) @ fixed + fixed_intercept).astype(float) - print( - f"\nholdout auc: {auc_of(scores[hold], y[hold]):.4f}" - f" train auc: {auc_of(scores[train], y[train]):.4f}" - ) + print(f"\ntrain auc: {auc_of(scores[train], y[train]):.4f}") print(f"holdout attention rate over every row: {rate(y[hold]):.3f}%") + print_holdout_aucs(scores[hold], y[hold], depth[hold], units, args.bootstrap) deep = depth[hold] >= 4 print_table( @@ -449,7 +611,9 @@ def main(argv=None): print_trivial_gate(X, y, train, hold) if args.out_dir: - write_csvs(args.out_dir, X, y, depth, rows, train, hold, weights, intercept) + write_csvs( + args.out_dir, X, y, depth, rows, keys, train, hold, weights, intercept + ) return 0 diff --git a/scripts/tests/test_fit_attention.py b/scripts/tests/test_fit_attention.py index 27080637..e1c2b193 100644 --- a/scripts/tests/test_fit_attention.py +++ b/scripts/tests/test_fit_attention.py @@ -16,6 +16,10 @@ data looks like: the rate is under a percent. So the sample covers the parse and the run, and the planted case below covers the fit, because a fit on one positive row says nothing about whether the arithmetic is right. + +The grouped split is tested on planted rows too. What it has to hold is that a +group goes to one side whole, since the leak it exists to close is a holdout +row sharing a parent with a training row. """ import subprocess @@ -29,9 +33,10 @@ SCRIPT = SCRIPTS / "fit_attention.py" SAMPLE = Path(__file__).resolve().parent / "reductions_sample.txt" +NL = chr(10) -def row(index, attention, fen_id, depth=6, history=0, history_max=0): +def row(index, attention, fen_id, depth=6, history=0, history_max=0, alpha_gap=19): """One ledger row, in the eighteen fields reduction.rs prints. A fail high is a row worth attention on its own and has no label to give, @@ -40,7 +45,7 @@ def row(index, attention, fen_id, depth=6, history=0, history_max=0): scout, label = ("high", "-") if attention else ("low", "harmless") return ( f"{depth} zw {index} {index + 1} 30 {history} {history_max} plain miss" - f" -20 19 -120 {scout} 139 6 {label} 1" + f" -20 {alpha_gap} -120 {scout} 139 6 {label} 1" f" 8/8/8/8/8/8/8/K6k w - - 0 {fen_id}" ) @@ -179,6 +184,137 @@ def test_one_class_alone_has_no_area_under_its_curve(self): assert np.isnan(alone) +class TestGrouping: + def map_of(self, tmp_path, text): + path = tmp_path / "map.txt" + path.write_text(text) + return path + + def two_ledgers(self, tmp_path): + for name, first in (("a.txt", 0), ("b.txt", 1000)): + rows = [row(at % 20, at % 20 < 4, first + at) for at in range(200)] + (tmp_path / name).write_text(ledger(rows)) + return [str(tmp_path / "a.txt"), str(tmp_path / "b.txt")] + + def test_a_map_reads_its_two_columns_and_skips_its_comments(self, tmp_path): + path = self.map_of( + tmp_path, + "# where these came from" + NL + NL + "a.txt g1" + NL + "b.txt g2" + NL, + ) + assert fit_attention.read_groups(path) == {"a.txt": "g1", "b.txt": "g2"} + + def test_a_ledger_in_two_groups_stops_the_run(self, tmp_path): + # a ledger whose rows are claimed by two groups has no group, and + # guessing one is what the grouped split exists to stop + path = self.map_of(tmp_path, "a.txt g1" + NL + "a.txt g2" + NL) + with pytest.raises(SystemExit) as raised: + fit_attention.read_groups(path) + assert "already in group g1" in str(raised.value) + + def test_a_line_that_is_not_a_name_and_a_key_stops_the_run(self, tmp_path): + path = self.map_of(tmp_path, "a.txt" + NL) + with pytest.raises(SystemExit): + fit_attention.read_groups(path) + + def test_a_ledger_with_no_group_named_stops_the_run(self, tmp_path): + ledgers = self.two_ledgers(tmp_path) + path = self.map_of(tmp_path, "a.txt g1" + NL) + done = run(*ledgers, "--group-by", str(path)) + assert done.returncode != 0 + assert "names no group for b.txt" in done.stderr + + def test_a_group_lands_on_one_side_whole(self, tmp_path): + # the two ledgers hold different positions, so a split by fen puts + # rows of both in both halves and a split by group cannot + ledgers = self.two_ledgers(tmp_path) + by_fen = run(*ledgers, "--bootstrap", "10") + assert by_fen.returncode == 0, by_fen.stderr + assert "train 200 rows" not in by_fen.stdout + # "one" and "two" hash to opposite sides, which is the split + path = self.map_of(tmp_path, "a.txt one" + NL + "b.txt two" + NL) + done = run(*ledgers, "--group-by", str(path), "--bootstrap", "10") + assert done.returncode == 0, done.stderr + assert "split by group: 2 groups over 400 rows" in done.stdout + assert "train 200 rows" in done.stdout + assert "holdout 200 rows" in done.stdout + + def test_the_key_and_not_the_row_order_decides_the_side(self): + assert fit_attention.key_parity("g1") == fit_attention.key_parity("g1") + assert fit_attention.fen_parity("8/8/8/8/8/8/8/K6k w - - 0 1") == ( + fit_attention.key_parity("8/8/8/8/8/8/8/K6k w - - 0 1") + ) + + +class TestDropping: + def test_a_dropped_feature_gets_a_zero_and_is_said_to_be_dropped(self, tmp_path): + # the engine reads a constant for every feature, so a feature left out + # of the fit has to arrive as a zero rather than be missing + rows = [row(at % 20, at % 20 < 4, at) for at in range(400)] + path = tmp_path / "ledger.txt" + path.write_text(ledger(rows)) + done = run(str(path), "--drop", "alpha_gap", "--bootstrap", "10") + assert done.returncode == 0, done.stderr + assert "dropped from the fit: alpha_gap" in done.stdout + assert "ATTENTION_ALPHA_GAP 0 (dropped)" in done.stdout + + def test_a_dropped_feature_is_left_out_of_the_fit_and_not_only_the_print( + self, tmp_path + ): + # every column is constant but alpha_gap, which alone carries the + # attention. Kept, it orders the holdout; dropped, nothing is left to + # order it by and every score ties + rows = [ + row(5, at % 5 == 0, at, alpha_gap=200 if at % 5 == 0 else 19) + for at in range(400) + ] + path = tmp_path / "ledger.txt" + path.write_text(ledger(rows)) + + def holdout_auc(*extra): + done = run(str(path), "--bootstrap", "10", *extra) + assert done.returncode == 0, done.stderr + line = next(one for one in done.stdout.splitlines() if "all rows" in one) + return float(line.split("auc ")[1].split()[0]) + + assert holdout_auc() > 0.9 + assert holdout_auc("--drop", "alpha_gap") == 0.5 + + def test_a_feature_that_is_not_one_is_refused(self, tmp_path): + rows = [row(at % 20, at % 20 < 4, at) for at in range(40)] + path = tmp_path / "ledger.txt" + path.write_text(ledger(rows)) + done = run(str(path), "--drop", "mobility") + assert done.returncode != 0 + + +class TestBootstrap: + def test_the_error_is_taken_over_the_groups_and_not_the_rows(self): + # twenty groups of twenty rows, each group separating its two + # classes by its own amount. Resampling rows averages that spread away + # and reports an error bar for a sample nobody drew. Resampling groups + # carries it, so the group error has to be the larger + rng = np.random.default_rng(3) + units, scores, y = [], [], [] + for group in range(20): + apart = rng.uniform(-2.0, 6.0) + labels = np.tile([0.0, 1.0], 10) + units += [group] * 20 + y += list(labels) + scores += list(labels * apart + rng.normal(0, 1, 20)) + units = np.array(units) + scores = np.array(scores) + y = np.array(y) + by_row = fit_attention.bootstrap_auc(scores, y, np.arange(len(y)), draws=100) + by_group = fit_attention.bootstrap_auc(scores, y, units, draws=100) + assert by_group > 2 * by_row + + def test_one_class_in_every_draw_has_no_error(self): + alone = fit_attention.bootstrap_auc( + np.arange(10.0), np.zeros(10), np.arange(10), draws=5 + ) + assert np.isnan(alone) + + class TestCommandLine: def test_the_sample_runs_and_prints_the_weights_and_the_table(self): done = run(str(SAMPLE))