From 104f2abf4fc99e6f335169761636ea2267c17ec3 Mon Sep 17 00:00:00 2001 From: blacktoe Date: Sun, 13 Sep 2026 22:02:32 +1000 Subject: [PATCH] refactor(eval): Move each leaf term into a module of its own under eval/ eval.rs held three terms and everything they share, and board.rs held the counts all three are read from. So a term was four places: its counts and masks in board.rs, its weights and fold in eval.rs, its slot arithmetic in tune.rs and the same arithmetic again in scripts/tune.py. Adding one meant editing all four and getting them to agree, and nothing said when they did not. eval.rs becomes eval/, with a file per term. mobility.rs, shelter.rs and pawn_structure.rs each own their counts, their masks, their weights with the fit that produced them, their fold, their memo where they have one, and the tests that pin them. The doc blocks explaining each fit move whole. mod.rs keeps the material values, the phase weights, the accumulator and the sum. The slot layout follows from one table. TERMS in eval/mod.rs is a descriptor per term: its name, how many counts it is measured in per side and per half of the taper, its weights and its counts. tune.rs reads its slot arithmetic off that, so SLOTS is a const fn over the table, weight() walks it instead of a nine branch ladder, and the three loops in Terms::of become one. The run prints the result as a second header line, `layout midgame 384 endgame 384 material 6 mobility 4 shelter 7 pawn_structure 8`, and scripts/tune.py derives its slots from that line rather than from ten constants of its own. Its five guards against earlier layouts go with them: a run this file cannot fit is now refused by the name of the term it does not price, which is the case those lengths were standing in for. A header with no layout line is refused too. The search carries one eval::Caches rather than two fields, and the sum is written once over a private memo trait with two implementations, the caches and a unit that recomputes. eval and eval_cached are the two doors on to it. board.rs opens pub(crate) readers on the six piece boards and the king's square, gains sides(color) for the (ours, theirs) match that was written four times, and names no evaluation term at all. Four judgment calls. files_of lives in pawn_structure.rs beside spread, which is its inverse, and shelter.rs reads it from there; the alternative was a bitboard helper in mod.rs, which is not what mod.rs is for. shelter_key moved with the term as shelter::key, which is what leaves board.rs naming no term, and it needed ZOBRIST to go pub(crate). The mobility count reads the knight table through a new knight_attacks reader rather than the whole attack mask table being opened. And the pawn span test stayed in board.rs, since what it pins is a board helper against the generator's own masks rather than any count. Node counts are identical by construction: the arithmetic is the same arithmetic in a different place. The bench is 3868987 at depth 7, both pinned node count tables are unchanged, and so are EXPECTED_PASSES and EXPECTED_POINTS. The release run goes from 457 tests to 460 and the python ones from 210 to 213; the three new rust tests say that every term writes the counts its width claims, that the table names the terms the sum adds and that the run prints the layout its slots are laid out from. Two of the three new python ones refuse a header with no layout line and a term with no bound; the third keeps a row whose id opens with the layout line's own word. Bench: 3868987 Co-Authored-By: Claude Fable 5.1 --- arche-core/src/board.rs | 1172 +------------------ arche-core/src/engine.rs | 21 +- arche-core/src/eval.rs | 1543 ------------------------- arche-core/src/eval/mobility.rs | 460 ++++++++ arche-core/src/eval/mod.rs | 850 ++++++++++++++ arche-core/src/eval/pawn_structure.rs | 869 ++++++++++++++ arche-core/src/eval/shelter.rs | 700 +++++++++++ arche-core/src/tune.rs | 287 +++-- docs/ARCHITECTURE.md | 45 +- docs/INSTRUMENTS.md | 20 +- scripts/tests/test_tune.py | 327 ++++-- scripts/tune.py | 391 ++++--- 12 files changed, 3584 insertions(+), 3101 deletions(-) delete mode 100644 arche-core/src/eval.rs create mode 100644 arche-core/src/eval/mobility.rs create mode 100644 arche-core/src/eval/mod.rs create mode 100644 arche-core/src/eval/pawn_structure.rs create mode 100644 arche-core/src/eval/shelter.rs diff --git a/arche-core/src/board.rs b/arche-core/src/board.rs index 4bdac51f..709aff33 100644 --- a/arche-core/src/board.rs +++ b/arche-core/src/board.rs @@ -78,7 +78,7 @@ impl Building { /// Pop the lowest set bit and return its index. #[inline(always)] -fn pop_lsb(bb: &mut u64) -> u8 { +pub(crate) fn pop_lsb(bb: &mut u64) -> u8 { let i = bb.trailing_zeros() as u8; *bb &= *bb - 1; i @@ -155,7 +155,7 @@ const H8: u8 = 63; /// naming which half of the board it is confined to. const LIGHT_SQUARES: u64 = 0x55AA_55AA_55AA_55AA; -static ZOBRIST: Zobrist = Zobrist::TABLE; +pub(crate) static ZOBRIST: Zobrist = Zobrist::TABLE; /// What each square leaves of the castling rights, as the four bytes /// `CastlePermissions` is laid out in. @@ -230,8 +230,6 @@ const fn castle_masks(leaving: bool) -> [u32; 64] { const NULL_HISTORY_SALT: u64 = 0x9e37_79b9_7f4a_7c15; static ATTACK_MASKS: AttackMasks = AttackMasks::new(); -static SHELTER_MASKS: ShelterMasks = ShelterMasks::new(); -static PAWN_MASKS: PawnMasks = PawnMasks::new(); // the squares strictly between two aligned squares, and empty for a pair // that shares no line. What a piece must land on to block a slider on one // square checking a king on the other. @@ -409,6 +407,16 @@ impl AttackMasks { } } +/// The squares a knight on `from` attacks, whatever stands on them. +/// +/// The one entry of the attack masks anything outside this file reads. +/// A reader is opened on it because the evaluation asks what a knight +/// covers; the rest of the table is the generator's own and stays here. +#[inline] +pub(crate) fn knight_attacks(from: u8) -> u64 { + ATTACK_MASKS.knights[from as usize] +} + /// Every square a side's pawns attack, as one span. /// /// A shift rather than a mask a pawn at a time, because the one caller wants @@ -417,7 +425,7 @@ impl AttackMasks { /// down it; a pawn on the a file has no capture to its left and one on the h /// file none to its right, which is what the two masks drop before the shift /// carries a bit around into the next rank. -const fn pawn_attacks(pawns: u64, color: Color) -> u64 { +pub(crate) const fn pawn_attacks(pawns: u64, color: Color) -> u64 { const A_FILE: u64 = 0x0101_0101_0101_0101; const H_FILE: u64 = 0x8080_8080_8080_8080; match color { @@ -426,197 +434,6 @@ const fn pawn_attacks(pawns: u64, color: Color) -> u64 { } } -/// The three files a king on `square` stands behind, as a bit per file. -/// -/// A king on the a file or the h file is read against three files rather than -/// two, by stepping the middle one in: the b file and the g file are the -/// centres a corner king keeps. So every king square names three files and -/// the counts off them are on one scale wherever the king stands. -const fn king_files(square: u8) -> u8 { - let file = square % 8; - let centre = match file { - 0 => 1, - 7 => 6, - file => file, - }; - 0b111 << (centre - 1) -} - -/// The three squares `ahead` ranks in front of a king on `square`, on the -/// files [`king_files`] names, and empty where that rank is off the board. -/// -/// `forward` is the direction the side's pawns push, so a white king is read -/// up the board and a black king down it. A king that has walked far enough -/// up is left with nothing in front of it, which is the answer rather than a -/// case to rule out: a king off its own back ranks has no shelter, and what -/// that is worth is for the weights to say. -const fn shelter_rank(square: u8, forward: i8, ahead: i8) -> u64 { - let rank = (square / 8) as i8 + forward * ahead; - if rank < 0 || rank > 7 { - return 0; - } - // a file bit and a square index share their low three bits, so the byte - // shifted to the rank is the three squares on it - (king_files(square) as u64) << (rank * 8) -} - -/// How many ranks in front of the king are masked. Three, which is as far as -/// an enemy pawn is counted: for a king at home that is the rank its own pawns -/// start on and the two beyond it. -const RANKS_AHEAD: usize = 3; - -/// The squares in front of each king, and which files it stands behind. -/// -/// `ahead` is indexed by how many ranks forward, then by `Color`'s -/// discriminant the way the accumulator's material is, then by the king's -/// square. A white king is read up the board and a black king down it, and the -/// files are the same either way. -/// -/// The same three masks serve both sides of the term. Read against this side's -/// pawns they are the cover the king has, and against the other side's they -/// are the pawns coming for it. -struct ShelterMasks { - ahead: [[[u64; 64]; 2]; RANKS_AHEAD], - files: [u8; 64], -} - -impl ShelterMasks { - /// Built at compile time, the way `AttackMasks` is. - const fn new() -> Self { - let mut masks = ShelterMasks { - ahead: [[[0; 64]; 2]; RANKS_AHEAD], - files: [0; 64], - }; - let mut square = 0u8; - while square < 64 { - let i = square as usize; - masks.files[i] = king_files(square); - let mut rank = 0; - while rank < RANKS_AHEAD { - let ahead = rank as i8 + 1; - masks.ahead[rank][Color::White as usize][i] = shelter_rank(square, 1, ahead); - masks.ahead[rank][Color::Black as usize][i] = shelter_rank(square, -1, ahead); - rank += 1; - } - square += 1; - } - masks - } -} - -/// A pawn's own file and the files beside it, as a bit per file. -/// -/// [`king_files`] steps the middle file in at the two edges so that a king -/// always names three; this does not, because the two are asking different -/// questions. A white pawn on a4 is passed while black has no pawn on the a -/// file or the b file, and a black pawn on the c file has nothing to say -/// about it. Stepping in would let that pawn stop it. -const fn pawn_files(square: u8) -> u8 { - let own = 1u8 << (square % 8); - // a shift off either end of the byte drops the bit, which is the edge - // case: the a file has no file to its left and the h file none to its - // right - own | (own << 1) | (own >> 1) -} - -/// The squares on `files` on every rank strictly ahead of `square`, ahead -/// meaning the direction `forward` pushes. -const fn span_ahead(square: u8, forward: i8, files: u8) -> u64 { - let mut mask = 0; - let mut rank = (square / 8) as i8 + forward; - while rank >= 0 && rank < 8 { - mask |= (files as u64) << (rank * 8); - rank += forward; - } - mask -} - -/// What stands in a pawn's way, as two masks a square names. -/// -/// `front_span` is the pawn's file and the two beside it, on every rank ahead -/// of it. A pawn of ours is passed when no pawn of theirs stands anywhere in -/// it, which is the whole of the standard definition bar one clause. -/// `file_ahead` is the same span without the neighbouring files, and it -/// answers that clause: whether a pawn of ours is already in front of this -/// one. It is kept as its own table rather than masked out of the first at -/// the leaf, because a leaf term pays for every instruction it adds and a -/// table costs half a kilobyte. -/// -/// Indexed by `Color`'s discriminant and then the square, the way -/// `ShelterMasks` is indexed and for the same reason. A white pawn is read up -/// the board and a black one down it. -struct PawnMasks { - front_span: [[u64; 64]; 2], - file_ahead: [[u64; 64]; 2], -} - -impl PawnMasks { - /// Built at compile time, the way `ShelterMasks` is. - const fn new() -> Self { - let mut masks = PawnMasks { - front_span: [[0; 64]; 2], - file_ahead: [[0; 64]; 2], - }; - let mut square = 0u8; - while square < 64 { - let i = square as usize; - let own = 1u8 << (square % 8); - let beside = pawn_files(square); - masks.front_span[Color::White as usize][i] = span_ahead(square, 1, beside); - masks.front_span[Color::Black as usize][i] = span_ahead(square, -1, beside); - masks.file_ahead[Color::White as usize][i] = span_ahead(square, 1, own); - masks.file_ahead[Color::Black as usize][i] = span_ahead(square, -1, own); - square += 1; - } - masks - } -} - -/// A set of files put back on the board: every square on every file the byte -/// names. What [`files_of`] undoes, and one multiply rather than eight -/// shifts, since a byte times the a file's eight squares lands a copy of the -/// byte on each rank. -const fn spread(files: u8) -> u64 { - (files as u64) * 0x0101_0101_0101_0101 -} - -/// Every square strictly in front of one of `pawns`, on that pawn's own file. -/// -/// The pawns shifted one rank on and then doubled three times, which carries -/// them the seven ranks a board has. Seeded with the shift rather than with -/// the pawns, so a pawn is never in its own fill, which is what leaves a lone -/// pawn undoubled. -const fn ahead_of(pawns: u64, color: Color) -> u64 { - match color { - Color::White => { - let mut filled = pawns << 8; - filled |= filled << 8; - filled |= filled << 16; - filled |= filled << 32; - filled - } - Color::Black => { - let mut filled = pawns >> 8; - filled |= filled >> 8; - filled |= filled >> 16; - filled |= filled >> 32; - filled - } - } -} - -/// Which files a set of pawns stands on, as a bit per file. -/// -/// The board folded in half three times, so the answer is three shifts, three -/// ors and a narrowing rather than eight masked tests. Nothing here says how -/// many pawns a file holds, which is all the two file counts want to know. -const fn files_of(pawns: u64) -> u8 { - let folded = pawns | (pawns >> 32); - let folded = folded | (folded >> 16); - let folded = folded | (folded >> 8); - folded as u8 -} - /// What each piece is worth to `see`, indexed by `Piece`. An ordering /// oracle, not the evaluation: these say which capture to try first, and /// `eval::material` says what a position is worth, so either can move @@ -695,10 +512,10 @@ pub struct Board { pub(crate) key: u64, /// The same kind of key over the pawns alone: both sides' pawns and the /// squares they stand on, and nothing else. Two positions with the same - /// pawns and different pieces share it, which is what a table keyed by - /// pawn structure wants. It carries no side to move, no castle rights - /// and no en passant square, so it says what the pawns are and not whose - /// turn it is. + /// pawns and different pieces share it, which is what anything + /// remembered under the pawns alone asks of a key. It carries no side to + /// move, no castle rights and no en passant square, so it says what the + /// pawns are and not whose turn it is. /// /// Kept in step beside the position key, and so by every path that moves /// a pawn or takes one off: `relocate_piece_index` for a push and for @@ -938,10 +755,7 @@ impl Board { /// the quiet-only sections, with nothing tested per move. fn generate(&self) -> MoveList { let mut moves = Building::new(); - let (color_mask, capture_mask) = match self.active_color { - Color::Black => (self.black, self.white), - Color::White => (self.white, self.black), - }; + let (color_mask, capture_mask) = self.sides(self.active_color); let all_pieces = self.black | self.white; let attack_masks = &ATTACK_MASKS; let magic = &MAGIC; @@ -1345,92 +1159,6 @@ impl Board { attackers } - /// How many squares this side's knights, bishops, rooks and queens cover, - /// a count per piece kind in that order, which is the order - /// `eval::MOBILE_PIECES` names them in. `KINDS` says which of the four to - /// count; the rest are not looked at and answer zero. - /// - /// A piece's count is its attack set over the real occupancy, less the - /// squares this side stands on, less the squares an enemy pawn attacks. So - /// a friendly piece blocks a slider rather than being seen through, and a - /// square an enemy pawn covers is not somewhere a piece goes. An enemy - /// piece standing on a square keeps that square in the count, because - /// attacking it is the point. Pins are ignored: a pinned bishop counts its - /// squares, and the search is what knows it cannot move. - /// - /// The enemy pawns are taken as one span rather than probed a square at a - /// time. The king and the pawn have no count of their own: a king's is a - /// danger signal rather than a scope one, and a pawn's is move generation. - /// - /// Both `eval` and the tuner's walk read this, and they may ask for - /// different kinds. The walk always asks for all four, because it is - /// offline and its coefficients are what prices a kind; `eval` asks for - /// the kinds whose weight is not zero, because a count multiplied by zero - /// is not worth the leaf it is taken at. Since the refit priced all four - /// the two sets are equal today. What keeps them honest whether or not - /// they are is that the difference is exactly the zero weights, which - /// `eval_counts_a_kind_exactly_when_its_weight_is_not_zero` pins. Within - /// one set of kinds the counts are still one answer rather than two, so a - /// second implementation of them would still be two chances to be wrong - /// rather than a check on one, and the hand counts below are what pins - /// them. That is the exception to the rule `tune.rs` states in its header, - /// which names it. - /// - /// `KINDS` is a compile time set, so a kind left out of it costs nothing: - /// its loop is not compiled rather than skipped. - /// - /// Inlined by force. Left to itself llvm keeps this out of line even under - /// link time optimisation, and `eval` asks for it twice at every leaf and - /// every quiescence node. That call was three fifths of what the term cost - /// over the bench: 4.30 billion instructions without the attribute against - /// 3.76 billion with it. - #[inline(always)] - pub(crate) fn mobility_counts( - &self, - color: Color, - ) -> [i32; eval::MOBILE_PIECES.len()] { - let occupied = self.occupied(); - let (ours, theirs) = match color { - Color::White => (self.white, self.black), - Color::Black => (self.black, self.white), - }; - let scope = !(ours | pawn_attacks(self.pawns() & theirs, !color)); - let attack_masks = &ATTACK_MASKS; - let magic = &MAGIC; - let mut counts = [0; 4]; - if eval::counted(KINDS, 0) { - let mut knights = self.knights() & ours; - while knights != 0 { - let from = pop_lsb(&mut knights); - counts[0] += (attack_masks.knights[from as usize] & scope).count_ones() as i32; - } - } - if eval::counted(KINDS, 1) { - let mut bishops = self.bishops() & ours; - while bishops != 0 { - let from = pop_lsb(&mut bishops); - counts[1] += (magic.get_diagonal_move(from, occupied) & scope).count_ones() as i32; - } - } - if eval::counted(KINDS, 2) { - let mut rooks = self.rooks() & ours; - while rooks != 0 { - let from = pop_lsb(&mut rooks); - counts[2] += (magic.get_straight_move(from, occupied) & scope).count_ones() as i32; - } - } - if eval::counted(KINDS, 3) { - let mut queens = self.queens() & ours; - while queens != 0 { - let from = pop_lsb(&mut queens); - let attacks = magic.get_straight_move(from, occupied) - | magic.get_diagonal_move(from, occupied); - counts[3] += (attacks & scope).count_ones() as i32; - } - } - counts - } - /// The least valuable piece of `set`: the bit of one such piece and what /// it is. `set` is a subset of one side's pieces. fn least_valuable(&self, set: u64) -> Option<(u64, Piece)> { @@ -2022,213 +1750,55 @@ impl Board { /// The six piece boards by name. Each is a constant index into `pieces`, /// so these read as the fields they replaced and compile to the same load. #[inline] - fn pawns(&self) -> u64 { + pub(crate) fn pawns(&self) -> u64 { self.pieces[Piece::Pawn as usize] } #[inline] - fn knights(&self) -> u64 { + pub(crate) fn knights(&self) -> u64 { self.pieces[Piece::Knight as usize] } #[inline] - fn bishops(&self) -> u64 { + pub(crate) fn bishops(&self) -> u64 { self.pieces[Piece::Bishop as usize] } #[inline] - fn rooks(&self) -> u64 { + pub(crate) fn rooks(&self) -> u64 { self.pieces[Piece::Rook as usize] } #[inline] - fn queens(&self) -> u64 { + pub(crate) fn queens(&self) -> u64 { self.pieces[Piece::Queen as usize] } #[inline] - fn kings(&self) -> u64 { + pub(crate) fn kings(&self) -> u64 { self.pieces[Piece::King as usize] } /// Where this side's king stands. Every board has exactly one king a side, /// which is what `from_fen` checks for: without a king this returns 64 and /// the attack masks are indexed off the end. - fn king_index(&self, color: Color) -> u8 { - let mask = match color { - Color::White => self.white, - Color::Black => self.black, - }; - (self.kings() & mask).trailing_zeros() as u8 + pub(crate) fn king_index(&self, color: Color) -> u8 { + let (ours, _) = self.sides(color); + (self.kings() & ours).trailing_zeros() as u8 } - /// What the king shelter depends on and nothing else: both sides' pawns - /// and both kings' squares. - /// - /// `shelter_counts` reads the pawn boards and the two king squares and - /// nothing else, so two positions whose pawns and kings agree agree on - /// every count of it whatever else has moved. This key stands in for that - /// agreement rather than being it: two positions can share it and differ, - /// which takes a collision across the whole sixty four bits. The pawn key - /// already hashes the pawns of both colours and is kept in step move by - /// move, so this is that key with the two kings folded in, from the same - /// zobrist table the position key uses. - /// - /// Composed here rather than maintained beside `pawn_key`, because a - /// king move would then have to write it and the cost of this is two - /// loads and two xors at the one place that asks. - #[inline] - pub(crate) fn shelter_key(&self) -> u64 { - self.pawn_key - ^ ZOBRIST.get_piece_key(self.king_index(Color::White), Piece::King, Color::White) - ^ ZOBRIST.get_piece_key(self.king_index(Color::Black), Piece::King, Color::Black) - } - - /// What stands between this side's king and the board, as seven counts in - /// the order `eval::SHELTER_TERMS` names them: this side's pawns one rank - /// in front of the king and two ranks in front, how many of the king's - /// three files hold no pawn of either colour, how many hold an enemy pawn - /// and none of this side's, and then the enemy pawns one, two and three - /// ranks in front of the king. - /// - /// All seven are read off the three files the king stands behind, which - /// `king_files` steps in at the two corners so that the counts mean the - /// same thing on every square. Pawns and nothing else: a piece in front of - /// the king shelters it too, but a term that pays for one pays a piece to - /// sit still, and the piece square tables already hold an opinion about - /// where a piece belongs. A pawn is the part of the cover the king cannot - /// get back. - /// - /// The last three are the storm, and they are the same masks read against - /// the other side's pawns. A pawn of ours on g3 is cover and a pawn of - /// theirs on g3 is not the absence of cover, it is a lever, so the two are - /// counted apart and each rank apart from the next: how far the storm has - /// come is most of what it is worth, and the weights are where that is - /// said. Three ranks is as far as it is followed, which for a king at home - /// reaches the fourth rank. - /// - /// The two file counts overlap the two pawn counts, since a file with no - /// pawn of ours on it adds nothing to either of those. They are kept apart - /// because they are different knowledge: a missing g pawn and a g pawn - /// pushed to g4 both leave the near count short, and only the first opens - /// the file to a rook. - /// - /// Nothing here is gated on the king standing at home. A king that has - /// walked up the board has no rank in front of it inside the masks and - /// counts nothing, so the term fades rather than falling off a cliff the - /// search could step over. - /// - /// Both `eval` and the tuner's walk read this, so the identity between - /// them cannot see a wrong count here. What pins it is the hand counts - /// beside this in the tests, the way the mobility counts are pinned. - #[inline] - pub(crate) fn shelter_counts(&self, color: Color) -> [i32; eval::SHELTER_TERMS] { - let masks = &SHELTER_MASKS; - let square = self.king_index(color) as usize; - let side = color as usize; - let (ours, theirs) = match color { - Color::White => (self.white, self.black), - Color::Black => (self.black, self.white), - }; - let pawns = self.pawns(); - let (our_pawns, their_pawns) = (pawns & ours, pawns & theirs); - let ahead = - |pawns: u64, rank: usize| (pawns & masks.ahead[rank][side][square]).count_ones() as i32; - // the king's files with no pawn of ours on them, split by whether the - // other side has one there - let files = masks.files[square]; - let bare = files & !files_of(our_pawns); - let theirs_on = files_of(their_pawns); - let open = (bare & !theirs_on).count_ones() as i32; - let half_open = (bare & theirs_on).count_ones() as i32; - [ - ahead(our_pawns, 0), - ahead(our_pawns, 1), - open, - half_open, - ahead(their_pawns, 0), - ahead(their_pawns, 1), - ahead(their_pawns, 2), - ] - } - - /// What this side's pawns stand as, in the eight counts - /// `eval::PAWN_TERMS` names: its passed pawns by relative rank, the - /// second through the seventh, then its isolated pawns and its doubled - /// ones. - /// - /// A pawn of ours is passed when no pawn of theirs stands on its file or - /// either file beside it on any rank ahead of it, and no pawn of ours - /// stands ahead of it on its own file. The second clause is what leaves - /// the rear of a doubled pair out: the front pawn is the runner, and the - /// one behind it is going nowhere the front one has not gone first. What - /// stands on the square in front of the pawn is not read, so a passer a - /// knight has blockaded is counted as a passer. That is on purpose and it - /// is the first thing this term leaves out: the stop square reads the - /// pieces, and a term that reads the pieces cannot sit behind a key over - /// the pawns. - /// - /// A pawn is isolated when no pawn of ours stands on either file beside - /// it, and doubled when a pawn of ours stands behind it on its own file. - /// Both are counted per pawn rather than per file, so an isolated pair on - /// one file pays the isolated weight twice and a tripled file is doubled - /// two. Per pawn is what one coefficient can state; per file would want a - /// second table to say how many. - /// - /// Relative rank is the rank a pawn has come, so a white pawn's is its - /// rank and a black pawn's is nine less. The relative second is a real - /// bucket and not a rounding of the others: a pawn still at home is - /// passed the moment the enemy pawns on its three files are gone. + /// This side's pieces and the other side's, in that order. /// - /// Pawns and nothing else is read here, which is the property the pawn - /// hash rests on. Neither king, no piece and not the side to move: two - /// positions whose pawns agree agree on all eight counts, and - /// `Board::pawn_key` already stands for that agreement. - /// - /// Both `eval` and the tuner's walk read this, so the identity between - /// them cannot see a wrong count here, at the fitted weights or at zero. - /// What pins it is the hand counts beside this in the tests, the way the - /// shelter counts and the mobility counts are pinned. + /// Named for what it returns, which is both of them. Every caller + /// wants the pair: a generator masks its targets with one and its + /// captures with the other, and a leaf term reads its own pawns + /// against the other side's. #[inline] - pub(crate) fn pawn_structure_counts(&self, color: Color) -> [i32; eval::PAWN_TERMS] { - let masks = &PAWN_MASKS; - let side = color as usize; - let (ours, theirs) = match color { + pub(crate) fn sides(&self, color: Color) -> (u64, u64) { + match color { Color::White => (self.white, self.black), Color::Black => (self.black, self.white), - }; - let pawns = self.pawns(); - let (our_pawns, their_pawns) = (pawns & ours, pawns & theirs); - let mut counts = [0; eval::PAWN_TERMS]; - let mut remaining = our_pawns; - while remaining != 0 { - let square = remaining.trailing_zeros() as usize; - remaining &= remaining - 1; - let relative = match color { - Color::White => square / 8, - Color::Black => 7 - square / 8, - }; - // from_fen accepts a pawn on either back rank, knowingly, and the - // search has to survive one. Such a pawn has come no ranks or all - // eight and so names none of the six counted; it is left out of - // the passed count rather than folded into the nearest bucket, - // and it still counts toward the two below, which read its file - // and not its rank - if !(1..eval::PASSED_RANKS + 1).contains(&relative) { - continue; - } - if their_pawns & masks.front_span[side][square] == 0 - && our_pawns & masks.file_ahead[side][square] == 0 - { - counts[relative - 1] += 1; - } } - let files = files_of(our_pawns); - let beside = (files << 1) | (files >> 1); - counts[eval::PASSED_RANKS] = (our_pawns & spread(files & !beside)).count_ones() as i32; - counts[eval::PASSED_RANKS + 1] = - (our_pawns & ahead_of(our_pawns, color)).count_ones() as i32; - counts } /// Whether the side to move stands in check, read from the checkers @@ -3709,33 +3279,6 @@ mod pawn_key { let board = Board::from_fen("4k3/8/8/8/8/8/8/4K3 w - - 0 1").unwrap(); assert_eq!(board.pawn_key, 0); } - - /// `shelter_key` is this key with the two kings folded in, which is what - /// the shelter cache is keyed on. What the shelter reads is the pawns and - /// the two king squares, so a piece that is neither has to leave it alone - /// and either king moving has to move it. A key that missed a king would - /// hand one position's shelter to another. - #[test] - fn the_shelter_key_follows_the_pawns_and_the_two_kings() { - let bare = Board::from_fen("4k3/pppppppp/8/8/8/8/PPPPPPPP/4K3 w - - 0 1").unwrap(); - - // the same pawns and the same two kings behind a boardful of other - // pieces, which the shelter does not read - let pieced = - Board::from_fen("rnbqk1nr/pppppppp/8/8/8/8/PPPPPPPP/RNBQK1NR w - - 0 1").unwrap(); - assert_eq!(bare.shelter_key(), pieced.shelter_key()); - - // either king one square along - let ours = Board::from_fen("4k3/pppppppp/8/8/8/8/PPPPPPPP/5K2 w - - 0 1").unwrap(); - let theirs = Board::from_fen("5k2/pppppppp/8/8/8/8/PPPPPPPP/4K3 w - - 0 1").unwrap(); - assert_ne!(bare.shelter_key(), ours.shelter_key()); - assert_ne!(bare.shelter_key(), theirs.shelter_key()); - assert_ne!(ours.shelter_key(), theirs.shelter_key()); - - // and one pawn pushed, the pawn key being the rest of it - let pushed = Board::from_fen("4k3/pppppppp/8/8/8/7P/PPPPPPP1/4K3 w - - 0 1").unwrap(); - assert_ne!(bare.shelter_key(), pushed.shelter_key()); - } } #[cfg(test)] @@ -5241,109 +4784,10 @@ mod gives_check { } #[cfg(test)] -mod mobility { - use super::{ATTACK_MASKS, Board, Color, pawn_attacks}; - use crate::eval::ALL_KINDS; +mod pawn_spans { + use super::{ATTACK_MASKS, Color, pawn_attacks}; use pretty_assertions::assert_eq; - /// The counts by hand, square by square, because nothing else pins them. - /// The tuner's identity folds a row against the live weights, which are - /// zero, so it is blind to a wrong count here and stays blind to one after - /// the fit: `eval` and the tuner's walk read the same helper, so the two - /// sides of the identity move together whatever the helper answers. These - /// cases are the only check this term has. - /// - /// Each case names what the count is made of. The two kings stand in - /// opposite corners and out of the way, so that nothing here is a count of - /// theirs and no piece is placed giving check. - #[test] - fn a_piece_covers_what_a_hand_count_says_it_does() { - for (fen, counts, why) in [ - // a knight in the corner has two squares and one in the middle - // has all eight - ( - "k7/8/8/8/8/8/8/N6K w - - 0 1", - [2, 0, 0, 0], - "a knight on a1", - ), - ( - "k7/8/8/8/3N4/8/8/7K w - - 0 1", - [8, 0, 0, 0], - "a knight on d4", - ), - // rays of three, four, three and three - ( - "k7/8/8/8/3B4/8/8/7K w - - 0 1", - [0, 13, 0, 0], - "a bishop on d4", - ), - // a rank and a file, less the square it stands on - ( - "k7/8/8/8/3R4/8/8/7K w - - 0 1", - [0, 0, 14, 0], - "a rook on d4", - ), - ( - "k7/8/8/8/3Q4/8/8/7K w - - 0 1", - [0, 0, 0, 27], - "a queen on d4", - ), - // the friendly pawn on d6 is not scope and is not seen through - // either, so the file gives d5, d3, d2 and d1 beside the rank - ( - "k7/8/3P4/8/3R4/8/8/7K w - - 0 1", - [0, 0, 11, 0], - "a rook on d4 behind its own pawn", - ), - // the pawn on b7 covers c6, which is one of the knight's eight - ( - "k7/1p6/8/8/3N4/8/8/7K w - - 0 1", - [7, 0, 0, 0], - "a knight on d4 against a pawn on b7", - ), - // the enemy rook stands on one of the same eight and keeps it: - // a square with something to take on it is still scope - ( - "k7/8/2r5/8/3N4/8/8/7K w - - 0 1", - [8, 0, 0, 0], - "a knight on d4 against a rook on c6", - ), - // the enemy pawn stops the file at d6 rather than being seen - // through, so the file gives d5 and d6 beside the rank. This is - // the case that says the magic lookup is asked about the whole - // occupancy and not about this side's half of it - ( - "8/2k5/3p4/8/3R4/8/8/6K1 w - - 0 1", - [0, 0, 12, 0], - "a rook on d4 in front of an enemy pawn", - ), - ] { - let board = Board::from_fen(fen).unwrap(); - assert_eq!( - board.mobility_counts::<{ ALL_KINDS }>(Color::White), - counts, - "{}", - why - ); - } - } - - /// Black's count of a position is white's count of its reflection, so the - /// two colours are read the same way round. - #[test] - fn the_two_colours_count_the_same_squares() { - let white = Board::from_fen("7k/1p6/8/8/3N4/8/8/7K w - - 0 1").unwrap(); - let black = Board::from_fen("7k/8/8/3n4/8/8/1P6/7K b - - 0 1").unwrap(); - assert_eq!( - white.mobility_counts::<{ ALL_KINDS }>(Color::White), - black.mobility_counts::<{ ALL_KINDS }>(Color::Black) - ); - assert_eq!( - white.mobility_counts::<{ ALL_KINDS }>(Color::Black), - [0, 0, 0, 0] - ); - } - /// The span is shifted rather than gathered a pawn at a time, and the /// shifts have to answer what the masks generation reads already say. A /// pawn attacks upward for white and downward for black, so the span of a @@ -5369,229 +4813,6 @@ mod mobility { } } -#[cfg(test)] -mod shelter { - use super::{Board, Color, RANKS_AHEAD, SHELTER_MASKS, files_of, king_files}; - use pretty_assertions::assert_eq; - - /// The counts by hand, square by square, because nothing else pins them. - /// The tuner's identity folds a row against the live weights, which are - /// zero, so it is blind to a wrong count here and stays blind to one after - /// the fit: `eval` and the tuner's walk read the same helper, so the two - /// sides of the identity move together whatever the helper answers. These - /// cases are the only check this term has. - /// - /// Each case names what the count is made of, in the order the helper - /// returns them: our pawns one rank ahead and two, the king's files that - /// hold no pawn at all, the ones that hold an enemy pawn and none of ours, - /// and then the enemy pawns one, two and three ranks ahead. The other king - /// stands out of the way. - #[test] - fn a_king_shelters_behind_what_a_hand_count_says_it_does() { - for (fen, counts, why) in [ - // three pawns where a castled king wants them - ( - "k7/8/8/8/8/8/5PPP/6K1 w - - 0 1", - [3, 0, 0, 0, 0, 0, 0], - "a king on g1 behind f2, g2 and h2", - ), - // the g pawn one square further on is the far rank rather than - // the near one - ( - "k7/8/8/8/8/6P1/5P1P/6K1 w - - 0 1", - [2, 1, 0, 0, 0, 0, 0], - "a king on g1 with the g pawn on g3", - ), - // the g file holds no pawn of either colour - ( - "k7/8/8/8/8/8/5P1P/6K1 w - - 0 1", - [2, 0, 1, 0, 0, 0, 0], - "a king on g1 with no g pawn", - ), - // the same file with a black pawn on it is half open rather than - // open. The pawn is on g7, which is past the three ranks the - // storm is followed over, so it is a file and not a storm - ( - "k7/6p1/8/8/8/8/5P1P/6K1 w - - 0 1", - [2, 0, 0, 1, 0, 0, 0], - "a king on g1 with a black pawn on g7", - ), - // h2 near, g3 far, and the f file holding a black pawn on f5, - // which is a rank further out than the storm reaches - ( - "k7/8/8/5p2/8/6P1/7P/6K1 w - - 0 1", - [1, 1, 0, 1, 0, 0, 0], - "a king on g1 with the f file gone", - ), - // an enemy pawn standing on a rank in front of the king is not - // cover. It is the storm, counted by the rank it has reached - ( - "k7/8/8/8/8/8/5PpP/6K1 w - - 0 1", - [2, 0, 0, 1, 1, 0, 0], - "a king on g1 with a black pawn on g2", - ), - ( - "k7/8/8/8/8/6p1/5P1P/6K1 w - - 0 1", - [2, 0, 0, 1, 0, 1, 0], - "a king on g1 with a black pawn on g3", - ), - ( - "k7/8/8/8/6p1/8/5P1P/6K1 w - - 0 1", - [2, 0, 0, 1, 0, 0, 1], - "a king on g1 with a black pawn on g4", - ), - // two ranks of storm at once, on two files - ( - "k7/8/8/8/7p/6p1/5P1P/6K1 w - - 0 1", - [2, 0, 0, 1, 0, 1, 1], - "a king on g1 against pawns on g3 and h4", - ), - // a king in the corner is read against three files, so the f file - // it does not stand beside is still counted. Two files would - // leave this at nothing - ( - "k7/8/8/8/8/8/6PP/7K w - - 0 1", - [2, 0, 1, 0, 0, 0, 0], - "a king on h1 behind g2 and h2", - ), - ( - "k7/8/8/8/8/8/PPP5/K7 w - - 0 1", - [3, 0, 0, 0, 0, 0, 0], - "a king on a1 behind a2, b2 and c2", - ), - // a king off its own ranks has no rank in front of it inside the - // masks, so the pawns it left behind are not shelter - ( - "k7/8/8/4K3/8/8/3PPP2/8 w - - 0 1", - [0, 0, 0, 0, 0, 0, 0], - "a king on e5 with its pawns at home", - ), - // a pawn on the file stops it counting as open wherever on the - // file it stands, so a passed pawn up the board is not a hole - // behind the king it left - ( - "k7/6P1/8/8/8/8/8/6K1 w - - 0 1", - [0, 0, 2, 0, 0, 0, 0], - "a king on g1 whose only pawn is on g7", - ), - // the ranks in front of a king on the eighth are off the board - // rather than round the other side of it - ( - "4K3/8/8/8/8/8/8/k7 w - - 0 1", - [0, 0, 3, 0, 0, 0, 0], - "a king on e8 with no pawns anywhere", - ), - ] { - let board = Board::from_fen(fen).unwrap(); - assert_eq!(board.shelter_counts(Color::White), counts, "{}", why); - } - } - - /// The same reading for black, whose king is measured down the board - /// rather than up it. - #[test] - fn a_black_king_is_measured_down_the_board() { - // the rank in front of a king on the first is off the board, and the - // three files all hold a white pawn and no black one - let board = Board::from_fen("7K/8/8/8/8/8/3PPP2/4k3 b - - 0 1").unwrap(); - assert_eq!(board.shelter_counts(Color::Black), [0, 0, 0, 3, 0, 0, 0]); - // and the storm the other way up: the white pawn on g6 stands two - // ranks in front of a black king on g8, and is not its cover - let stormed = Board::from_fen("6k1/5ppp/6P1/8/8/8/8/6K1 b - - 0 1").unwrap(); - assert_eq!(stormed.shelter_counts(Color::Black), [3, 0, 0, 0, 0, 1, 0]); - } - - /// Black's count of a position is white's count of its reflection, so the - /// two colours are read the same way round. - #[test] - fn the_two_colours_count_the_same_squares() { - let white = Board::from_fen("k7/8/8/5p2/8/6P1/7P/6K1 w - - 0 1").unwrap(); - let black = Board::from_fen("6k1/7p/6p1/8/5P2/8/8/K7 b - - 0 1").unwrap(); - assert_eq!( - white.shelter_counts(Color::White), - black.shelter_counts(Color::Black) - ); - // and the storm half of it, which the pair above leaves at zero - let stormed = Board::from_fen("k7/8/8/8/7p/6p1/5P1P/6K1 w - - 0 1").unwrap(); - let mirrored = Board::from_fen("6k1/5p1p/6P1/7P/8/8/8/K7 b - - 0 1").unwrap(); - assert_eq!( - stormed.shelter_counts(Color::White), - mirrored.shelter_counts(Color::Black) - ); - } - - /// Every square names three files, the two corners included, and the - /// three are the king's own file and its neighbours wherever there is - /// room for them. - #[test] - fn every_king_square_names_three_files() { - for square in 0..64u8 { - let files = king_files(square); - assert_eq!(files.count_ones(), 3, "the files of {}", square); - assert_eq!( - files & (1 << (square % 8)), - 1 << (square % 8), - "the king's own file is not among the files of {}", - square - ); - assert_eq!( - files.trailing_zeros() + 2, - 7 - files.leading_zeros(), - "the files of {} are not three in a row", - square - ); - } - } - - /// Each mask holds the rank its index names, on those same three files, - /// and nothing where the board has run out. - #[test] - fn the_masks_hold_the_ranks_in_front_of_the_king() { - for square in 0..64u8 { - let rank = i32::from(square / 8); - for (side, forward) in [(Color::White, 1), (Color::Black, -1)] { - let i = side as usize; - for step in 0..RANKS_AHEAD { - let mask = SHELTER_MASKS.ahead[step][i][square as usize]; - let target = rank + forward * (step as i32 + 1); - if !(0..8).contains(&target) { - assert_eq!(mask, 0, "{:?} on {} at {} ahead", side, square, step + 1); - continue; - } - assert_eq!(mask.count_ones(), 3, "{:?} on {}", side, square); - assert_eq!( - files_of(mask), - king_files(square), - "{:?} on {} covers other files", - side, - square - ); - let rank_mask = 0xffu64 << (target * 8); - assert_eq!( - mask & rank_mask, - mask, - "{:?} on {} is off its rank", - side, - square - ); - } - } - } - } - - /// The fold down to a file a bit answers what a walk of the squares does. - #[test] - fn the_file_fold_answers_a_walk_of_the_squares() { - for square in 0..64u8 { - assert_eq!(files_of(1u64 << square), 1 << (square % 8), "{}", square); - } - let board = Board::from_fen("k7/8/8/5p2/8/6P1/7P/6K1 w - - 0 1").unwrap(); - // white's pawns stand on g3 and h2, and black's on f5 - assert_eq!(files_of(board.pawns() & board.white), 0b1100_0000); - assert_eq!(files_of(board.pawns() & board.black), 0b0010_0000); - } -} - #[cfg(test)] mod evasion_targets { use super::fens; @@ -5732,327 +4953,6 @@ mod evasion_targets { } } -#[cfg(test)] -mod pawn_structure { - use super::{Board, Color, PAWN_MASKS, ahead_of, files_of, pawn_files, spread}; - use pretty_assertions::assert_eq; - - /// The counts by hand, position by position, because nothing else pins - /// them. The tuner's identity folds a row against the live weights, which - /// are zero, so it is blind to a wrong count here, and it stays blind to - /// one after a fit: `eval` and the tuner's walk read this same helper, so - /// the two sides of the identity move together whatever it answers. These - /// cases are the only check this term has. - /// - /// Each case names the eight counts in the order the helper returns them: - /// the passed pawns on the relative second through the relative seventh, - /// then the isolated pawns, then the doubled ones. The other king stands - /// out of the way. - #[test] - fn pawns_count_as_a_hand_count_says_they_do() { - for (fen, counts, why) in [ - // a pawn with nothing in front of it anywhere is passed, and a - // pawn with no pawn beside it is isolated. The lone pawn is both - ( - "4k3/8/8/8/4P3/8/8/4K3 w - - 0 1", - [0, 0, 1, 0, 0, 0, 1, 0], - "a white pawn on e4 and no black pawn", - ), - // an enemy pawn on an adjacent file ahead of it stops it - ( - "4k3/8/8/3p4/4P3/8/8/4K3 w - - 0 1", - [0, 0, 0, 0, 0, 0, 1, 0], - "the same against a black pawn on d5", - ), - // level is not ahead. A black pawn beside the white one has - // already been passed - ( - "4k3/8/8/8/3pP3/8/8/4K3 w - - 0 1", - [0, 0, 1, 0, 0, 0, 1, 0], - "the same against a black pawn on d4", - ), - // the whole file ahead is read and not the next rank or two - ( - "4k3/5p2/8/8/4P3/8/8/4K3 w - - 0 1", - [0, 0, 0, 0, 0, 0, 1, 0], - "the same against a black pawn on f7", - ), - // a pawn of ours in front of it stops it too, which is what - // leaves the rear of a doubled pair out of the passed count - ( - "4k3/8/8/4P3/4P3/8/8/4K3 w - - 0 1", - [0, 0, 0, 1, 0, 0, 2, 1], - "white pawns on e4 and e5", - ), - // a tripled file is two doubled pawns and not one or three - ( - "4k3/8/8/8/4P3/4P3/4P3/4K3 w - - 0 1", - [0, 0, 1, 0, 0, 0, 3, 2], - "white pawns on e2, e3 and e4", - ), - // the relative second is a real bucket. A pawn still at home is - // passed once the enemy pawns on its three files are gone - ( - "4k3/8/8/8/8/8/4P3/4K3 w - - 0 1", - [1, 0, 0, 0, 0, 0, 1, 0], - "a white pawn on e2 with no black pawn", - ), - ( - "4k3/4P3/8/8/8/8/8/4K3 w - - 0 1", - [0, 0, 0, 0, 0, 1, 1, 0], - "a white pawn on e7", - ), - // the stop square is not read, so a blockaded passer is a passer. - // On purpose: the stop square is the first thing this term leaves - // out, and a later arm has to be able to find the place it goes - ( - "4k3/4n3/4P3/8/8/8/8/4K3 w - - 0 1", - [0, 0, 0, 0, 1, 0, 1, 0], - "a white pawn on e6 behind a black knight on e7", - ), - // a pawn two files away is no company - ( - "4k3/8/8/8/8/8/P1P5/4K3 w - - 0 1", - [2, 0, 0, 0, 0, 0, 2, 0], - "white pawns on a2 and c2", - ), - ( - "4k3/8/8/8/8/8/PP6/4K3 w - - 0 1", - [2, 0, 0, 0, 0, 0, 0, 0], - "white pawns on a2 and b2", - ), - // the two edge files at once. The file mask is a shift and not a - // rotate, so the a file has no neighbour off the left of the byte - // and the h file none off the right; a rotate would make each of - // these the other's neighbour and leave both counted as company - ( - "4k3/8/8/8/8/8/P6P/4K3 w - - 0 1", - [2, 0, 0, 0, 0, 0, 2, 0], - "white pawns on a2 and h2", - ), - // the edge cases the file mask decides. A pawn on the a file is - // read against the a and b files and not against the c file, so - // stepping the mask in the way `king_files` steps it would let - // the pawn on c5 stop this one - ( - "4k3/8/8/1p6/P7/8/8/4K3 w - - 0 1", - [0, 0, 0, 0, 0, 0, 1, 0], - "a white pawn on a4 against a black pawn on b5", - ), - ( - "4k3/8/8/2p5/P7/8/8/4K3 w - - 0 1", - [0, 0, 1, 0, 0, 0, 1, 0], - "a white pawn on a4 against a black pawn on c5", - ), - ( - "4k3/8/8/6p1/7P/8/8/4K3 w - - 0 1", - [0, 0, 0, 0, 0, 0, 1, 0], - "a white pawn on h4 against a black pawn on g5", - ), - // from_fen accepts a pawn on either back rank and the search has - // to survive one. It has come no ranks or all eight, so it names - // none of the six passed buckets, and it still counts toward the - // two that read its file - ( - "4k3/8/8/8/8/8/8/3K1P2 w - - 0 1", - [0, 0, 0, 0, 0, 0, 1, 0], - "a white pawn on f1", - ), - ( - "4k1P1/8/8/8/8/8/8/4K3 w - - 0 1", - [0, 0, 0, 0, 0, 0, 1, 0], - "a white pawn on g8", - ), - ] { - let board = Board::from_fen(fen).unwrap(); - assert_eq!(board.pawn_structure_counts(Color::White), counts, "{}", why); - } - } - - /// The same reading for black, whose pawns are measured down the board - /// rather than up it. Each case is the reflection of one above, so a - /// relative rank read the wrong way up shows as a count in the wrong - /// bucket rather than as no count at all. - #[test] - fn a_black_pawn_is_measured_down_the_board() { - for (fen, counts, why) in [ - ( - "4k3/8/8/4p3/8/8/8/4K3 w - - 0 1", - [0, 0, 1, 0, 0, 0, 1, 0], - "a black pawn on e5 and no white pawn", - ), - ( - "4k3/8/8/4p3/3P4/8/8/4K3 w - - 0 1", - [0, 0, 0, 0, 0, 0, 1, 0], - "the same against a white pawn on d4", - ), - ( - "4k3/8/8/4p3/4p3/8/8/4K3 w - - 0 1", - [0, 0, 0, 1, 0, 0, 2, 1], - "black pawns on e4 and e5", - ), - ( - "4k3/8/8/8/8/8/4p3/4K3 w - - 0 1", - [0, 0, 0, 0, 0, 1, 1, 0], - "a black pawn on e2", - ), - ( - "4k3/4p3/8/8/8/8/8/4K3 w - - 0 1", - [1, 0, 0, 0, 0, 0, 1, 0], - "a black pawn on e7", - ), - ] { - let board = Board::from_fen(fen).unwrap(); - assert_eq!(board.pawn_structure_counts(Color::Black), counts, "{}", why); - } - } - - /// Black's count of a position is white's count of its reflection, so the - /// two colours are read the same way round. - #[test] - fn the_two_colours_count_the_same_way() { - let white = Board::from_fen("4k3/P4p2/8/3P2p1/3P4/PP2p2p/1P6/4K3 w - - 0 1").unwrap(); - let black = Board::from_fen("4k3/1p6/pp2P2P/3p4/3p2P1/8/p4P2/4K3 w - - 0 1").unwrap(); - assert_eq!( - white.pawn_structure_counts(Color::White), - black.pawn_structure_counts(Color::Black) - ); - assert_eq!( - white.pawn_structure_counts(Color::Black), - black.pawn_structure_counts(Color::White) - ); - } - - /// Nothing but the pawns decides the counts, which is the property the - /// pawn hash rests on. The same pawns behind different pieces, and with - /// the two kings somewhere else, count the same. - #[test] - fn nothing_but_the_pawns_is_counted() { - let bare = Board::from_fen("4k3/pp3ppp/8/8/8/8/PPP2PP1/4K3 w - - 0 1").unwrap(); - let full = - Board::from_fen("r1bq1rk1/pp3ppp/2n5/8/8/2N5/PPP2PP1/R1BQK2R w KQ - 0 1").unwrap(); - for color in [Color::White, Color::Black] { - assert_eq!( - bare.pawn_structure_counts(color), - full.pawn_structure_counts(color), - "{:?}", - color - ); - } - } - - /// A pawn's own file and the files beside it, and no more than that. The - /// edges are the case: two files there and not three, and not three with - /// the middle one stepped in the way `king_files` steps it. - #[test] - fn a_pawn_names_its_own_file_and_the_ones_beside_it() { - for square in 0..64u8 { - let file = u32::from(square % 8); - let files = pawn_files(square); - let expected = if file == 0 || file == 7 { 2 } else { 3 }; - assert_eq!(files.count_ones(), expected, "the files of {}", square); - assert_eq!( - files & (1 << file), - 1 << file, - "not its own file: {}", - square - ); - assert_eq!( - files.trailing_zeros() + expected - 1, - 7 - files.leading_zeros(), - "the files of {} are not in a row", - square - ); - } - } - - /// Each front span holds every rank ahead of the pawn and no rank level - /// with it or behind it, on the files the pawn names and no others. - #[test] - fn the_front_span_is_the_files_beside_the_pawn_on_the_ranks_ahead() { - for square in 0..64u8 { - let rank = i32::from(square / 8); - for (side, forward) in [(Color::White, 1), (Color::Black, -1)] { - let span = PAWN_MASKS.front_span[side as usize][square as usize]; - let ranks = if forward == 1 { 7 - rank } else { rank }; - assert_eq!( - span.count_ones(), - pawn_files(square).count_ones() * ranks as u32, - "{:?} on {}", - side, - square - ); - if span != 0 { - assert_eq!( - files_of(span), - pawn_files(square), - "{:?} on {} covers other files", - side, - square - ); - } - for step in 0..8i32 { - let on_rank = span & (0xffu64 << (step * 8)); - let ahead = (step - rank) * forward > 0; - assert_eq!( - on_rank != 0, - ahead, - "{:?} on {} holds rank {}", - side, - square, - step - ); - } - } - } - } - - /// The file ahead is the front span with the neighbouring files taken - /// off, which is the relationship the two tables are built to have. Kept - /// as its own table rather than masked out at the leaf, so this is what - /// says the two agree. - #[test] - fn the_file_ahead_is_the_front_span_on_the_pawns_own_file() { - for square in 0..64u8 { - let own = spread(1u8 << (square % 8)); - for side in [Color::White, Color::Black] { - let i = side as usize; - assert_eq!( - PAWN_MASKS.file_ahead[i][square as usize], - PAWN_MASKS.front_span[i][square as usize] & own, - "{:?} on {}", - side, - square - ); - } - } - } - - /// The forward fill holds every square in front of a pawn and never the - /// pawn itself, which is what leaves a lone pawn undoubled. - #[test] - fn the_fill_starts_one_rank_in_front_of_the_pawn() { - for square in 0..64u8 { - let pawn = 1u64 << square; - for side in [Color::White, Color::Black] { - let filled = ahead_of(pawn, side); - assert_eq!( - filled & pawn, - 0, - "{:?} on {} is in its own fill", - side, - square - ); - assert_eq!( - filled, PAWN_MASKS.file_ahead[side as usize][square as usize], - "{:?} on {}", - side, square - ); - } - } - } -} - #[cfg(test)] mod drawn_by_material { use super::{A1, B1, Board, LIGHT_SQUARES}; diff --git a/arche-core/src/engine.rs b/arche-core/src/engine.rs index 99cff950..114a1b77 100644 --- a/arche-core/src/engine.rs +++ b/arche-core/src/engine.rs @@ -647,15 +647,11 @@ pub struct AlphaBeta { /// The move ordering and its scratch buffer, search state like the /// limits above: one per engine, reused by every node. ordering: MoveOrdering, - /// The king shelter's memo, scratch on the same terms as the ordering. - /// Never cleared between searches: an entry is read only against the key - /// that wrote it, so what the last search left is a warm start and not a - /// stale answer. - shelter: eval::ShelterCache, - /// The pawn structure's memo, on the same terms. A second cache rather - /// than a wider entry in the shelter's, because the two are keyed on - /// different things: this one misses only on a pawn move. - pawns: eval::PawnCache, + /// What the leaf terms that remember themselves remember, scratch on the + /// same terms as the ordering. Never cleared between searches: an entry + /// is read only against the key that wrote it, so what the last search + /// left is a warm start and not a stale answer. + caches: eval::Caches, /// The residual sampler, or none, which is what every constructor here /// builds and what the engine plays and benches with. An engine with /// none takes no branch a search without a sampler did not take, which @@ -749,8 +745,7 @@ impl AlphaBeta { stop: None, quiescence_nodes: 0, ordering: MoveOrdering::new(), - shelter: eval::ShelterCache::default(), - pawns: eval::PawnCache::default(), + caches: eval::Caches::default(), sampler: None, census: None, ledger: None, @@ -1104,13 +1099,13 @@ impl AlphaBeta { } /// The score at this node, with the shelter and the pawn structure taken - /// from the engine's two memos. + /// from the engine's memos. /// /// `&mut self` for the memos alone. The score is the one `eval::eval` /// gives, so nothing about the tree turns on which of the two a node /// asked. fn eval(&mut self) -> Score { - crate::eval::eval_cached(&self.board, &mut self.shelter, &mut self.pawns) + crate::eval::eval_cached(&self.board, &mut self.caches) } /// The ply the quiet memories are indexed by at this node, or none when diff --git a/arche-core/src/eval.rs b/arche-core/src/eval.rs deleted file mode 100644 index 9ef56ca2..00000000 --- a/arche-core/src/eval.rs +++ /dev/null @@ -1,1543 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later -// Copyright (C) 2022-2026 Andrew Wright - -//! The evaluation: what a position scores, and every number that opinion is -//! built from. The board hosts an [`Accumulator`] and tells it about each -//! piece placed, removed and moved from one square to another; the search -//! asks [`eval`] for the score. A term cheap enough to keep incrementally -//! belongs in the accumulator; one computed at the leaf belongs in [`eval`], -//! reading the board, which is where mobility is. - -use crate::board::Board; -use crate::misc::{Color, Piece, Score}; -use crate::psqt::{PieceSquareTables, eg_value, mg_value, pack}; - -static PIECE_SQUARE_TABLES: PieceSquareTables = PieceSquareTables::TABLES; - -/// What each piece leaves on the board, in `Piece` order, on the scale the -/// two halves of a tapered score are interpolated on. A queen counts for four, -/// a rook two and a minor one, so the opening's complement of pieces comes to -/// `TOTAL_PHASE` and a bare king and pawns to nothing. Pawns count for nothing -/// because an ending is an ending whether or not there are pawns in it. -static PHASE_WEIGHTS: [i32; 6] = [0, 1, 1, 2, 4, 0]; -/// What the opening's pieces add up to under `PHASE_WEIGHTS`. -pub(crate) const TOTAL_PHASE: i32 = 24; - -/// A table rather than a match. The match compiled to a jump table, and -/// once the piece arrives as a load from the board's square array the -/// target is data the predictor cannot see through: most of the search's -/// indirect mispredicts were this dispatch inside the accumulator's count. -/// Indexed the way the piece square tables and `Zobrist` already index by -/// piece; the assertion in `misc` pins the discriminants. -const MATERIAL: [u32; 6] = [100, 310, 320, 500, 900, 10000]; - -/// The material weight of one piece, for the board's own seeding walk. -pub(crate) fn material(piece: Piece) -> u32 { - MATERIAL[piece as usize] -} - -/// The pieces that carry a mobility weight, in the order [`MOBILITY`] and -/// `Board::mobility_counts` are indexed by. The pawn and the king carry none. -pub(crate) const MOBILE_PIECES: [Piece; 4] = - [Piece::Knight, Piece::Bishop, Piece::Rook, Piece::Queen]; - -/// What one square of scope is worth to each of [`MOBILE_PIECES`], as the -/// packed pairs the taper is read from. -/// -/// Refitted 2026-09-13 by `scripts/tune.py` over the whole archived strength -/// run: 259 artifacts across 54 runs, 43,675 games, whose 5,694,775 post-book -/// plies gave 5,355,792 positions and 2,461,322 quiet rows, extracted by -/// `arche terms` at 2105b55. The corpus is sha256 `b37ebf0c` and the rows are -/// sha256 `49aa715a`. K was held at 1.1350 and the games split by pair: -/// 12,743 pairs trained, 4,339 chose the ridge and 4,747 were sealed. The 768 -/// table entries, the six material values, the fourteen shelter weights and -/// the sixteen pawn structure ones were all held, so these eight are the only -/// thing that moved. Every number here is of the rounded vector that ships. -/// -/// The selection group scores 0.087671 at the old weights and 0.086874 at -/// these, a paired difference of -0.000798 against a standard error of -/// 0.000113 at a design factor of 3.8. The sealed group, opened once -/// afterwards over 4,747 pairs and 9,452 games that no fit and no ridge -/// choice had read, scores 0.080507 and 0.079997, a paired difference of -/// -0.000510 against a standard error of 0.000107 at a design factor of 4.1. -/// Both are outside their intervals, and they are 1.85 standard errors apart, -/// so the selection group overstates this fit by about a third. The king -/// safety fit's two agreed to 0.68 and the pawn structure fit's to 0.15. -/// -/// The first fit of this term read 1,812 games and priced one piece. This one -/// reads twenty four times as many and prices all four, which is the whole of -/// what changed: nothing about the term is different, the corpus simply has -/// enough games to say something about a knight. A preliminary fit taken -/// before the pawn structure term existed predicted `[5,6,5,1]` and -/// `[0,2,2,7]`, and it came out `[4,6,4,1]` and `[1,3,4,7]`. Pawn structure -/// took almost nothing away from mobility, despite both terms reading open -/// lines. -/// -/// The ridge is zero, which the grid ranked first, and here that is not the -/// trap it was for the pawn structure fit. The vector quantizes identically -/// at zero, 1e-8 and 1e-7, the largest weight is 7, and every one of the -/// eight slots carries a coefficient in 37% to 70% of the training rows with -/// summed coefficients in the tens of millions. There is no low leverage -/// direction here for an unregularised fit to hide error in. -/// -/// By phase the sealed reading is -0.000160 at six pieces or fewer, -/// -0.001725 from seven to twelve, and +0.000703 at thirteen or more. The -/// term pays in the middlegame, barely in the ending, and costs something in -/// the opening, which is the phase a mobility count would be expected to earn -/// most. That is worth an ablation and is not one this arm ran. -/// -/// Every weight is non-zero, so `SCORED_KINDS` has nothing left to skip and -/// all four kinds are counted at every leaf again. What that costs is in the -/// commit that landed this, and it is the larger half of the arm. -/// -/// A count is at most twenty seven for a queen and a boardful comes to a few -/// hundred, so a weight in single figures leaves the same order of magnitude -/// in hand that `pack` asks for. -const MOBILITY: [i32; MOBILE_PIECES.len()] = [pack(4, 1), pack(6, 3), pack(4, 4), pack(1, 7)]; - -/// The mobility weight of one of the four pieces that carries one, as the -/// packed pair. The tuner's seam asks, so that a slot names the live weight -/// rather than a copy of it, the way it reads the tables. -pub(crate) const fn mobility_weight(index: usize) -> i32 { - MOBILITY[index] -} - -/// A set of [`MOBILE_PIECES`], a bit per index, which is what -/// `Board::mobility_counts` takes. A kind left out of the set is not counted -/// and answers zero. -/// -/// This is all four of them. The tuner's walk asks for it whatever the -/// weights hold, because that walk is offline and its coefficients are what -/// lets a later fit price a kind that is worth nothing today. -pub(crate) const ALL_KINDS: u8 = (1 << MOBILE_PIECES.len()) - 1; - -/// The kinds [`eval`] counts: the ones whose [`MOBILITY`] weight is not zero -/// at one end of the taper or the other. The 2026-09-13 refit priced all -/// eight weights, so this is now all four kinds and skips nothing. -/// -/// Derived from the weights rather than written out, which is what made the -/// refit start counting three kinds again rather than leaving them ignored at -/// every leaf. It cost what the refit's commit records. -/// `eval_counts_a_kind_exactly_when_its_weight_is_not_zero` is what holds the -/// two together. -pub(crate) const SCORED_KINDS: u8 = scored_kinds(); - -/// Whether `kinds` names the piece at `index` in [`MOBILE_PIECES`]. -pub(crate) const fn counted(kinds: u8, index: usize) -> bool { - kinds & (1 << index) != 0 -} - -/// [`SCORED_KINDS`], read off the weights at compile time. Both halves are -/// asked about: a weight worth nothing in the midgame and something in the -/// ending is still a weight and still has to be counted. -const fn scored_kinds() -> u8 { - let mut kinds = 0; - let mut index = 0; - while index < MOBILE_PIECES.len() { - let weight = mobility_weight(index); - if mg_value(weight) != 0 || eg_value(weight) != 0 { - kinds |= 1 << index; - } - index += 1; - } - kinds -} - -/// How many counts the king's shelter is measured in, and so how many weights -/// it carries at each end of the taper. `Board::shelter_counts` prints them in -/// this order: this side's pawns one rank in front of its king, its pawns two -/// ranks in front, the king's files with no pawn of either colour on them, the -/// king's files holding an enemy pawn and none of ours, and then the enemy -/// pawns one, two and three ranks in front of the king. -pub(crate) const SHELTER_TERMS: usize = 7; - -/// What one of those seven counts is worth, as the packed pairs the taper is -/// read from. -/// -/// Fitted 2026-09-12 by `scripts/tune.py` over the whole archived strength -/// run: 28,675 games (27,133 at 10+0.1, 1,500 at 30+0.3 and 42 at 2+0.02), -/// whose 3,711,074 post-book plies gave 3,536,193 positions and 1,623,149 -/// quiet rows in 28,618 of them, extracted by `arche terms` at 4e5bd7d, with -/// K held at 1.2071 and the games split 17,192 that trained, 5,654 that chose -/// the ridge of 1e-8 and 5,772 that were sealed. The 768 table entries, the -/// six material values and the eight mobility weights were all held where -/// they stand, so these fourteen are the only thing that moved. Every number -/// here is of the rounded vector that ships. The selection group scores -/// 0.090395 at zero and 0.089746 at these, a paired difference of -0.000649 -/// against a standard error of 0.000116 over its 5,654 games, at a design -/// factor of 3.9. That is outside its interval, which the mobility fit's -/// reading was not. -/// -/// The sealed group was opened once, after the games had accepted this -/// vector, over 5,772 games and 324,273 positions no fit and no ridge choice -/// had read. It scores 0.090914 at zero and 0.090377 at these, a paired -/// difference of -0.000537 against a standard error of 0.000118 at a design -/// factor of 3.9. That is outside its interval, and 0.68 standard errors -/// from the selection group's reading, so the two groups agree. The piece -/// square fit's two did not. -/// -/// Not one of the fourteen rounded to nothing, so every count is priced and -/// none can be left uncounted at the leaf the way `SCORED_KINDS` leaves a -/// mobility kind. [`ShelterCache`] is what pays for the seven instead. -/// -/// Two of the storm's three signs are not what the term was named for. An -/// enemy pawn one rank in front of the king reads 11 and 35, and three ranks -/// out reads 14, so the fit likes the near storm where the term expected it -/// to fear it, and only the middle rank is negative. Our own cover is worth -/// 21 in the midgame and -26 in the ending, which reads as a king that wants -/// to be active rather than covered. The corpus is the first place to look -/// and not the term: 66.4% of its appearances have six or fewer pieces left -/// on the board and 6.0% have thirteen or more of the fourteen, so the -/// midgame half of the taper, which is the half king safety is about, is -/// fitted on the thinnest slice of the games. The count with the oddest -/// weight is also the thinnest supported: an enemy pawn one rank in front of -/// a king carries a coefficient in 4.65% of the rows against 47.83% for the -/// near cover, because a king usually takes such a pawn and the position is -/// then not quiet. The loss says these fourteen score the corpus better than -/// zero did, and that is all it says. -/// -/// A side's seven counts come to eighteen at the very most. Five of them are -/// at most three pawns each, and the other two share three files between them -/// rather than reaching three each. Against these weights the largest total -/// any legal set of counts reaches on one side is 168 in the midgame half and -/// -207 in the ending half, so a boardful of both colours leaves the sixteen -/// bits `pack` gives each half a long way off. Six of the fourteen are past -/// single figures, which the paragraph here said they would not be while they -/// were all zero. -static SHELTER: [i32; SHELTER_TERMS] = [ - pack(10, -11), - pack(21, -26), - pack(-11, -26), - pack(-8, -1), - pack(11, 35), - pack(-18, 2), - pack(14, -6), -]; - -/// The shelter weight of one of the seven counts, as the packed pair. The -/// tuner's seam asks, so that a slot names the live weight rather than a copy -/// of it, the way it reads the tables. -pub(crate) fn shelter_weight(index: usize) -> i32 { - SHELTER[index] -} - -/// How many ranks the passed pawn count is split over: a pawn's relative -/// second through its relative seventh, which is every rank one can stand on. -/// -/// A table by rank rather than one weight times the rank. What a passer is -/// worth is convex in how far it has come, and a ramp cannot say so; six -/// numbers can, and the tables already spend sixty four on where a pawn -/// stands. -pub(crate) const PASSED_RANKS: usize = 6; - -/// What the pawn structure is measured in, in the order [`PAWN_STRUCTURE`] -/// and `Board::pawn_structure_counts` are indexed by: the passed pawns by -/// relative rank, then the isolated pawns, then the doubled ones. -pub(crate) const PAWN_TERMS: usize = PASSED_RANKS + 2; - -/// What one of those eight counts is worth, as the packed pairs the taper is -/// read from. -/// -/// Fitted 2026-09-12 by `scripts/tune.py` over the whole archived strength -/// run: 164 artifacts across 37 runs, 34,175 games, whose 4,428,569 post-book -/// plies gave 4,196,989 positions and 1,922,548 quiet rows, extracted by -/// `arche terms` at 39da0fe. The corpus is sha256 `f9ff326c` and the rows are -/// sha256 `35fd184d`. K was held at 1.1959 and the games split by pair: 10,696 -/// pairs trained, 3,638 chose the ridge and 2,750 were sealed. The 768 table -/// entries, the six material values, the eight mobility weights and the -/// fourteen shelter ones were all held where they stand, so these sixteen are -/// the only thing that moved. Every number here is of the rounded vector that -/// ships. -/// -/// The selection group scores 0.090580 at zero and 0.089555 at these, a -/// paired difference of -0.001025 against a standard error of 0.000128 over -/// its 7,263 games, at a design factor of 4.8. That is eight standard errors -/// outside its interval, and larger than the king safety fit read on a corpus -/// four fifths this size. By phase it is -0.001274 where six or fewer pieces -/// are left, -0.000671 from seven to twelve, and +0.000135 at thirteen or -/// more: the term pays in the ending, pays a little in the middlegame and -/// costs a little in the opening, which is the shape a pawn structure term -/// should have. -/// -/// The sealed group was opened once, after the weights were frozen, over -/// 2,750 pairs and 5,480 games and 312,012 positions that no fit and no -/// ridge choice had read. It scores 0.084236 at zero and 0.083240 at these, -/// a paired difference of -0.000995 against a standard error of 0.000146 at -/// a design factor of 5.0. That is outside its interval, and 0.15 standard -/// errors from the selection group's reading, so the two agree more closely -/// than the king safety fit's two did. -/// -/// It also answers the one thing the selection group said against the term. -/// There the opening bucket got worse by 0.000135; on the sealed games it -/// improves by 0.000249, so that was noise rather than a cost. By phase the -/// sealed reading is -0.001107 at six pieces or fewer, -0.000867 from seven -/// to twelve and -0.000249 at thirteen or more, which is the same shape -/// without the sting in its tail. -/// -/// The ridge is 1e-8, chosen on the selection group from a grid the zero was -/// taken out of. The grid as it stands includes no regularisation at all and -/// ranked it first, and the vector it produced put 239 on a passed pawn's -/// seventh rank in the midgame and -29 in the ending. `PAWNS_END` already -/// pays a pawn on the seventh 78 to 83, so that says a passed pawn one square -/// from queening is worth less than a blockaded one in the phase that decides -/// it. The seventh rank count carries the fewest rows of the eight (5.31%) -/// and the smallest summed midgame coefficient of all sixteen slots, a -/// twenty seventh of the isolated count's, so it is the direction the corpus -/// constrains least and the one an unregularised fit put its error in. -/// Overruling the grid cost 0.000132 of selection loss against a standard -/// error of 0.00015, and bought a largest weight of 53 rather than 239. -/// -/// The selection group did not catch that, which is the part worth knowing: -/// zero scored better there, on games no fit had read. A held-out group -/// detects a vector that has memorised its training games and does not detect -/// one that has learned a real but unrepresentative regularity, and the quiet -/// filter manufactures exactly that wherever a feature's interesting cases -/// are tactical. -/// -/// Three of the sixteen are not what the term was built to say, and they are -/// left as the fit gave them rather than tidied. A passed pawn on the seventh -/// is worth less than one on the sixth at both ends, and that survives every -/// ridge on the grid, so it is the corpus and not the regularisation: a -/// position with a passer on the seventh is rarely quiet unless the pawn is -/// blockaded or about to be lost, so the rows that reach the fit are the ones -/// where it is not winning. Passed pawns on the second through the fourth are -/// a midgame penalty. A doubled pawn is worth 8 in the midgame. The isolated -/// count, which carries a coefficient in 56.98% of the rows and has by far -/// the most evidence behind it, reads -11 and -6, which is what a player -/// would have guessed. -/// -/// A side's eight pawns can fill the eight counts twenty four times over -/// between them, since one pawn can be passed and isolated and doubled at -/// once, but no count can exceed eight and the six passed counts share the -/// eight between them. So the most this term adds to one half of the packed -/// pair is 776 in the midgame and 440 in the ending, both sides counted, -/// against the 32,767 a half has to stay inside. `bounds_hold` charges eight -/// of every count rather than eight across them, which is looser again and -/// is the screen rather than the arithmetic: it puts the whole vector's -/// boardful at 8,622, of which this term is at most 3,024. -static PAWN_STRUCTURE: [i32; PAWN_TERMS] = [ - pack(-18, 18), - pack(-25, 15), - pack(-22, 28), - pack(6, 30), - pack(53, 39), - pack(46, 13), - pack(-11, -6), - pack(8, -10), -]; - -/// The pawn structure weight of one of the eight counts, as the packed pair. -/// The tuner's seam asks, so that a slot names the live weight rather than a -/// copy of it, the way it reads the tables. -pub(crate) fn pawn_weight(index: usize) -> i32 { - PAWN_STRUCTURE[index] -} - -/// What one piece leaves on the board, on the scale the taper is read at. -/// The tuner's walk asks, because a position's phase decides what its -/// coefficients are and a copy of the table there would be a second opinion -/// about the taper. -pub(crate) fn phase_weight(piece: Piece) -> i32 { - PHASE_WEIGHTS[piece as usize] -} - -/// How many shelter scores the cache holds. A power of two, so the index is -/// a mask rather than a remainder. -/// -/// Eight thousand entries at sixteen bytes is a hundred and twenty eight -/// kilobytes, which is past the first level cache and inside the second. -/// The size was measured rather than reasoned about. Callgrind over the -/// bench with the cache simulated, at eleven, twelve, thirteen and fourteen -/// bits, reads 3,968,905,639, 3,958,897,319, 3,950,063,019 and 3,943,552,801 -/// instructions against last level misses of 284,996, 285,117, 286,191 and -/// 292,291. Each bit buys fewer instructions than the one before it and the -/// misses are flat until fourteen, where they turn up by six thousand. So -/// this is the last size the memory does not notice, and the whole range is -/// within two thirds of a percent of instructions: the constant is not -/// load bearing and a later working set can move it. -const SHELTER_CACHE_BITS: usize = 13; -const SHELTER_CACHE_SLOTS: usize = 1 << SHELTER_CACHE_BITS; - -/// One remembered shelter score, under the key that decides it. -/// -/// The whole key is kept rather than the bits the index does not use, so a -/// hit is a hit on the position's pawns and kings and not on a tag that -/// happens to agree. A wrong hit here would be a silently wrong evaluation, -/// which is the one error a search does not report, and sixty four bits of -/// key costs four bytes against the alternative. -#[derive(Copy, Clone)] -struct ShelterEntry { - key: u64, - packed: i32, -} - -/// The king shelter, remembered by what it depends on. -/// -/// The term reads both sides' pawns and the two king squares and nothing -/// else, so a position that agrees with a remembered one on those scores the -/// same however its pieces stand. Most moves in a search are piece moves, -/// which leave every one of those alone, so the score computed at one leaf -/// answers a great many of the leaves after it. -/// -/// Direct mapped and never cleared. An entry is only ever read against the -/// key that wrote it, so a stale one is a miss rather than a wrong answer, -/// and a search that begins with the last search's entries begins with a warm -/// cache. That is also what leaves the node counts alone: the cache changes -/// how a score is arrived at and not what it is, so the counts are the ones -/// the weights alone produce. -/// -/// Owned by the searcher rather than by the board, because it is scratch and -/// not position. A board carries what it would take to undo a move and -/// compares equal to another board holding the same position; a cache does -/// neither, and one thread's cache is its own. -pub(crate) struct ShelterCache { - entries: Box<[ShelterEntry]>, -} - -impl Default for ShelterCache { - fn default() -> Self { - // an empty entry is key zero holding a score of zero, so a position - // whose pawns and kings xor to nothing would read it as its own and - // take nothing from it. That is a sixty four bit coincidence, which - // is the same one a wrong hit needs anywhere else in the table - ShelterCache { - entries: vec![ShelterEntry { key: 0, packed: 0 }; SHELTER_CACHE_SLOTS] - .into_boxed_slice(), - } - } -} - -impl ShelterCache { - /// What white's shelter stands ahead by, remembered or computed. - #[inline] - fn shelter(&mut self, board: &Board) -> i32 { - let key = board.shelter_key(); - let slot = (key as usize) & (SHELTER_CACHE_SLOTS - 1); - let entry = &mut self.entries[slot]; - if entry.key == key { - return entry.packed; - } - let packed = shelter(board); - *entry = ShelterEntry { key, packed }; - packed - } -} - -/// How many pawn structure scores the cache holds. A power of two, so the -/// index is a mask rather than a remainder. -/// -/// Four thousand entries at sixteen bytes is sixty four kilobytes, half what -/// the shelter's table takes. The size was measured rather than reasoned -/// about, the way `SHELTER_CACHE_BITS` was, and the measurement was retaken -/// on the fitted tree when the weights arrived, since a different tree is a -/// different working set. Callgrind over the bench with the cache simulated, -/// at eleven, twelve, thirteen and fourteen bits, reads 4,098,418,313, -/// 4,092,752,759, 4,089,023,904 and 4,086,576,985 instructions against last -/// level misses of 287,010, 287,113, 290,125 and 294,220. Each bit buys -/// fewer instructions than the one before it, and the misses are flat from -/// eleven to twelve and then turn up by three thousand at thirteen and four -/// thousand more at fourteen. So twelve is the last size the memory does not -/// notice, and the whole range is inside three tenths of a percent of -/// instructions: the constant is not load bearing and a later working set -/// can move it. The sweep at zero weights, before the fit, put the turn in -/// the same place. -/// -/// Half the shelter's table is what this term's key predicted before either -/// sweep was run. The same pawns under two different pairs of king squares -/// are two entries there and one entry here, so the working set behind this -/// key is the smaller of the two. -/// -/// The bench counts the same 4,066,438 nodes at all four sizes, which is -/// what says the cache changes how a score is arrived at and not what it is. -const PAWN_CACHE_BITS: usize = 12; -const PAWN_CACHE_SLOTS: usize = 1 << PAWN_CACHE_BITS; - -/// One remembered pawn structure score, under the key that decides it. The -/// whole key, for the reason [`ShelterEntry`] keeps the whole of its own. -#[derive(Copy, Clone)] -struct PawnEntry { - key: u64, - packed: i32, -} - -/// The pawn structure, remembered by what it depends on. -/// -/// The term reads the two pawn boards and nothing else, so its key is -/// `Board::pawn_key` with nothing folded in. That is the difference between -/// this cache and the shelter's, and it is the whole reason this term could -/// be cached in the commit that introduced it: the shelter's key carries both -/// kings and so misses on every king move, and this one does not. What misses -/// here is a pawn move and the capture of a pawn, which is every way the two -/// pawn boards change and nothing else. -/// -/// Direct mapped and never cleared, on the terms [`ShelterCache`] sets out. -/// An empty entry is key zero holding zero, and here that is exact rather -/// than a coincidence to be argued about: a board with no pawns has a pawn -/// key of zero, which `a_board_with_no_pawns_has_no_key` pins, and -/// the structure of no pawns is eight zero counts. The one position that -/// reads an empty entry as its own reads the right answer from it. -pub(crate) struct PawnCache { - entries: Box<[PawnEntry]>, -} - -impl Default for PawnCache { - fn default() -> Self { - PawnCache { - entries: vec![PawnEntry { key: 0, packed: 0 }; PAWN_CACHE_SLOTS].into_boxed_slice(), - } - } -} - -impl PawnCache { - /// What white's pawn structure stands ahead by, remembered or computed. - #[inline] - fn pawn_structure(&mut self, board: &Board) -> i32 { - let key = board.pawn_key; - let slot = (key as usize) & (PAWN_CACHE_SLOTS - 1); - let entry = &mut self.entries[slot]; - if entry.key == key { - return entry.packed; - } - let packed = pawn_structure(board); - *entry = PawnEntry { key, packed }; - packed - } -} - -/// The score of the position from the side to move's point of view. -/// -/// Everything incremental is read off the board's accumulator; a term -/// computed at the leaf is added here, from the board itself. -/// -/// The shelter and the pawn structure are computed every time. -/// `eval_cached` is the same score with those two terms remembered, and is -/// what the search asks. The tuner's walk and the instruments ask this one. -/// Neither is hot, and both want a score that depends on the position alone. -/// -/// Material that cannot mate reads zero before any of it. That is the one -/// place in the evaluation that is not a dot product against the weights, -/// which is why the tuner turns such a position away rather than fitting it: -/// see `tune::run`. It sits here rather than at the node because the model -/// gate, the tuner's walk and the instruments all read this function, and a -/// zero returned from the search instead would leave them saying a dead draw -/// is worth a piece. -#[inline] -pub(crate) fn eval(board: &Board) -> Score { - if board.drawn_by_material() { - return 0; - } - board.eval.score( - board.active_color, - mobility(board) + shelter(board) + pawn_structure(board), - ) -} - -/// The same score with the shelter and the pawn structure taken from their -/// caches where they are there, and the same zero on material that cannot -/// mate. -/// -/// Equal to [`eval`] for every position, which is what -/// `the_cache_answers_what_the_full_evaluation_does` holds both of them to. -/// The search calls this and the node counts do not move, because a -/// remembered score is the score that would have been computed. -/// -/// Two caches rather than one wider entry under the shelter's key. The -/// alternative is one probe for both terms, and it would recompute the pawn -/// structure on every king move, which is the half of the shelter's key this -/// term does not need. Which is cheaper is a measurement and not an opinion, -/// and it is made on the fitted build. -#[inline] -pub(crate) fn eval_cached( - board: &Board, - shelter: &mut ShelterCache, - pawns: &mut PawnCache, -) -> Score { - if board.drawn_by_material() { - return 0; - } - board.eval.score( - board.active_color, - mobility(board) + shelter.shelter(board) + pawns.pawn_structure(board), - ) -} - -/// What white's mobility stands ahead by, as a packed pair on the scale the -/// piece square pair is on. -/// -/// Read off the board rather than accumulated. A piece that moves changes what -/// every slider looking through its square sees, so there is nothing here for -/// `Accumulator::count` to add and take away. -/// -/// Only [`SCORED_KINDS`] are counted. A kind whose weight is zero contributes -/// nothing however many squares it covers, so counting it is work no score -/// can see. The rule is what is written down, not a list: when the first fit -/// left six of the eight weights at zero, leaving three of the four kinds out -/// took a bit over a third off what the term cost. The refit priced all four, -/// so today this counts every kind and the skip is waiting for a weight to -/// round to nothing again. -#[inline] -fn mobility(board: &Board) -> i32 { - mobility_with::(board, &MOBILITY) -} - -/// The same fold over the kinds `KINDS` names, against weights named by the -/// caller. A kind outside `KINDS` counts zero and so has to be worth zero. -/// -/// Six of the eight live weights are zero and the other two are equal, so a -/// swapped pair of halves scores every position exactly as the right answer -/// does and a permuted [`MOBILITY`] shows only on the one piece that carries a -/// weight. The tests supply weights of their own through here, over all four -/// kinds, which is what says the sign, the order and the packing are right -/// whatever the fit holds. -#[inline] -fn mobility_with(board: &Board, weights: &[i32; MOBILE_PIECES.len()]) -> i32 { - let white = board.mobility_counts::(Color::White); - let black = board.mobility_counts::(Color::Black); - let mut packed = 0; - for ((weight, white), black) in weights.iter().zip(white).zip(black) { - packed += weight * (white - black); - } - packed -} - -/// What white's king shelter stands ahead by, as a packed pair on the scale -/// the piece square pair is on. -/// -/// Read off the board rather than accumulated. All seven counts are read off -/// the king's square, so a king move rewrites the side's whole reading, and a -/// pawn move changes it wherever the pawn stood. There is nothing here for -/// `Accumulator::count` to add and take away a piece at a time. -#[inline] -fn shelter(board: &Board) -> i32 { - shelter_with(board, &SHELTER) -} - -/// The same fold against weights named by the caller. -/// -/// The live weights are the fit's now, and the two halves of every one of -/// them differ, so the sign of this term, the order of the seven counts and -/// the packing all show in an evaluation the engine prints and in the rows -/// the tuner's walk states. The tests still supply weights of their own -/// through here, because what they pin is the fold rather than the fit: a -/// permuted [`SHELTER`] would be a different evaluation and not a wrong -/// one. -#[inline] -fn shelter_with(board: &Board, weights: &[i32; SHELTER_TERMS]) -> i32 { - let white = board.shelter_counts(Color::White); - let black = board.shelter_counts(Color::Black); - let mut packed = 0; - for ((weight, white), black) in weights.iter().zip(white).zip(black) { - packed += weight * (white - black); - } - packed -} - -/// What white's pawn structure stands ahead by, as a packed pair on the scale -/// the piece square pair is on. -/// -/// Read off the board rather than accumulated. A pawn that moves changes -/// which of the pawns behind and beside it are passed, isolated or doubled, -/// so there is nothing here for `Accumulator::count` to add and take away a -/// pawn at a time. What makes that cheap anyway is that only a pawn move -/// changes it at all, which is what [`PawnCache`] is built on. -#[inline] -fn pawn_structure(board: &Board) -> i32 { - pawn_structure_with(board, &PAWN_STRUCTURE) -} - -/// The same fold against weights named by the caller. -/// -/// The live weights are the fit's now, and no two of the eight pairs agree, -/// so the sign of this term, the order of the eight counts and the packing -/// all show in an evaluation the engine prints. The tests still supply -/// weights of their own, because what they pin is the fold rather than the -/// fit: a permuted [`PAWN_STRUCTURE`] would be a different evaluation and not -/// a wrong one, and a pin written against the shipped numbers would have to -/// be rewritten by every refit. -#[inline] -fn pawn_structure_with(board: &Board, weights: &[i32; PAWN_TERMS]) -> i32 { - let white = board.pawn_structure_counts(Color::White); - let black = board.pawn_structure_counts(Color::Black); - let mut packed = 0; - for ((weight, white), black) in weights.iter().zip(white).zip(black) { - packed += weight * (white - black); - } - packed -} - -/// The evaluation's incremental state, hosted by the board and kept in step -/// by being told about every piece placed, removed and relocated. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub(crate) struct Accumulator { - /// Each side's material, indexed by `Color`'s discriminant the way the - /// weight tables index by `Piece`'s: an index is a load where a match - /// on the colour was a branch. - material: [u32; 2], - /// Piece square table score, as the packed pair of midgame and endgame - /// scores the tables hold: see `psqt::pack`. Summing a boardful of pairs - /// is one add, so carrying both phases costs what carrying one did, and - /// neither half comes near the sixteen bits it has to stay inside. - psqt: i32, - /// What is left on the board, on the scale `PHASE_WEIGHTS` measures, and - /// so which of the two halves above the position is scored by. - /// Accumulated rather than counted off the piece boards at every leaf: - /// four popcounts there measured dearer than one add per piece touched - /// here. - phase: i32, -} - -impl Accumulator { - /// A board with nothing on it scores nothing. - pub(crate) const EMPTY: Self = Self { - material: [0; 2], - psqt: 0, - phase: 0, - }; - - /// Count a piece on to or off of a square. The two directions written - /// once: they are the same arithmetic with every sign reversed, and - /// `SET` is settled at compile time, so no branch on it survives into - /// the search. - #[inline(always)] - pub(crate) fn count(&mut self, index: u8, piece: Piece, color: Color) { - // a packed pair, negated whole for black: negating the sum negates - // both halves, so neither is unpacked until the leaf asks for it - let psqt = match color { - Color::White => PIECE_SQUARE_TABLES.get_value(index as usize, piece, Color::White), - Color::Black => -PIECE_SQUARE_TABLES.get_value(index as usize, piece, Color::Black), - }; - let phase = PHASE_WEIGHTS[piece as usize]; - let value = MATERIAL[piece as usize]; - if SET { - self.psqt += psqt; - self.phase += phase; - self.material[color as usize] += value; - } else { - self.psqt -= psqt; - self.phase -= phase; - self.material[color as usize] -= value; - } - } - - /// A piece moving between two squares, which is `count` off one square - /// and on to the other with the halves that cancel left out. - /// - /// It never leaves the board, so the material and the phase it counts - /// for are the same before and after and their two updates undo each - /// other exactly. What is left is the piece square score, and the pair - /// is added and subtracted whole either way, so a borrow between the two - /// halves cancels here as it does there. - #[inline(always)] - pub(crate) fn relocate(&mut self, from: u8, to: u8, piece: Piece, color: Color) { - let moved = PIECE_SQUARE_TABLES.get_value(to as usize, piece, color) - - PIECE_SQUARE_TABLES.get_value(from as usize, piece, color); - match color { - Color::White => self.psqt += moved, - Color::Black => self.psqt -= moved, - } - } - - /// The accumulator the position deserves, computed from the board rather - /// than accumulated as pieces moved. The hosted one is meant to equal - /// this at all times, which the board's state check asks on every move - /// made. A second implementation on purpose, and only worth having while - /// it stays one: factoring shared code out of this walk and `count` - /// would leave both sides wrong together and the check passing, which is - /// worse than not checking at all. - pub(crate) fn recomputed(board: &Board) -> Self { - let mut recomputed = Self::EMPTY; - // walking the occupied squares rather than all sixty four, an empty - // board is then free rather than sixty four misses - let mut occupied = board.occupied(); - while occupied != 0 { - let index = occupied.trailing_zeros() as u8; - occupied &= occupied - 1; - if let Some((piece, color)) = board.get_piece_and_color_index(index) { - let psqt = PIECE_SQUARE_TABLES.get_value(index as usize, piece, color); - match color { - Color::White => recomputed.psqt += psqt, - Color::Black => recomputed.psqt -= psqt, - } - recomputed.material[color as usize] += MATERIAL[piece as usize]; - recomputed.phase += PHASE_WEIGHTS[piece as usize]; - } - } - recomputed - } - - /// Material seeded from a recount rather than accumulated, which is how - /// `from_fen` fills a parsed board in: the state check then compares the - /// seeding against an implementation that did not do the seeding. - pub(crate) fn seed_material(&mut self, (white, black): (u32, u32)) { - self.material[Color::White as usize] = white; - self.material[Color::Black as usize] = black; - } - - /// What white stands ahead by, for the board's debug print. - pub(crate) fn material_difference(&self) -> i64 { - i64::from(self.material[Color::White as usize]) - - i64::from(self.material[Color::Black as usize]) - } - - /// The score from `side`'s point of view. - /// - /// The piece square half is read at the phase the position is in rather - /// than at either end of it, so that a king walks out as the pieces come - /// off instead of on the move that takes the last one. Material is not - /// tapered: an endgame piece value is the same thing as a constant added - /// to that piece's endgame table, and the tables are the tidier place to - /// say it. - /// - /// `leaf` is the leaf terms [`eval`] reads off the board, mobility, the - /// king's shelter and the pawn structure summed, as a packed pair on the - /// same scale. Summing them before the call is exact, since each is a pair - /// on this scale, and it is what keeps one divide however many such terms - /// there are. The pair joins the piece square pair before the - /// interpolation rather than being tapered beside it, so the two share one - /// divide. A second divide would answer a centipawn away wherever a - /// numerator is negative and does not divide evenly, and `tune::reconstruct` - /// folds a whole row with one. - #[inline] - fn score(&self, side: Color, leaf: i32) -> Score { - // promotions can leave more on the board than the opening had, so the - // phase is capped. It cannot go the other way: no weight is negative. - let phase = self.phase.min(TOTAL_PHASE); - let tapered = self.psqt + leaf; - let scaled = - (mg_value(tapered) * phase + eg_value(tapered) * (TOTAL_PHASE - phase)) / TOTAL_PHASE; - let eval = (self.material[Color::White as usize] as i32 - - self.material[Color::Black as usize] as i32 - + scaled) as Score; - match side { - Color::White => eval, - Color::Black => -eval, - } - } -} - -#[cfg(test)] -mod evaluate { - use super::{ - ALL_KINDS, Board, MOBILE_PIECES, MOBILITY, PAWN_CACHE_SLOTS, PAWN_STRUCTURE, PAWN_TERMS, - PawnCache, SCORED_KINDS, SHELTER, SHELTER_CACHE_SLOTS, SHELTER_TERMS, ShelterCache, - TOTAL_PHASE, eval, eval_cached, mobility, mobility_weight, mobility_with, - pawn_structure_with, shelter_with, - }; - use crate::board::fens; - use crate::misc::{Color, File, coordinate_to_index}; - use crate::psqt::{eg_value, mg_value, pack}; - use pretty_assertions::assert_eq; - use std::collections::{HashMap, HashSet}; - - /// Both the accumulator and its recompute read `PHASE_WEIGHTS`, so the - /// state-in-step check holds them to each other and neither to what the - /// weights should be. This says what they add up to: a full board is the - /// midgame end of the taper, kings and pawns alone the endgame end, and - /// each piece is worth what the interpolation was written expecting. - #[test] - fn a_full_board_is_one_end_of_the_taper_and_a_pawn_ending_the_other() { - assert_eq!(Board::new().eval.phase, TOTAL_PHASE); - assert_eq!( - Board::from_fen("4k3/pppppppp/8/8/8/8/PPPPPPPP/4K3 w - - 0 1") - .unwrap() - .eval - .phase, - 0 - ); - for (fen, phase) in [ - ("4k3/8/8/8/8/8/8/3QK3 w - - 0 1", 4), - ("4k3/8/8/8/8/8/8/3RK3 w - - 0 1", 2), - ("4k3/8/8/8/8/8/8/3BK3 w - - 0 1", 1), - ("4k3/8/8/8/8/8/8/3NK3 w - - 0 1", 1), - ] { - assert_eq!(Board::from_fen(fen).unwrap().eval.phase, phase, "{}", fen); - } - } - - /// After every legal move in the shared positions, the material - /// accumulators must equal a recount, and the score must be the exact - /// negative of the opponent's view of it. - #[test] - fn material_stays_counted_and_the_eval_stays_antisymmetric() { - for fen in fens::CORE { - let mut board = Board::from_fen(fen).unwrap(); - for m in &board.generate_moves() { - if board.make_move(m) { - assert_eq!( - ( - board.eval.material[crate::misc::Color::White as usize], - board.eval.material[crate::misc::Color::Black as usize] - ), - board.material_value(), - "{} in {}", - m, - fen - ); - let score = eval(&board); - board.active_color = !board.active_color; - assert_eq!(score, -eval(&board), "{} in {}", m, fen); - board.active_color = !board.active_color; - board.undo_move(); - } - } - } - } - - /// The assertions above hold whichever way up the piece square tables are, - /// because both colours read them the same way and the symmetry survives. - /// These say which way is up, a colour at a time. - #[test] - fn a_pawn_is_worth_more_the_closer_it_is_to_promoting() { - for (advanced, home) in [ - ( - "4k3/4P3/8/8/8/8/8/4K3 w - - 0 1", - "4k3/8/8/8/8/8/4P3/4K3 w - - 0 1", - ), - ( - "4k3/8/8/8/8/8/4p3/4K3 b - - 0 1", - "4k3/4p3/8/8/8/8/8/4K3 b - - 0 1", - ), - ] { - let advanced = Board::from_fen(advanced).unwrap(); - let home = Board::from_fen(home).unwrap(); - assert!( - eval(&advanced) > eval(&home), - "the advanced pawn scored {} and the one at home {}", - eval(&advanced), - eval(&home) - ); - } - } - - /// Material that cannot mate scores zero from either side, through both - /// entry points. - /// - /// Both, because the search reads the cached one and the tuner, the model - /// gate and the instruments read the other, and a rule in one of them - /// would have the two disagree about the same node. Both caches are - /// passed and neither is read: a drawn position has no pawn on it, so the - /// pawn structure would have contributed nothing had it been reached. - /// - /// The figures in the comments are what the evaluation returned before the - /// rule. The search played every one of them as a win. - #[test] - fn material_that_cannot_mate_scores_zero() { - for fen in [ - "8/8/8/8/8/4k3/8/4K1N1 w - - 0 1", // a knight, read as +320 and a table - "8/8/8/8/8/4k3/8/4K1N1 b - - 0 1", - "8/8/8/8/8/4k3/8/4KB2 w - - 0 1", // a bishop, +320 and a table - "8/8/8/8/8/4k3/8/4KB2 b - - 0 1", - "8/8/8/8/8/4k3/8/4K1NN w - - 0 1", // two knights, +640 and a table - "8/8/8/8/8/4k3/8/4K1NN b - - 0 1", - "4k3/8/8/8/4K3/8/8/8 w - - 0 1", // already zero, by cancellation - "4k3/8/8/8/4K3/8/8/8 b - - 0 1", - "4k3/8/8/8/8/B7/8/2B1K3 w - - 0 1", - "4k3/8/8/8/8/B7/8/2B1K3 b - - 0 1", - "3bk3/8/8/8/8/8/8/2B1K3 w - - 0 1", - "3bk3/8/8/8/8/8/8/2B1K3 b - - 0 1", - ] { - let board = Board::from_fen(fen).unwrap(); - let mut shelter = ShelterCache::default(); - let mut pawns = PawnCache::default(); - assert_eq!(eval(&board), 0, "{}", fen); - assert_eq!(eval_cached(&board, &mut shelter, &mut pawns), 0, "{}", fen); - } - } - - /// The same signatures with a pawn on the board are outside the rule and - /// are scored as they were. - /// - /// Without this, the test above would pass on a rule that answered zero - /// for every pawnless position, or for every position at all. - #[test] - fn a_pawn_takes_a_position_out_of_the_rule() { - for fen in [ - "8/8/8/4p3/8/4k3/8/4K1N1 w - - 0 1", - "8/8/8/4p3/8/4k3/8/4KB2 w - - 0 1", - "8/8/8/4p3/8/4k3/8/4K1NN w - - 0 1", - ] { - let board = Board::from_fen(fen).unwrap(); - let mut shelter = ShelterCache::default(); - let mut pawns = PawnCache::default(); - assert_ne!(eval(&board), 0, "{}", fen); - assert_eq!( - eval_cached(&board, &mut shelter, &mut pawns), - eval(&board), - "{}", - fen - ); - } - } - - /// The point of tapering: the same king on the same square is scored - /// differently depending on what is left on the board. A bare king wants - /// the middle; a king with the pieces still on wants the back rank. - /// - /// Each pair below differs by the king's square and nothing else, material - /// included, so the difference between them is the king's table alone. A - /// phase read the wrong way round would still land inside a pair, so what - /// is pinned is the direction the score moves in as the board empties - /// rather than only that it moves. - #[test] - fn a_king_is_worth_more_in_the_middle_the_emptier_the_board() { - // two king squares, e4 and g1, at three phases - fn centre_over_corner(centre: &str, corner: &str) -> i32 { - i32::from(eval(&Board::from_fen(centre).unwrap())) - - i32::from(eval(&Board::from_fen(corner).unwrap())) - } - let opening = centre_over_corner( - "rnbqkbnr/pppppppp/8/8/4K3/8/PPPPPPPP/RNBQ1B1R w kq - 0 1", - "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQ1BKR w kq - 0 1", - ); - // a rook a side, which is a phase of four out of twenty four - let middlegame = centre_over_corner( - "r3k3/8/8/8/4K3/8/8/R7 w q - 0 1", - "r3k3/8/8/8/8/8/8/R5K1 w q - 0 1", - ); - // a pawn a side, because two bare kings are drawn by material and - // both fens would read zero. A pawn does not count towards the phase, - // so this is still the ending, and the two pawns stand on the same - // squares in both fens and away from either king's files, which - // leaves the difference the king's own square - let ending = centre_over_corner( - "4k3/p7/8/8/4K3/8/P7/8 w - - 0 1", - "4k3/p7/8/8/8/8/P7/6K1 w - - 0 1", - ); - assert!( - opening < middlegame && middlegame < ending, - "e4 over g1 scored {} in the opening, {} with a rook a side and {} bare", - opening, - middlegame, - ending - ); - assert!(opening < 0 && ending > 0, "{} then {}", opening, ending); - } - - /// A position and its reflection, colours swapped, have to score the same - /// for whoever is to move. This does not catch the tables being upside down, - /// since that happens to both colours at once, but it does catch one colour - /// being changed without the other. - #[test] - fn a_mirrored_position_scores_the_same() { - for (white, black) in [ - ( - "4k3/4P3/8/8/8/8/8/4K3 w - - 0 1", - "4k3/8/8/8/8/8/4p3/4K3 b - - 0 1", - ), - ( - "4k3/8/8/8/8/8/8/R3K3 w - - 0 1", - "r3k3/8/8/8/8/8/8/4K3 b - - 0 1", - ), - ( - "rnbqkbnr/pppppppp/8/8/4P3/8/PPPP1PPP/RNBQKBNR b KQkq - 0 1", - "rnbqkbnr/pppp1ppp/8/4p3/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1", - ), - ] { - let white = Board::from_fen(white).unwrap(); - let black = Board::from_fen(black).unwrap(); - assert_eq!(eval(&white), eval(&black), "{} against {}", white, black); - } - } - - /// The position the two tests below are read against, and what each side - /// covers in it, worked out square by square rather than read back off - /// `mobility_counts`. - /// - /// White's knight on b1 has a3, c3 and d2. Its bishop on f1 has g2 and h3, - /// the pawn on e2 standing in the way of the other diagonal. Its rook on - /// a1 has the a file, the knight beside it being neither scope nor - /// something to see through. Its queen on d1 has c1, then c2, b3 and a4, - /// then the d file up to the black queen it takes, less d6, which the - /// black pawn on e7 covers. - /// - /// Black has no knight. Its bishop on c8 has b7 and a6 one way and d7 out - /// to h3 the other. Its rook on h8 has the h file and g8 and f8. Its queen - /// on d8 has c7, b6 and a5, then the d file down to the white queen, less - /// d3, which the white pawn on e2 covers. - const COUNTED: &str = "2bqk2r/4p3/8/8/8/8/4P3/RN1QKB2 w - - 0 1"; - const WHITE_COVERS: [i32; MOBILE_PIECES.len()] = [3, 2, 7, 10]; - const BLACK_COVERS: [i32; MOBILE_PIECES.len()] = [0, 7, 9, 9]; - - /// Four weights that differ from each other at both ends of the taper, so - /// that a pair read into the wrong piece's slot lands on a different - /// number. The four differences are 3, -5, -2 and 1, which differ from - /// each other too, so a permutation of either array shows. - const TRIAL: [i32; MOBILE_PIECES.len()] = - [pack(11, 2), pack(-7, 13), pack(3, -5), pack(29, 41)]; - - /// What the fold does with weights that are not zero. - /// - /// Six of the shipped weights are zero and the other two are equal, so - /// almost nothing about the sign, the order of the four pieces or the - /// packing changes an evaluation the engine prints, and a swapped pair of - /// halves changes none of them at all. This hands the fold weights of its - /// own and asserts the packed pair against the arithmetic: white's count - /// less black's, piece by piece, each half of the pair summed on its own. - #[test] - fn the_mobility_fold_reads_white_less_black_piece_by_piece() { - let board = Board::from_fen(COUNTED).unwrap(); - assert_eq!( - board.mobility_counts::<{ ALL_KINDS }>(Color::White), - WHITE_COVERS - ); - assert_eq!( - board.mobility_counts::<{ ALL_KINDS }>(Color::Black), - BLACK_COVERS - ); - let midgame: i32 = (0..MOBILE_PIECES.len()) - .map(|i| mg_value(TRIAL[i]) * (WHITE_COVERS[i] - BLACK_COVERS[i])) - .sum(); - let endgame: i32 = (0..MOBILE_PIECES.len()) - .map(|i| eg_value(TRIAL[i]) * (WHITE_COVERS[i] - BLACK_COVERS[i])) - .sum(); - assert_ne!( - midgame, endgame, - "the two halves would not tell a swap apart" - ); - let packed = mobility_with::<{ ALL_KINDS }>(&board, &TRIAL); - assert_ne!(midgame, 0, "black less white would answer the same here"); - assert_eq!((mg_value(packed), eg_value(packed)), (midgame, endgame)); - } - - /// What `eval` is allowed to leave out, which is the whole of the contract - /// between it and the tuner's walk. - /// - /// The walk counts all four kinds and `eval` counts [`SCORED_KINDS`], so - /// the two no longer read one answer. What makes that safe is the size of - /// the difference and nothing else: a count multiplied by zero adds - /// nothing, so a kind worth zero can go uncounted without moving a score, - /// and any other kind cannot. So this asserts the difference is exactly - /// that, kind by kind, and then that the two fold to the same packed pair - /// at the shipped weights. - /// - /// A later fit that prices one of the three kinds `eval` skips today fails - /// here, rather than being silently thrown away at every leaf and every - /// quiescence node. - /// - /// The position has to give every kind of both colours something to cover, - /// or a kind that is skipped and a kind that covers nothing read the same - /// and the test passes without having looked at anything. - #[test] - fn eval_counts_a_kind_exactly_when_its_weight_is_not_zero() { - let board = Board::from_fen(fens::KIWIPETE).unwrap(); - for color in [Color::White, Color::Black] { - let all = board.mobility_counts::<{ ALL_KINDS }>(color); - let scored = board.mobility_counts::<{ SCORED_KINDS }>(color); - for (index, piece) in MOBILE_PIECES.into_iter().enumerate() { - assert_ne!( - all[index], 0, - "{:?} covers nothing here, so this position cannot tell a skipped kind \ - from a counted one", - piece - ); - let weight = mobility_weight(index); - let priced = mg_value(weight) != 0 || eg_value(weight) != 0; - assert_eq!( - scored[index], - if priced { all[index] } else { 0 }, - "{:?} is worth {} in the midgame and {} in the ending, so it should be \ - {}, and eval counted {} of its {} squares", - piece, - mg_value(weight), - eg_value(weight), - if priced { "counted" } else { "skipped" }, - scored[index], - all[index] - ); - } - } - let whole = mobility_with::<{ ALL_KINDS }>(&board, &MOBILITY); - assert_ne!( - whole, 0, - "mobility is level here, so this position says nothing about the fold" - ); - assert_eq!(mobility(&board), whole); - } - - /// Mobility enters the numerator of the one interpolation rather than - /// being tapered beside it. - /// - /// A position whose piece square numerator is negative and does not divide - /// by twenty four evenly, so the two readings differ: inside the divide - /// the whole numerator is truncated once, and a second divide would - /// truncate each part on its own and answer a centipawn away. Nothing else - /// pins this, and it is what `tune::reconstruct` folds a row with. - #[test] - fn mobility_joins_the_numerator_rather_than_being_tapered_beside_it() { - let board = Board::from_fen("4k3/8/8/8/8/8/4P3/1N2K3 w - - 0 1").unwrap(); - let accumulator = board.eval; - let phase = accumulator.phase.min(TOTAL_PHASE); - let mobility = pack(11, -1); - let together = accumulator.psqt + mobility; - let inside = - (mg_value(together) * phase + eg_value(together) * (TOTAL_PHASE - phase)) / TOTAL_PHASE; - let beside = (mg_value(accumulator.psqt) * phase - + eg_value(accumulator.psqt) * (TOTAL_PHASE - phase)) - / TOTAL_PHASE - + (mg_value(mobility) * phase + eg_value(mobility) * (TOTAL_PHASE - phase)) - / TOTAL_PHASE; - assert_ne!( - inside, beside, - "the two divides agree here, so this position says nothing" - ); - let material = accumulator.material[Color::White as usize] as i32 - - accumulator.material[Color::Black as usize] as i32; - assert_eq!( - accumulator.score(Color::White, mobility), - (material + inside) as crate::misc::Score - ); - } - /// The position the test below is read against, and what each side counts - /// in it, worked out by hand rather than read back off `shelter_counts`. - /// - /// White's king on g1 stands behind the f, g and h files. It has f2 and h2 - /// one rank ahead and g3 two, and all three files hold a pawn of its own, - /// so neither file count fires. Coming the other way it faces g2 one rank - /// ahead, f3 and h3 two, and f4, g4 and h4 three. - /// - /// Black's king on b8 stands behind the a, b and c files with nothing on - /// either rank in front of it and no white pawn within three ranks, so its - /// storm is empty. White's pawn on a4 leaves that file half open and the b - /// and c files hold no pawn at all. - /// - /// The seven differences are 2, 1, -2, -1, 1, 2 and 3. None is zero, so - /// every slot is doing work in the assertion below, which a position with - /// an empty storm would not manage. - const SHELTERED: &str = "1k6/8/8/8/P4ppp/5pPp/5PpP/6K1 w - - 0 1"; - const WHITE_SHELTERS: [i32; SHELTER_TERMS] = [2, 1, 0, 0, 1, 2, 3]; - const BLACK_SHELTERS: [i32; SHELTER_TERMS] = [0, 0, 2, 1, 0, 0, 0]; - - /// Seven weights that differ from each other at both ends of the taper, so - /// that a pair read into the wrong count's slot lands on a different - /// number. The seven differences between the halves are 9, -20, 8, -12, - /// 20, -29 and -14, which differ from each other too, so a permutation of - /// either array shows. - const SHELTER_TRIAL: [i32; SHELTER_TERMS] = [ - pack(11, 2), - pack(-7, 13), - pack(3, -5), - pack(29, 41), - pack(17, -3), - pack(-23, 6), - pack(5, 19), - ]; - - /// What the fold does with weights that are not the shipped ones. - /// - /// The shipped weights would do here now that they are not zero. Weights - /// of this test's own are kept anyway, because the shipped ones are the - /// fit's and will move again: a pin written against them would have to - /// be rewritten by every refit, and what it is pinning is the fold. So - /// this hands the fold seven pairs that differ from each other at both - /// ends and asserts the packed pair against the arithmetic: white's count - /// less black's, count by count, each half of the pair summed on its - /// own. - #[test] - fn the_shelter_fold_reads_white_less_black_count_by_count() { - let board = Board::from_fen(SHELTERED).unwrap(); - assert_eq!(board.shelter_counts(Color::White), WHITE_SHELTERS); - assert_eq!(board.shelter_counts(Color::Black), BLACK_SHELTERS); - let midgame: i32 = (0..SHELTER_TERMS) - .map(|i| mg_value(SHELTER_TRIAL[i]) * (WHITE_SHELTERS[i] - BLACK_SHELTERS[i])) - .sum(); - let endgame: i32 = (0..SHELTER_TERMS) - .map(|i| eg_value(SHELTER_TRIAL[i]) * (WHITE_SHELTERS[i] - BLACK_SHELTERS[i])) - .sum(); - assert_ne!( - midgame, endgame, - "the two halves would not tell a swap apart" - ); - let packed = shelter_with(&board, &SHELTER_TRIAL); - assert_ne!(midgame, 0, "black less white would answer the same here"); - assert_eq!((mg_value(packed), eg_value(packed)), (midgame, endgame)); - } - - /// The position the pawn structure test below is read against, and what - /// each side counts in it, worked out by hand rather than read back off - /// `pawn_structure_counts`. - /// - /// The two sides are kept to files that do not meet, white on a, b and d - /// and black on e, f, g and h, so no pawn of one colour stands in any - /// pawn of the other's way and every passer here is passed for a reason - /// a reader can check in one line. - /// - /// White has a7, a3, b3, b2, d5 and d4. a7 is passed on the seventh and - /// a3 is not, because a7 is ahead of it on the file; b3 is passed on the - /// third and b2 is not; d5 is passed on the fifth and d4 is not. The d - /// file has no white pawn beside it, so d5 and d4 are the two isolated - /// pawns. a7, b3 and d5 each have a pawn of their own behind them on - /// their file, which is the three doubled. - /// - /// Black has f7, g5, e3 and h3, one to a file and all four passed: f7 on - /// its relative second, g5 on its relative fourth, and e3 and h3 on its - /// relative sixth. The four files run together, so nothing is isolated, - /// and no file holds two, so nothing is doubled. - /// - /// The eight differences are -1, 1, -1, 1, -2, 1, 2 and 3. None is zero, - /// so every slot does work in the assertion below. They are not all - /// distinct, so this alone would not tell the first count from the third; - /// what tells those apart is - /// `every_pawn_count_writes_both_ends_of_the_taper` in tune.rs, which - /// reads the coefficients bucket by bucket. - const STRUCTURED: &str = "3k4/P4p2/8/3P2p1/3P4/PP2p2p/1P6/6K1 w - - 0 1"; - const WHITE_STRUCTURE: [i32; PAWN_TERMS] = [0, 1, 0, 1, 0, 1, 2, 3]; - const BLACK_STRUCTURE: [i32; PAWN_TERMS] = [1, 0, 1, 0, 2, 0, 0, 0]; - - /// Eight weights that differ from each other at both ends of the taper, - /// so that a pair read into the wrong count's slot lands on a different - /// number. The eight differences between the halves are 38, 20, -24, -30, - /// 36, 25, -26 and -28, which differ from each other too, so a - /// permutation of either array shows. - const PAWN_TRIAL: [i32; PAWN_TERMS] = [ - pack(3, 41), - pack(-7, 13), - pack(29, 5), - pack(11, -19), - pack(17, 53), - pack(-23, 2), - pack(-5, -31), - pack(37, 9), - ]; - - /// What the fold does with weights that are not the shipped ones. - /// - /// The shipped weights would do here now that they are not zero. Weights - /// of this test's own are kept anyway, because the shipped ones are the - /// fit's and will move again: a pin written against them would have to be - /// rewritten by every refit, and what it is pinning is the fold. So this - /// hands the fold eight pairs that differ from each other at both ends - /// and asserts the packed pair against the arithmetic: white's count less - /// black's, count by count, each half of the pair summed on its own. - #[test] - fn the_pawn_structure_fold_reads_white_less_black_count_by_count() { - let board = Board::from_fen(STRUCTURED).unwrap(); - assert_eq!(board.pawn_structure_counts(Color::White), WHITE_STRUCTURE); - assert_eq!(board.pawn_structure_counts(Color::Black), BLACK_STRUCTURE); - let midgame: i32 = (0..PAWN_TERMS) - .map(|i| mg_value(PAWN_TRIAL[i]) * (WHITE_STRUCTURE[i] - BLACK_STRUCTURE[i])) - .sum(); - let endgame: i32 = (0..PAWN_TERMS) - .map(|i| eg_value(PAWN_TRIAL[i]) * (WHITE_STRUCTURE[i] - BLACK_STRUCTURE[i])) - .sum(); - assert_ne!( - midgame, endgame, - "the two halves would not tell a swap apart" - ); - let packed = pawn_structure_with(&board, &PAWN_TRIAL); - assert_ne!(midgame, 0, "black less white would answer the same here"); - assert_eq!((mg_value(packed), eg_value(packed)), (midgame, endgame)); - } - - /// The two caches and the keys a walk saw through them. One thing to - /// carry rather than four, and the keys are what say whether the run - /// evicted anything. - #[derive(Default)] - struct Caches { - shelter: ShelterCache, - pawns: PawnCache, - shelter_keys: HashSet, - pawn_keys: HashSet, - counted: HashMap, - compared: usize, - } - - impl Caches { - /// What the pawn key claims, checked against what the counts say. - /// - /// The cache hands a remembered score to every position whose pawn - /// key it matches, so the key has to decide the counts. The identity - /// below sees a wrong key now that the weights are the fit's, and it - /// sees it as a wrong score; this says the same thing one step - /// earlier and in the terms the term is defined in. Two positions - /// under one key that disagree on the eight counts are one position - /// handed the other's score, whatever the weights make that worth. - fn note(&mut self, board: &Board) { - let counts = [ - board.pawn_structure_counts(Color::White), - board.pawn_structure_counts(Color::Black), - ]; - if let Some(seen) = self.counted.insert(board.pawn_key, counts) { - self.compared += 1; - assert_eq!( - seen, - counts, - "two positions share a pawn key and not its counts, at {}", - board.to_fen() - ); - } - } - } - - /// How many of `keys` there are and how many slots of a table of `slots` - /// they land in. The second being the smaller is what says two keys - /// shared a slot, so an entry was written over rather than only written. - fn filled(keys: &HashSet, slots: usize) -> (usize, usize) { - let landed: HashSet = keys - .iter() - .map(|key| (*key as usize) & (slots - 1)) - .collect(); - (keys.len(), landed.len()) - } - - /// Every position reachable inside `budget` moves of `board`, scored both - /// ways through the two caches. - fn walk(board: &Board, depth: usize, caches: &mut Caches, budget: &mut usize) { - if depth == 0 || *budget == 0 { - return; - } - for m in &board.generate_moves() { - if *budget == 0 { - break; - } - let mut played = board.clone(); - if !played.make_move(m) { - continue; - } - *budget -= 1; - caches.shelter_keys.insert(played.shelter_key()); - caches.pawn_keys.insert(played.pawn_key); - caches.note(&played); - assert_eq!( - eval_cached(&played, &mut caches.shelter, &mut caches.pawns), - eval(&played), - "a cache and the evaluation part company at {}", - played.to_fen() - ); - walk(&played, depth - 1, caches, budget); - } - } - - /// The cache answers what the full evaluation does, position for - /// position. - /// - /// A shelter score is remembered under the pawns and the two king - /// squares, and every other piece is outside that key, so a key missing - /// something the term reads would hand one position's shelter to - /// another. Nothing in a search would say so: the score is simply not - /// the position's, and a wrong evaluation is the one error it does not - /// report. - /// - /// A walk that never evicted would be testing a cache that only ever - /// grows, so the run has to overwrite entries as well as write and read - /// them. Counting the positions does not say it happened: the walk - /// revisits keys, and far fewer keys than positions reach the table. What - /// says it is the keys against the slots they land in, so the test - /// collects the keys and asserts that two of them shared a slot. - /// - /// `Caches::note` carries the same claim in the counts rather than in - /// the score, which is what it was written to do while the sixteen - /// weights were zero and the score said nothing. It is kept now that they - /// are fitted, because a count is a sharper thing to compare than a sum - /// of sixteen products: two wrong counts whose weights happen to cancel - /// would pass the score and fail the note. - #[test] - fn the_cache_answers_what_the_full_evaluation_does() { - let mut caches = Caches::default(); - let mut budget = 4 * SHELTER_CACHE_SLOTS.max(PAWN_CACHE_SLOTS); - for fen in fens::CORE { - let board = Board::from_fen(fen).unwrap(); - assert_eq!( - eval_cached(&board, &mut caches.shelter, &mut caches.pawns), - eval(&board), - "{}", - fen - ); - caches.shelter_keys.insert(board.shelter_key()); - caches.pawn_keys.insert(board.pawn_key); - caches.note(&board); - walk(&board, 3, &mut caches, &mut budget); - } - assert!( - caches.compared > 0, - "every pawn key here was seen once, so no two positions were held against each other" - ); - for (name, keys, slots) in [ - ("shelter", &caches.shelter_keys, SHELTER_CACHE_SLOTS), - ("pawn", &caches.pawn_keys, PAWN_CACHE_SLOTS), - ] { - let (keys, landed) = filled(keys, slots); - assert!( - keys > landed, - "{} keys over {} slots of the {} cache, so no slot was written twice and nothing was evicted", - keys, - landed, - name - ); - } - } - - /// A remembered score is read back rather than recomputed, which is the - /// whole point of the cache and is not visible in what it answers. - /// - /// Asked twice for one position, the second answer comes from the entry - /// the first wrote. There is no counter to read, so this says it the way - /// a caller could: the entry the key lands on holds the score after the - /// first call, and holds it under that key and no other. - #[test] - fn a_score_is_remembered_under_the_key_that_wrote_it() { - let board = Board::from_fen(fens::MIDDLEGAME).unwrap(); - let mut caches = Caches::default(); - let key = board.shelter_key(); - let slot = (key as usize) & (SHELTER_CACHE_SLOTS - 1); - assert_eq!(caches.shelter.entries[slot].key, 0, "the slot starts empty"); - let first = eval_cached(&board, &mut caches.shelter, &mut caches.pawns); - assert_eq!(caches.shelter.entries[slot].key, key); - assert_eq!( - caches.shelter.entries[slot].packed, - shelter_with(&board, &SHELTER) - ); - assert_eq!( - eval_cached(&board, &mut caches.shelter, &mut caches.pawns), - first - ); - } - - /// The same for the pawn structure, under the pawn key rather than that - /// key with the two kings folded in. - /// - /// What this reads is the key as much as the score: an entry written - /// under the position's pawn key is the entry the next position with - /// those pawns is handed. The score is now the fit's rather than zero, so - /// the entry holding it says something too. - #[test] - fn a_pawn_structure_is_remembered_under_the_pawn_key() { - let board = Board::from_fen(fens::MIDDLEGAME).unwrap(); - let mut caches = Caches::default(); - let key = board.pawn_key; - assert_ne!(key, 0, "an empty entry would answer this one correctly"); - let slot = (key as usize) & (PAWN_CACHE_SLOTS - 1); - assert_eq!(caches.pawns.entries[slot].key, 0, "the slot starts empty"); - let first = eval_cached(&board, &mut caches.shelter, &mut caches.pawns); - assert_eq!(caches.pawns.entries[slot].key, key); - assert_eq!( - caches.pawns.entries[slot].packed, - pawn_structure_with(&board, &PAWN_STRUCTURE) - ); - assert_eq!( - eval_cached(&board, &mut caches.shelter, &mut caches.pawns), - first - ); - } - - /// A king move leaves the pawn key alone and moves the shelter key, so - /// the entry this term wrote answers the position after it and the - /// shelter's does not. - /// - /// The difference between the two caches, stated rather than implied. It - /// is why this term could be cached in the commit that introduced it - /// where the shelter's cache had to wait for a fit to show it was needed. - #[test] - fn a_king_move_keeps_the_pawn_entry_and_loses_the_shelter_one() { - let board = Board::from_fen("4k3/pppppppp/8/8/8/8/PPPPPPPP/4K3 w - - 0 1").unwrap(); - let from = coordinate_to_index(1, File::E); - let king = board - .generate_moves() - .into_iter() - .find(|m| m.from == from) - .expect("the king on e1 has a move here"); - let mut moved = board.clone(); - assert!(moved.make_move(&king)); - assert_eq!(moved.pawn_key, board.pawn_key); - assert_ne!(moved.shelter_key(), board.shelter_key()); - } -} diff --git a/arche-core/src/eval/mobility.rs b/arche-core/src/eval/mobility.rs new file mode 100644 index 00000000..3d79df2f --- /dev/null +++ b/arche-core/src/eval/mobility.rs @@ -0,0 +1,460 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +// Copyright (C) 2022-2026 Andrew Wright + +//! How many squares each side's pieces cover, and what a square is worth. +//! +//! The term owns its counts, its weights and its fold. Nothing here is +//! remembered between positions: a piece that moves changes what every slider +//! looking through its square sees, so there is no key a score could sit +//! behind and nothing for `Accumulator::count` to add and take away. + +use crate::board::{Board, knight_attacks, pawn_attacks, pop_lsb}; +use crate::magic::MAGIC; +use crate::misc::{Color, Piece}; +use crate::psqt::{eg_value, mg_value, pack}; + +/// The pieces that carry a mobility weight, in the order [`MOBILITY`] and +/// [`counts`] are indexed by. The pawn and the king carry none. +pub(crate) const PIECES: [Piece; 4] = [Piece::Knight, Piece::Bishop, Piece::Rook, Piece::Queen]; + +/// How many counts the term is measured in, which is one per piece that +/// carries a weight. +pub(crate) const COUNTS: usize = PIECES.len(); + +/// What one square of scope is worth to each of [`PIECES`], as the packed +/// pairs the taper is read from. +/// +/// Refitted 2026-09-13 by `scripts/tune.py` over the whole archived strength +/// run: 259 artifacts across 54 runs, 43,675 games, whose 5,694,775 post-book +/// plies gave 5,355,792 positions and 2,461,322 quiet rows, extracted by +/// `arche terms` at 2105b55. The corpus is sha256 `b37ebf0c` and the rows are +/// sha256 `49aa715a`. K was held at 1.1350 and the games split by pair: +/// 12,743 pairs trained, 4,339 chose the ridge and 4,747 were sealed. The 768 +/// table entries, the six material values, the fourteen shelter weights and +/// the sixteen pawn structure ones were all held, so these eight are the only +/// thing that moved. Every number here is of the rounded vector that ships. +/// +/// The selection group scores 0.087671 at the old weights and 0.086874 at +/// these, a paired difference of -0.000798 against a standard error of +/// 0.000113 at a design factor of 3.8. The sealed group, opened once +/// afterwards over 4,747 pairs and 9,452 games that no fit and no ridge +/// choice had read, scores 0.080507 and 0.079997, a paired difference of +/// -0.000510 against a standard error of 0.000107 at a design factor of 4.1. +/// Both are outside their intervals, and they are 1.85 standard errors apart, +/// so the selection group overstates this fit by about a third. The king +/// safety fit's two agreed to 0.68 and the pawn structure fit's to 0.15. +/// +/// The first fit of this term read 1,812 games and priced one piece. This one +/// reads twenty four times as many and prices all four, which is the whole of +/// what changed: nothing about the term is different, the corpus simply has +/// enough games to say something about a knight. A preliminary fit taken +/// before the pawn structure term existed predicted `[5,6,5,1]` and +/// `[0,2,2,7]`, and it came out `[4,6,4,1]` and `[1,3,4,7]`. Pawn structure +/// took almost nothing away from mobility, despite both terms reading open +/// lines. +/// +/// The ridge is zero, which the grid ranked first, and here that is not the +/// trap it was for the pawn structure fit. The vector quantizes identically +/// at zero, 1e-8 and 1e-7, the largest weight is 7, and every one of the +/// eight slots carries a coefficient in 37% to 70% of the training rows with +/// summed coefficients in the tens of millions. There is no low leverage +/// direction here for an unregularised fit to hide error in. +/// +/// By phase the sealed reading is -0.000160 at six pieces or fewer, +/// -0.001725 from seven to twelve, and +0.000703 at thirteen or more. The +/// term pays in the middlegame, barely in the ending, and costs something in +/// the opening, which is the phase a mobility count would be expected to earn +/// most. That is worth an ablation and is not one this arm ran. +/// +/// Every weight is non-zero, so [`SCORED_KINDS`] has nothing left to skip and +/// all four kinds are counted at every leaf again. What that costs is in the +/// commit that landed this, and it is the larger half of the arm. +/// +/// A count is at most twenty seven for a queen and a boardful comes to a few +/// hundred, so a weight in single figures leaves the same order of magnitude +/// in hand that `pack` asks for. +const MOBILITY: [i32; COUNTS] = [pack(4, 1), pack(6, 3), pack(4, 4), pack(1, 7)]; + +/// The weight of one of the four pieces that carries one, as the packed pair. +/// The tuner's seam asks through [`super::TERMS`], so that a slot names the +/// live weight rather than a copy of it, the way it reads the tables. +pub(crate) const fn weight(index: usize) -> i32 { + MOBILITY[index] +} + +/// A set of [`PIECES`], a bit per index, which is what [`counts_of`] takes. A +/// kind left out of the set is not counted and answers zero. +/// +/// This is all four of them. The tuner's walk asks for it whatever the +/// weights hold, because that walk is offline and its coefficients are what +/// lets a later fit price a kind that is worth nothing today. +pub(crate) const ALL_KINDS: u8 = (1 << COUNTS) - 1; + +/// The kinds [`super::eval`] counts: the ones whose [`MOBILITY`] weight is not +/// zero at one end of the taper or the other. The 2026-09-13 refit priced all +/// eight weights, so this is now all four kinds and skips nothing. +/// +/// Derived from the weights rather than written out, which is what made the +/// refit start counting three kinds again rather than leaving them ignored at +/// every leaf. It cost what the refit's commit records. +/// `eval_counts_a_kind_exactly_when_its_weight_is_not_zero` is what holds the +/// two together. +pub(crate) const SCORED_KINDS: u8 = scored_kinds(); + +/// Whether `kinds` names the piece at `index` in [`PIECES`]. +pub(crate) const fn counted(kinds: u8, index: usize) -> bool { + kinds & (1 << index) != 0 +} + +/// [`SCORED_KINDS`], read off the weights at compile time. Both halves are +/// asked about: a weight worth nothing in the midgame and something in the +/// ending is still a weight and still has to be counted. +const fn scored_kinds() -> u8 { + let mut kinds = 0; + let mut index = 0; + while index < COUNTS { + let weight = weight(index); + if mg_value(weight) != 0 || eg_value(weight) != 0 { + kinds |= 1 << index; + } + index += 1; + } + kinds +} + +/// How many squares this side's knights, bishops, rooks and queens cover, a +/// count per piece kind in that order, which is the order [`PIECES`] names +/// them in. `KINDS` says which of the four to count; the rest are not looked +/// at and answer zero. +/// +/// A piece's count is its attack set over the real occupancy, less the +/// squares this side stands on, less the squares an enemy pawn attacks. So +/// a friendly piece blocks a slider rather than being seen through, and a +/// square an enemy pawn covers is not somewhere a piece goes. An enemy +/// piece standing on a square keeps that square in the count, because +/// attacking it is the point. Pins are ignored: a pinned bishop counts its +/// squares, and the search is what knows it cannot move. +/// +/// The enemy pawns are taken as one span rather than probed a square at a +/// time. The king and the pawn have no count of their own: a king's is a +/// danger signal rather than a scope one, and a pawn's is move generation. +/// +/// Both [`super::eval`] and the tuner's walk read this, and they may ask for +/// different kinds. The walk always asks for all four, because it is offline +/// and its coefficients are what prices a kind; the evaluation asks for the +/// kinds whose weight is not zero, because a count multiplied by zero is not +/// worth the leaf it is taken at. Since the refit priced all four the two +/// sets are equal today. What keeps them honest whether or not they are is +/// that the difference is exactly the zero weights, which +/// `eval_counts_a_kind_exactly_when_its_weight_is_not_zero` pins. Within one +/// set of kinds the counts are still one answer rather than two, so a second +/// implementation of them would still be two chances to be wrong rather than a +/// check on one, and the hand counts below are what pins them. That is the +/// exception to the rule `tune.rs` states in its header, which names it. +/// +/// `KINDS` is a compile time set, so a kind left out of it costs nothing: +/// its loop is not compiled rather than skipped. +/// +/// Inlined by force. Left to itself llvm keeps this out of line even under +/// link time optimisation, and the evaluation asks for it twice at every leaf +/// and every quiescence node. That call was three fifths of what the term cost +/// over the bench: 4.30 billion instructions without the attribute against +/// 3.76 billion with it. +#[inline(always)] +pub(crate) fn counts_of(board: &Board, color: Color) -> [i32; COUNTS] { + let occupied = board.occupied(); + let (ours, theirs) = board.sides(color); + let scope = !(ours | pawn_attacks(board.pawns() & theirs, !color)); + let magic = &MAGIC; + let mut counts = [0; COUNTS]; + if counted(KINDS, 0) { + let mut knights = board.knights() & ours; + while knights != 0 { + let from = pop_lsb(&mut knights); + counts[0] += (knight_attacks(from) & scope).count_ones() as i32; + } + } + if counted(KINDS, 1) { + let mut bishops = board.bishops() & ours; + while bishops != 0 { + let from = pop_lsb(&mut bishops); + counts[1] += (magic.get_diagonal_move(from, occupied) & scope).count_ones() as i32; + } + } + if counted(KINDS, 2) { + let mut rooks = board.rooks() & ours; + while rooks != 0 { + let from = pop_lsb(&mut rooks); + counts[2] += (magic.get_straight_move(from, occupied) & scope).count_ones() as i32; + } + } + if counted(KINDS, 3) { + let mut queens = board.queens() & ours; + while queens != 0 { + let from = pop_lsb(&mut queens); + let attacks = + magic.get_straight_move(from, occupied) | magic.get_diagonal_move(from, occupied); + counts[3] += (attacks & scope).count_ones() as i32; + } + } + counts +} + +/// All four counts, written into `into`, which is what [`super::TERMS`] hands +/// the tuner's walk. +pub(crate) fn counts(board: &Board, color: Color, into: &mut [i32]) { + into.copy_from_slice(&counts_of::(board, color)); +} + +/// What white's mobility stands ahead by, as a packed pair on the scale the +/// piece square pair is on. +/// +/// Only [`SCORED_KINDS`] are counted. A kind whose weight is zero contributes +/// nothing however many squares it covers, so counting it is work no score +/// can see. The rule is what is written down, not a list: when the first fit +/// left six of the eight weights at zero, leaving three of the four kinds out +/// took a bit over a third off what the term cost. The refit priced all four, +/// so today this counts every kind and the skip is waiting for a weight to +/// round to nothing again. +#[inline] +pub(crate) fn fold(board: &Board) -> i32 { + fold_with::(board, &MOBILITY) +} + +/// The same fold over the kinds `KINDS` names, against weights named by the +/// caller. A kind outside `KINDS` counts zero and so has to be worth zero. +/// +/// The live weights are the fit's now and no two of the four pairs agree, so +/// the sign of this term, the order of the four pieces and the packing all +/// show in an evaluation the engine prints. The tests supply weights of their +/// own through here anyway, over all four kinds, because what they pin is the +/// fold rather than the fit: a permuted [`MOBILITY`] would be a different +/// evaluation and not a wrong one. +#[inline] +fn fold_with(board: &Board, weights: &[i32; COUNTS]) -> i32 { + let white = counts_of::(board, Color::White); + let black = counts_of::(board, Color::Black); + let mut packed = 0; + for ((weight, white), black) in weights.iter().zip(white).zip(black) { + packed += weight * (white - black); + } + packed +} + +#[cfg(test)] +mod tests { + use super::{ALL_KINDS, Board, COUNTS, Color, MOBILITY, PIECES, SCORED_KINDS, counts_of, fold}; + use crate::board::fens; + use crate::psqt::{eg_value, mg_value, pack}; + use pretty_assertions::assert_eq; + + /// The counts by hand, square by square, because nothing else pins them. + /// The tuner's identity folds a row against the live weights, and `eval` + /// and the walk read this same function, so the two sides of the identity + /// move together whatever it answers. These cases are the only check this + /// term has. + /// + /// Each case names what the count is made of. The two kings stand in + /// opposite corners and out of the way, so that nothing here is a count of + /// theirs and no piece is placed giving check. + #[test] + fn a_piece_covers_what_a_hand_count_says_it_does() { + for (fen, counts, why) in [ + // a knight in the corner has two squares and one in the middle + // has all eight + ( + "k7/8/8/8/8/8/8/N6K w - - 0 1", + [2, 0, 0, 0], + "a knight on a1", + ), + ( + "k7/8/8/8/3N4/8/8/7K w - - 0 1", + [8, 0, 0, 0], + "a knight on d4", + ), + // rays of three, four, three and three + ( + "k7/8/8/8/3B4/8/8/7K w - - 0 1", + [0, 13, 0, 0], + "a bishop on d4", + ), + // a rank and a file, less the square it stands on + ( + "k7/8/8/8/3R4/8/8/7K w - - 0 1", + [0, 0, 14, 0], + "a rook on d4", + ), + ( + "k7/8/8/8/3Q4/8/8/7K w - - 0 1", + [0, 0, 0, 27], + "a queen on d4", + ), + // the friendly pawn on d6 is not scope and is not seen through + // either, so the file gives d5, d3, d2 and d1 beside the rank + ( + "k7/8/3P4/8/3R4/8/8/7K w - - 0 1", + [0, 0, 11, 0], + "a rook on d4 behind its own pawn", + ), + // the pawn on b7 covers c6, which is one of the knight's eight + ( + "k7/1p6/8/8/3N4/8/8/7K w - - 0 1", + [7, 0, 0, 0], + "a knight on d4 against a pawn on b7", + ), + // the enemy rook stands on one of the same eight and keeps it: + // a square with something to take on it is still scope + ( + "k7/8/2r5/8/3N4/8/8/7K w - - 0 1", + [8, 0, 0, 0], + "a knight on d4 against a rook on c6", + ), + // the enemy pawn stops the file at d6 and the ray does not carry + // past it, so the file gives d5 and d6 beside the rank. This is + // the case that says the magic lookup is asked about the whole + // occupancy and not about this side's half of it + ( + "8/2k5/3p4/8/3R4/8/8/6K1 w - - 0 1", + [0, 0, 12, 0], + "a rook on d4 in front of an enemy pawn", + ), + ] { + let board = Board::from_fen(fen).unwrap(); + assert_eq!( + counts_of::<{ ALL_KINDS }>(&board, Color::White), + counts, + "{}", + why + ); + } + } + + /// Black's count of a position is white's count of its reflection, so the + /// two colours are read the same way round. + #[test] + fn the_two_colours_count_the_same_squares() { + let white = Board::from_fen("7k/1p6/8/8/3N4/8/8/7K w - - 0 1").unwrap(); + let black = Board::from_fen("7k/8/8/3n4/8/8/1P6/7K b - - 0 1").unwrap(); + assert_eq!( + counts_of::<{ ALL_KINDS }>(&white, Color::White), + counts_of::<{ ALL_KINDS }>(&black, Color::Black) + ); + assert_eq!( + counts_of::<{ ALL_KINDS }>(&white, Color::Black), + [0, 0, 0, 0] + ); + } + + /// The position the two tests below are read against, and what each side + /// covers in it, worked out square by square rather than read back off + /// [`counts_of`]. + /// + /// White's knight on b1 has a3, c3 and d2. Its bishop on f1 has g2 and h3, + /// the pawn on e2 standing in the way of the other diagonal. Its rook on + /// a1 has the a file, the knight beside it being neither scope nor + /// something to see through. Its queen on d1 has c1, then c2, b3 and a4, + /// then the d file up to the black queen it takes, less d6, which the + /// black pawn on e7 covers. + /// + /// Black has no knight. Its bishop on c8 has b7 and a6 one way and d7 out + /// to h3 the other. Its rook on h8 has the h file and g8 and f8. Its queen + /// on d8 has c7, b6 and a5, then the d file down to the white queen, less + /// d3, which the white pawn on e2 covers. + const COUNTED: &str = "2bqk2r/4p3/8/8/8/8/4P3/RN1QKB2 w - - 0 1"; + const WHITE_COVERS: [i32; COUNTS] = [3, 2, 7, 10]; + const BLACK_COVERS: [i32; COUNTS] = [0, 7, 9, 9]; + + /// Four weights that differ from each other at both ends of the taper, so + /// that a pair read into the wrong piece's slot lands on a different + /// number. The four differences are 3, -5, -2 and 1, which differ from + /// each other too, so a permutation of either array shows. + const TRIAL: [i32; COUNTS] = [pack(11, 2), pack(-7, 13), pack(3, -5), pack(29, 41)]; + + /// What the fold does with weights that are not the shipped ones. + /// + /// The shipped weights would do here now that no two of them agree. + /// Weights of this test's own are kept anyway, because the shipped ones + /// are the fit's and will move again. This hands the fold four pairs that + /// differ from each other at both ends and asserts the packed pair against + /// the arithmetic: white's count less black's, piece by piece, each half + /// of the pair summed on its own. + #[test] + fn the_mobility_fold_reads_white_less_black_piece_by_piece() { + let board = Board::from_fen(COUNTED).unwrap(); + assert_eq!( + counts_of::<{ ALL_KINDS }>(&board, Color::White), + WHITE_COVERS + ); + assert_eq!( + counts_of::<{ ALL_KINDS }>(&board, Color::Black), + BLACK_COVERS + ); + let midgame: i32 = (0..COUNTS) + .map(|i| mg_value(TRIAL[i]) * (WHITE_COVERS[i] - BLACK_COVERS[i])) + .sum(); + let endgame: i32 = (0..COUNTS) + .map(|i| eg_value(TRIAL[i]) * (WHITE_COVERS[i] - BLACK_COVERS[i])) + .sum(); + assert_ne!( + midgame, endgame, + "the two halves would not tell a swap apart" + ); + let packed = super::fold_with::<{ ALL_KINDS }>(&board, &TRIAL); + assert_ne!(midgame, 0, "black less white would answer the same here"); + assert_eq!((mg_value(packed), eg_value(packed)), (midgame, endgame)); + } + + /// What the evaluation is allowed to leave out, which is the whole of the + /// contract between it and the tuner's walk. + /// + /// The walk counts all four kinds and the evaluation counts + /// [`SCORED_KINDS`], so the two no longer read one answer. What makes that + /// safe is the size of the difference and nothing else: a count multiplied + /// by zero adds nothing, so a kind worth zero can go uncounted without + /// moving a score, and any other kind cannot. So this asserts the + /// difference is exactly that, kind by kind, and then that the two fold to + /// the same packed pair at the shipped weights. + /// + /// A later fit that puts one of the four kinds back at zero fails here, + /// rather than being silently counted at every leaf for nothing. + /// + /// The position has to give every kind of both colours something to cover, + /// or a kind that is skipped and a kind that covers nothing read the same + /// and the test passes without having looked at anything. + #[test] + fn eval_counts_a_kind_exactly_when_its_weight_is_not_zero() { + let board = Board::from_fen(fens::KIWIPETE).unwrap(); + for color in [Color::White, Color::Black] { + let all = counts_of::<{ ALL_KINDS }>(&board, color); + let scored = counts_of::<{ SCORED_KINDS }>(&board, color); + for (index, piece) in PIECES.into_iter().enumerate() { + assert_ne!( + all[index], 0, + "{:?} covers nothing here, so this position cannot tell a skipped kind \ + from a counted one", + piece + ); + let weight = super::weight(index); + let priced = mg_value(weight) != 0 || eg_value(weight) != 0; + assert_eq!( + scored[index], + if priced { all[index] } else { 0 }, + "{:?} is worth {} in the midgame and {} in the ending, so it should be \ + {}, and eval counted {} of its {} squares", + piece, + mg_value(weight), + eg_value(weight), + if priced { "counted" } else { "skipped" }, + scored[index], + all[index] + ); + } + } + let whole = super::fold_with::<{ ALL_KINDS }>(&board, &MOBILITY); + assert_ne!( + whole, 0, + "mobility is level here, so this position says nothing about the fold" + ); + assert_eq!(fold(&board), whole); + } +} diff --git a/arche-core/src/eval/mod.rs b/arche-core/src/eval/mod.rs new file mode 100644 index 00000000..ce688d46 --- /dev/null +++ b/arche-core/src/eval/mod.rs @@ -0,0 +1,850 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +// Copyright (C) 2022-2026 Andrew Wright + +//! The evaluation: what a position scores, and every number that opinion is +//! built from. The board hosts an [`Accumulator`] and tells it about each +//! piece placed, removed and moved from one square to another; the search +//! asks [`eval`] for the score. A term cheap enough to keep incrementally +//! belongs in the accumulator; one computed at the leaf belongs in a module +//! of its own beside this one. +//! +//! This file holds what the terms share: the material values, the phase +//! weights the taper is read at, the accumulator, the sum the search asks for +//! and [`TERMS`], the list the tuner's seam reads the leaf terms through. +//! Each leaf term owns the rest of itself, its counts and the masks they are +//! read off, its weights with the fit that produced them, its fold and its +//! memo where it has one. + +mod mobility; +mod pawn_structure; +mod shelter; + +use crate::board::Board; +use crate::misc::{Color, Piece, Score}; +use crate::psqt::{PieceSquareTables, eg_value, mg_value}; + +static PIECE_SQUARE_TABLES: PieceSquareTables = PieceSquareTables::TABLES; + +/// What each piece leaves on the board, in `Piece` order, on the scale the +/// two halves of a tapered score are interpolated on. A queen counts for four, +/// a rook two and a minor one, so the opening's complement of pieces comes to +/// `TOTAL_PHASE` and a bare king and pawns to nothing. Pawns count for nothing +/// because an ending is an ending whether or not there are pawns in it. +static PHASE_WEIGHTS: [i32; 6] = [0, 1, 1, 2, 4, 0]; +/// What the opening's pieces add up to under `PHASE_WEIGHTS`. +pub(crate) const TOTAL_PHASE: i32 = 24; + +/// A table rather than a match. The match compiled to a jump table, and +/// once the piece arrives as a load from the board's square array the +/// target is data the predictor cannot see through: most of the search's +/// indirect mispredicts were this dispatch inside the accumulator's count. +/// Indexed the way the piece square tables and `Zobrist` already index by +/// piece; the assertion in `misc` pins the discriminants. +const MATERIAL: [u32; 6] = [100, 310, 320, 500, 900, 10000]; + +/// The material weight of one piece, for the board's own seeding walk. +pub(crate) fn material(piece: Piece) -> u32 { + MATERIAL[piece as usize] +} + +/// What one piece leaves on the board, on the scale the taper is read at. +/// The tuner's walk asks, because a position's phase decides what its +/// coefficients are and a copy of the table there would be a second opinion +/// about the taper. +pub(crate) fn phase_weight(piece: Piece) -> i32 { + PHASE_WEIGHTS[piece as usize] +} + +/// One leaf term, as everything outside its own file sees it. +/// +/// Four fields, and between them they are the whole of what the tuner needs +/// to lay a term out, price it and read its coefficients off a position. The +/// evaluation does not go through here: [`sum`] names the three folds +/// directly, so the hot path costs no indirect call and the descriptor is +/// free to be the offline seam it is for. +pub(crate) struct Term { + /// What the layout line calls the term, which is the name + /// `scripts/tune.py` keys its bounds and its holds by. + pub(crate) name: &'static str, + /// How many counts the term is measured in, per side and per half of the + /// taper. It occupies twice this in the weight vector, the midgame half + /// first. + pub(crate) width: usize, + /// The packed weight pair of one of those counts, read out of the live + /// array rather than a copy of it. + pub(crate) weight: fn(usize) -> i32, + /// One side's counts, written into the first `width` entries of the + /// slice. + pub(crate) counts: fn(&Board, Color, &mut [i32]), +} + +/// The leaf terms, in the order the weight vector holds them. +/// +/// A term is appended rather than inserted, so that adding one moves no slot +/// a fit has already been written against. This order is the order +/// `tune::SLOTS` sums, the order `Terms::of` walks and the order the run's +/// layout line prints, so the three follow from one list rather than agreeing +/// with each other. +pub(crate) const TERMS: &[Term] = &[ + Term { + name: "mobility", + width: mobility::COUNTS, + weight: mobility::weight, + counts: mobility::counts, + }, + Term { + name: "shelter", + width: shelter::COUNTS, + weight: shelter::weight, + counts: shelter::counts, + }, + Term { + name: "pawn_structure", + width: pawn_structure::COUNTS, + weight: pawn_structure::weight, + counts: pawn_structure::counts, + }, +]; + +/// The widest term, which is how long a buffer the tuner's walk needs to ask +/// any of them for its counts. +pub(crate) const WIDEST: usize = widest(); + +const fn widest() -> usize { + let mut widest = 0; + let mut index = 0; + while index < TERMS.len() { + if TERMS[index].width > widest { + widest = TERMS[index].width; + } + index += 1; + } + widest +} + +/// What the two remembered terms answer, asked of whatever the caller is +/// carrying. +/// +/// The sum is written once and reads the position through this, so the cached +/// evaluation and the uncached one are one body rather than two that have to +/// be kept saying the same thing. A method per cached term, so a fourth such +/// term adds one here and an implementation in each of the two below. That is +/// the seam, and it is meant to be written twice. +trait Memo { + fn shelter(&mut self, board: &Board) -> i32; + fn pawn_structure(&mut self, board: &Board) -> i32; +} + +/// The memo that remembers nothing, which is what [`eval`] hands the sum. +struct NoMemo; + +impl Memo for NoMemo { + #[inline] + fn shelter(&mut self, board: &Board) -> i32 { + shelter::fold(board) + } + + #[inline] + fn pawn_structure(&mut self, board: &Board) -> i32 { + pawn_structure::fold(board) + } +} + +/// What a searcher carries so that the two remembered terms are not computed +/// again at every leaf. +/// +/// One value rather than a cache per term: the search holds one field and +/// passes one argument, and a term that learns to remember itself is a field +/// here rather than a third parameter everywhere a score is asked for. +/// +/// Two tables inside it rather than one wider entry under the shelter's key. +/// The alternative is one probe for both terms, and it would recompute the +/// pawn structure on every king move, which is the half of the shelter's key +/// that term does not need. Which is cheaper is a measurement and not an +/// opinion, and it was made on the fitted build. +#[derive(Default)] +pub(crate) struct Caches { + shelter: shelter::Cache, + pawns: pawn_structure::Cache, +} + +impl Memo for Caches { + #[inline] + fn shelter(&mut self, board: &Board) -> i32 { + self.shelter.get(board) + } + + #[inline] + fn pawn_structure(&mut self, board: &Board) -> i32 { + self.pawns.get(board) + } +} + +/// The score of the position from the side to move's point of view, with the +/// two remembered terms taken from `memo`. +/// +/// Everything incremental is read off the board's accumulator; a term +/// computed at the leaf is added here, from the board itself. The three leaf +/// terms are summed before the call, which is exact since all three are pairs +/// on one scale, and it is what keeps one divide however many such terms +/// there are. +/// +/// Material that cannot mate reads zero before any of it. That is the one +/// place in the evaluation that is not a dot product against the weights, +/// which is why the tuner turns such a position away rather than fitting it: +/// see `tune::run`. It sits here rather than at the node because the model +/// gate, the tuner's walk and the instruments all read this function, and a +/// zero returned from the search instead would leave them saying a dead draw +/// is worth a piece. +#[inline] +fn sum(board: &Board, memo: &mut impl Memo) -> Score { + if board.drawn_by_material() { + return 0; + } + let leaf = mobility::fold(board) + memo.shelter(board) + memo.pawn_structure(board); + board.eval.score(board.active_color, leaf) +} + +/// The score with the shelter and the pawn structure computed every time. +/// +/// The tuner's walk and the instruments ask this one, and so do the five +/// places in the search that want a score depending on the position alone. +/// Neither door is hot enough there for the difference between them to +/// matter. +#[inline] +pub(crate) fn eval(board: &Board) -> Score { + sum(board, &mut NoMemo) +} + +/// The same score with those two terms taken from the searcher's caches +/// wherever they are there. +/// +/// Equal to [`eval`] for every position, which is what +/// `the_cache_answers_what_the_full_evaluation_does` holds both of them to. +/// The search calls this and the node counts do not move, because a +/// remembered score is the score that would have been computed. +#[inline] +pub(crate) fn eval_cached(board: &Board, caches: &mut Caches) -> Score { + sum(board, caches) +} + +/// The evaluation's incremental state, hosted by the board and kept in step +/// by being told about every piece placed, removed and relocated. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct Accumulator { + /// Each side's material, indexed by `Color`'s discriminant the way the + /// weight tables index by `Piece`'s: an index is a load where a match + /// on the colour was a branch. + material: [u32; 2], + /// Piece square table score, as the packed pair of midgame and endgame + /// scores the tables hold: see `psqt::pack`. Summing a boardful of pairs + /// is one add, so carrying both phases costs what carrying one did, and + /// neither half comes near the sixteen bits it has to stay inside. + psqt: i32, + /// What is left on the board, on the scale `PHASE_WEIGHTS` measures, and + /// so which of the two halves above the position is scored by. + /// Accumulated rather than counted off the piece boards at every leaf: + /// four popcounts there measured dearer than one add per piece touched + /// here. + phase: i32, +} + +impl Accumulator { + /// A board with nothing on it scores nothing. + pub(crate) const EMPTY: Self = Self { + material: [0; 2], + psqt: 0, + phase: 0, + }; + + /// Count a piece on to or off of a square. The two directions written + /// once: they are the same arithmetic with every sign reversed, and + /// `SET` is settled at compile time, so no branch on it survives into + /// the search. + #[inline(always)] + pub(crate) fn count(&mut self, index: u8, piece: Piece, color: Color) { + // a packed pair, negated whole for black: negating the sum negates + // both halves, so neither is unpacked until the leaf asks for it + let psqt = match color { + Color::White => PIECE_SQUARE_TABLES.get_value(index as usize, piece, Color::White), + Color::Black => -PIECE_SQUARE_TABLES.get_value(index as usize, piece, Color::Black), + }; + let phase = PHASE_WEIGHTS[piece as usize]; + let value = MATERIAL[piece as usize]; + if SET { + self.psqt += psqt; + self.phase += phase; + self.material[color as usize] += value; + } else { + self.psqt -= psqt; + self.phase -= phase; + self.material[color as usize] -= value; + } + } + + /// A piece moving between two squares, which is `count` off one square + /// and on to the other with the halves that cancel left out. + /// + /// It never leaves the board, so the material and the phase it counts + /// for are the same before and after and their two updates undo each + /// other exactly. What is left is the piece square score, and the pair + /// is added and subtracted whole either way, so a borrow between the two + /// halves cancels here as it does there. + #[inline(always)] + pub(crate) fn relocate(&mut self, from: u8, to: u8, piece: Piece, color: Color) { + let moved = PIECE_SQUARE_TABLES.get_value(to as usize, piece, color) + - PIECE_SQUARE_TABLES.get_value(from as usize, piece, color); + match color { + Color::White => self.psqt += moved, + Color::Black => self.psqt -= moved, + } + } + + /// The accumulator the position deserves, computed from the board rather + /// than accumulated as pieces moved. The hosted one is meant to equal + /// this at all times, which the board's state check asks on every move + /// made. A second implementation on purpose, and only worth having while + /// it stays one: factoring shared code out of this walk and `count` + /// would leave both sides wrong together and the check passing, which is + /// worse than not checking at all. + pub(crate) fn recomputed(board: &Board) -> Self { + let mut recomputed = Self::EMPTY; + // walking the occupied squares rather than all sixty four, an empty + // board is then free rather than sixty four misses + let mut occupied = board.occupied(); + while occupied != 0 { + let index = occupied.trailing_zeros() as u8; + occupied &= occupied - 1; + if let Some((piece, color)) = board.get_piece_and_color_index(index) { + let psqt = PIECE_SQUARE_TABLES.get_value(index as usize, piece, color); + match color { + Color::White => recomputed.psqt += psqt, + Color::Black => recomputed.psqt -= psqt, + } + recomputed.material[color as usize] += MATERIAL[piece as usize]; + recomputed.phase += PHASE_WEIGHTS[piece as usize]; + } + } + recomputed + } + + /// Material seeded from a recount rather than accumulated, which is how + /// `from_fen` fills a parsed board in: the state check then compares the + /// seeding against an implementation that did not do the seeding. + pub(crate) fn seed_material(&mut self, (white, black): (u32, u32)) { + self.material[Color::White as usize] = white; + self.material[Color::Black as usize] = black; + } + + /// What white stands ahead by, for the board's debug print. + pub(crate) fn material_difference(&self) -> i64 { + i64::from(self.material[Color::White as usize]) + - i64::from(self.material[Color::Black as usize]) + } + + /// The score from `side`'s point of view. + /// + /// The piece square half is read at the phase the position is in rather + /// than at either end of it, so that a king walks out as the pieces come + /// off instead of on the move that takes the last one. Material is not + /// tapered: an endgame piece value is the same thing as a constant added + /// to that piece's endgame table, and the tables are the tidier place to + /// say it. + /// + /// `leaf` is the leaf terms [`eval`] reads off the board, mobility, the + /// king's shelter and the pawn structure summed, as a packed pair on the + /// same scale. The pair joins the piece square pair before the + /// interpolation rather than being tapered beside it, so the two share one + /// divide. A second divide would answer a centipawn away wherever a + /// numerator is negative and does not divide evenly, and + /// `tune::reconstruct` folds a whole row with one. + #[inline] + fn score(&self, side: Color, leaf: i32) -> Score { + // promotions can leave more on the board than the opening had, so the + // phase is capped. It cannot go the other way: no weight is negative. + let phase = self.phase.min(TOTAL_PHASE); + let tapered = self.psqt + leaf; + let scaled = + (mg_value(tapered) * phase + eg_value(tapered) * (TOTAL_PHASE - phase)) / TOTAL_PHASE; + let eval = (self.material[Color::White as usize] as i32 + - self.material[Color::Black as usize] as i32 + + scaled) as Score; + match side { + Color::White => eval, + Color::Black => -eval, + } + } +} + +#[cfg(test)] +mod evaluate { + use super::{ + Board, Caches, TERMS, TOTAL_PHASE, eval, eval_cached, mobility, pawn_structure, shelter, + }; + use crate::board::fens; + use crate::misc::{Color, File, coordinate_to_index}; + use crate::psqt::{eg_value, mg_value, pack}; + use pretty_assertions::assert_eq; + use std::collections::{HashMap, HashSet}; + + /// Both the accumulator and its recompute read `PHASE_WEIGHTS`, so the + /// state-in-step check holds them to each other and neither to what the + /// weights should be. This says what they add up to: a full board is the + /// midgame end of the taper, kings and pawns alone the endgame end, and + /// each piece is worth what the interpolation was written expecting. + #[test] + fn a_full_board_is_one_end_of_the_taper_and_a_pawn_ending_the_other() { + assert_eq!(Board::new().eval.phase, TOTAL_PHASE); + assert_eq!( + Board::from_fen("4k3/pppppppp/8/8/8/8/PPPPPPPP/4K3 w - - 0 1") + .unwrap() + .eval + .phase, + 0 + ); + for (fen, phase) in [ + ("4k3/8/8/8/8/8/8/3QK3 w - - 0 1", 4), + ("4k3/8/8/8/8/8/8/3RK3 w - - 0 1", 2), + ("4k3/8/8/8/8/8/8/3BK3 w - - 0 1", 1), + ("4k3/8/8/8/8/8/8/3NK3 w - - 0 1", 1), + ] { + assert_eq!(Board::from_fen(fen).unwrap().eval.phase, phase, "{}", fen); + } + } + + /// After every legal move in the shared positions, the material + /// accumulators must equal a recount, and the score must be the exact + /// negative of the opponent's view of it. + #[test] + fn material_stays_counted_and_the_eval_stays_antisymmetric() { + for fen in fens::CORE { + let mut board = Board::from_fen(fen).unwrap(); + for m in &board.generate_moves() { + if board.make_move(m) { + assert_eq!( + ( + board.eval.material[crate::misc::Color::White as usize], + board.eval.material[crate::misc::Color::Black as usize] + ), + board.material_value(), + "{} in {}", + m, + fen + ); + let score = eval(&board); + board.active_color = !board.active_color; + assert_eq!(score, -eval(&board), "{} in {}", m, fen); + board.active_color = !board.active_color; + board.undo_move(); + } + } + } + } + + /// The assertions above hold whichever way up the piece square tables are, + /// because both colours read them the same way and the symmetry survives. + /// These say which way is up, a colour at a time. + #[test] + fn a_pawn_is_worth_more_the_closer_it_is_to_promoting() { + for (advanced, home) in [ + ( + "4k3/4P3/8/8/8/8/8/4K3 w - - 0 1", + "4k3/8/8/8/8/8/4P3/4K3 w - - 0 1", + ), + ( + "4k3/8/8/8/8/8/4p3/4K3 b - - 0 1", + "4k3/4p3/8/8/8/8/8/4K3 b - - 0 1", + ), + ] { + let advanced = Board::from_fen(advanced).unwrap(); + let home = Board::from_fen(home).unwrap(); + assert!( + eval(&advanced) > eval(&home), + "the advanced pawn scored {} and the one at home {}", + eval(&advanced), + eval(&home) + ); + } + } + + /// Material that cannot mate scores zero from either side, through both + /// entry points. + /// + /// Both, because the search reads the cached one and the tuner, the model + /// gate and the instruments read the other, and a rule in one of them + /// would have the two disagree about the same node. The caches are passed + /// and neither table is read: a drawn position has no pawn on it, so the + /// pawn structure would have contributed nothing had it been reached. + /// + /// The figures in the comments are what the evaluation returned before the + /// rule. The search played every one of them as a win. + #[test] + fn material_that_cannot_mate_scores_zero() { + for fen in [ + "8/8/8/8/8/4k3/8/4K1N1 w - - 0 1", // a knight, read as +320 and a table + "8/8/8/8/8/4k3/8/4K1N1 b - - 0 1", + "8/8/8/8/8/4k3/8/4KB2 w - - 0 1", // a bishop, +320 and a table + "8/8/8/8/8/4k3/8/4KB2 b - - 0 1", + "8/8/8/8/8/4k3/8/4K1NN w - - 0 1", // two knights, +640 and a table + "8/8/8/8/8/4k3/8/4K1NN b - - 0 1", + "4k3/8/8/8/4K3/8/8/8 w - - 0 1", // already zero, by cancellation + "4k3/8/8/8/4K3/8/8/8 b - - 0 1", + "4k3/8/8/8/8/B7/8/2B1K3 w - - 0 1", + "4k3/8/8/8/8/B7/8/2B1K3 b - - 0 1", + "3bk3/8/8/8/8/8/8/2B1K3 w - - 0 1", + "3bk3/8/8/8/8/8/8/2B1K3 b - - 0 1", + ] { + let board = Board::from_fen(fen).unwrap(); + let mut caches = Caches::default(); + assert_eq!(eval(&board), 0, "{}", fen); + assert_eq!(eval_cached(&board, &mut caches), 0, "{}", fen); + } + } + + /// The same signatures with a pawn on the board are outside the rule and + /// are scored as they were. + /// + /// Without this, the test above would pass on a rule that answered zero + /// for every pawnless position, or for every position at all. + #[test] + fn a_pawn_takes_a_position_out_of_the_rule() { + for fen in [ + "8/8/8/4p3/8/4k3/8/4K1N1 w - - 0 1", + "8/8/8/4p3/8/4k3/8/4KB2 w - - 0 1", + "8/8/8/4p3/8/4k3/8/4K1NN w - - 0 1", + ] { + let board = Board::from_fen(fen).unwrap(); + let mut caches = Caches::default(); + assert_ne!(eval(&board), 0, "{}", fen); + assert_eq!(eval_cached(&board, &mut caches), eval(&board), "{}", fen); + } + } + + /// The point of tapering: the same king on the same square is scored + /// differently depending on what is left on the board. A bare king wants + /// the middle; a king with the pieces still on wants the back rank. + /// + /// Each pair below differs by the king's square and nothing else, material + /// included, so the difference between them is the king's table alone. A + /// phase read the wrong way round would still land inside a pair, so what + /// is pinned is the direction the score moves in as the board empties + /// rather than only that it moves. + #[test] + fn a_king_is_worth_more_in_the_middle_the_emptier_the_board() { + // two king squares, e4 and g1, at three phases + fn centre_over_corner(centre: &str, corner: &str) -> i32 { + i32::from(eval(&Board::from_fen(centre).unwrap())) + - i32::from(eval(&Board::from_fen(corner).unwrap())) + } + let opening = centre_over_corner( + "rnbqkbnr/pppppppp/8/8/4K3/8/PPPPPPPP/RNBQ1B1R w kq - 0 1", + "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQ1BKR w kq - 0 1", + ); + // a rook a side, which is a phase of four out of twenty four + let middlegame = centre_over_corner( + "r3k3/8/8/8/4K3/8/8/R7 w q - 0 1", + "r3k3/8/8/8/8/8/8/R5K1 w q - 0 1", + ); + // a pawn a side, because two bare kings are drawn by material and + // both fens would read zero. A pawn does not count towards the phase, + // so this is still the ending, and the two pawns stand on the same + // squares in both fens and away from either king's files, which + // leaves the difference the king's own square + let ending = centre_over_corner( + "4k3/p7/8/8/4K3/8/P7/8 w - - 0 1", + "4k3/p7/8/8/8/8/P7/6K1 w - - 0 1", + ); + assert!( + opening < middlegame && middlegame < ending, + "e4 over g1 scored {} in the opening, {} with a rook a side and {} bare", + opening, + middlegame, + ending + ); + assert!(opening < 0 && ending > 0, "{} then {}", opening, ending); + } + + /// A position and its reflection, colours swapped, have to score the same + /// for whoever is to move. This does not catch the tables being upside down, + /// since that happens to both colours at once, but it does catch one colour + /// being changed without the other. + #[test] + fn a_mirrored_position_scores_the_same() { + for (white, black) in [ + ( + "4k3/4P3/8/8/8/8/8/4K3 w - - 0 1", + "4k3/8/8/8/8/8/4p3/4K3 b - - 0 1", + ), + ( + "4k3/8/8/8/8/8/8/R3K3 w - - 0 1", + "r3k3/8/8/8/8/8/8/4K3 b - - 0 1", + ), + ( + "rnbqkbnr/pppppppp/8/8/4P3/8/PPPP1PPP/RNBQKBNR b KQkq - 0 1", + "rnbqkbnr/pppp1ppp/8/4p3/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1", + ), + ] { + let white = Board::from_fen(white).unwrap(); + let black = Board::from_fen(black).unwrap(); + assert_eq!(eval(&white), eval(&black), "{} against {}", white, black); + } + } + + /// Mobility enters the numerator of the one interpolation rather than + /// being tapered beside it. + /// + /// A position whose piece square numerator is negative and does not divide + /// by twenty four evenly, so the two readings differ: inside the divide + /// the whole numerator is truncated once, and a second divide would + /// truncate each part on its own and answer a centipawn away. Nothing else + /// pins this, and it is what `tune::reconstruct` folds a row with. + #[test] + fn mobility_joins_the_numerator_rather_than_being_tapered_beside_it() { + let board = Board::from_fen("4k3/8/8/8/8/8/4P3/1N2K3 w - - 0 1").unwrap(); + let accumulator = board.eval; + let phase = accumulator.phase.min(TOTAL_PHASE); + let mobility = pack(11, -1); + let together = accumulator.psqt + mobility; + let inside = + (mg_value(together) * phase + eg_value(together) * (TOTAL_PHASE - phase)) / TOTAL_PHASE; + let beside = (mg_value(accumulator.psqt) * phase + + eg_value(accumulator.psqt) * (TOTAL_PHASE - phase)) + / TOTAL_PHASE + + (mg_value(mobility) * phase + eg_value(mobility) * (TOTAL_PHASE - phase)) + / TOTAL_PHASE; + assert_ne!( + inside, beside, + "the two divides agree here, so this position says nothing" + ); + let material = accumulator.material[Color::White as usize] as i32 + - accumulator.material[Color::Black as usize] as i32; + assert_eq!( + accumulator.score(Color::White, mobility), + (material + inside) as crate::misc::Score + ); + } + + /// The searcher's caches and the keys a walk saw through them. One thing + /// to carry rather than three, and the keys are what say whether the run + /// evicted anything. + #[derive(Default)] + struct Walk { + caches: Caches, + shelter_keys: HashSet, + pawn_keys: HashSet, + counted: HashMap, + compared: usize, + } + + impl Walk { + /// What the pawn key claims, checked against what the counts say. + /// + /// The cache hands a remembered score to every position whose pawn + /// key it matches, so the key has to decide the counts. The identity + /// below sees a wrong key now that the weights are the fit's, and it + /// sees it as a wrong score; this says the same thing one step + /// earlier and in the terms the term is defined in. Two positions + /// under one key that disagree on the eight counts are one position + /// handed the other's score, whatever the weights make that worth. + fn note(&mut self, board: &Board) { + let counts = [ + pawn_structure::counts_of(board, Color::White), + pawn_structure::counts_of(board, Color::Black), + ]; + if let Some(seen) = self.counted.insert(board.pawn_key, counts) { + self.compared += 1; + assert_eq!( + seen, + counts, + "two positions share a pawn key and not its counts, at {}", + board.to_fen() + ); + } + } + } + + /// How many of `keys` there are and how many slots of a table of `slots` + /// they land in. The second being the smaller is what says two keys + /// shared a slot, so an entry was written over rather than only written. + fn filled(keys: &HashSet, slots: usize) -> (usize, usize) { + let landed: HashSet = keys + .iter() + .map(|key| (*key as usize) & (slots - 1)) + .collect(); + (keys.len(), landed.len()) + } + + /// Every position reachable inside `budget` moves of `board`, scored both + /// ways through the caches. + fn walk(board: &Board, depth: usize, walked: &mut Walk, budget: &mut usize) { + if depth == 0 || *budget == 0 { + return; + } + for m in &board.generate_moves() { + if *budget == 0 { + break; + } + let mut played = board.clone(); + if !played.make_move(m) { + continue; + } + *budget -= 1; + walked.shelter_keys.insert(shelter::key(&played)); + walked.pawn_keys.insert(played.pawn_key); + walked.note(&played); + assert_eq!( + eval_cached(&played, &mut walked.caches), + eval(&played), + "a cache and the evaluation part company at {}", + played.to_fen() + ); + walk(&played, depth - 1, walked, budget); + } + } + + /// The cache answers what the full evaluation does, position for + /// position. + /// + /// A shelter score is remembered under the pawns and the two king + /// squares, and every other piece is outside that key, so a key missing + /// something the term reads would hand one position's shelter to + /// another. Nothing in a search would say so: the score is simply not + /// the position's, and a wrong evaluation is the one error it does not + /// report. + /// + /// A walk that never evicted would be testing a cache that only ever + /// grows, so the run has to overwrite entries as well as write and read + /// them. Counting the positions does not say it happened: the walk + /// revisits keys, and far fewer keys than positions reach the table. What + /// says it is the keys against the slots they land in, so the test + /// collects the keys and asserts that two of them shared a slot. + /// + /// `Walk::note` carries the same claim in the counts rather than in the + /// score, which is what it was written to do while the sixteen weights + /// were zero and the score said nothing. It is kept now that they are + /// fitted, because a count is a sharper thing to compare than a sum of + /// sixteen products: two wrong counts whose weights happen to cancel + /// would pass the score and fail the note. + #[test] + fn the_cache_answers_what_the_full_evaluation_does() { + let mut walked = Walk::default(); + let mut budget = 4 * shelter::CACHE_SLOTS.max(pawn_structure::CACHE_SLOTS); + for fen in fens::CORE { + let board = Board::from_fen(fen).unwrap(); + assert_eq!( + eval_cached(&board, &mut walked.caches), + eval(&board), + "{}", + fen + ); + walked.shelter_keys.insert(shelter::key(&board)); + walked.pawn_keys.insert(board.pawn_key); + walked.note(&board); + walk(&board, 3, &mut walked, &mut budget); + } + assert!( + walked.compared > 0, + "every pawn key here was seen once, so no two positions were held against each other" + ); + for (name, keys, slots) in [ + ("shelter", &walked.shelter_keys, shelter::CACHE_SLOTS), + ("pawn", &walked.pawn_keys, pawn_structure::CACHE_SLOTS), + ] { + let (keys, landed) = filled(keys, slots); + assert!( + keys > landed, + "{} keys over {} slots of the {} cache, so no slot was written twice and nothing was evicted", + keys, + landed, + name + ); + } + } + + /// A king move leaves the pawn key alone and moves the shelter key, so + /// the entry this term wrote answers the position after it and the + /// shelter's does not. + /// + /// The difference between the two caches, stated rather than implied. It + /// is why this term could be cached in the commit that introduced it + /// where the shelter's cache had to wait for a fit to show it was needed. + #[test] + fn a_king_move_keeps_the_pawn_entry_and_loses_the_shelter_one() { + let board = Board::from_fen("4k3/pppppppp/8/8/8/8/PPPPPPPP/4K3 w - - 0 1").unwrap(); + let from = coordinate_to_index(1, File::E); + let king = board + .generate_moves() + .into_iter() + .find(|m| m.from == from) + .expect("the king on e1 has a move here"); + let mut moved = board.clone(); + assert!(moved.make_move(&king)); + assert_eq!(moved.pawn_key, board.pawn_key); + assert_ne!(shelter::key(&moved), shelter::key(&board)); + } + + /// Every term the table names carries a width, a weight and a count + /// helper that agree with each other, and the helper fills every count + /// the width claims. + /// + /// The descriptor is what the tuner lays its slots out from, so a width + /// that disagrees with the helper would move every slot after it with + /// nothing here to say so. The buffer starts at `i32::MIN` and every + /// entry the width names has to have been written over, which is the + /// half of that a slice cannot catch: a helper handed `width` entries + /// and writing fewer leaves the rest of them where they stood, where one + /// writing more is out of bounds and panics. + #[test] + fn every_term_writes_the_counts_its_width_claims() { + let board = Board::from_fen(fens::KIWIPETE).unwrap(); + let mut named: HashSet<&str> = HashSet::new(); + for term in TERMS { + assert!(term.width > 0, "{} is measured in nothing", term.name); + assert!( + term.name + .chars() + .all(|c| c.is_ascii_lowercase() || c == '_') + ); + assert!(named.insert(term.name), "two terms called {}", term.name); + assert!(term.width <= super::WIDEST, "{} is wider", term.name); + let mut counts = [i32::MIN; super::WIDEST]; + (term.counts)(&board, Color::White, &mut counts[..term.width]); + assert!( + counts[..term.width].iter().all(|c| *c != i32::MIN), + "{} left a count its width claims unwritten", + term.name + ); + let priced = (0..term.width).any(|index| { + mg_value((term.weight)(index)) != 0 || eg_value((term.weight)(index)) != 0 + }); + assert!(priced, "{} is worth nothing at either end", term.name); + } + } + + /// The table names the three leaf terms the sum adds, which is what makes + /// the tuner's row and the evaluation the same arithmetic. + /// + /// The sum is hand written rather than a walk over the table, so this is + /// the one place the two lists are held against each other. A term added + /// to the table and left out of the sum would print coefficients the + /// evaluation never reads, and the tuner's identity would fail on the + /// first position that touched it; this says which of the two is wrong. + #[test] + fn the_table_names_the_terms_the_sum_adds() { + let names: Vec<&str> = TERMS.iter().map(|term| term.name).collect(); + assert_eq!(names, ["mobility", "shelter", "pawn_structure"]); + // two queens and a rook against none, a king in each corner of the + // board and pawns of both colours on six files, so that no one of the + // three folds to nothing and the test says something about each + let board = Board::from_fen("3k4/P4p2/8/3P2p1/3P4/PP2p2p/1P6/QQ4KR w - - 0 1").unwrap(); + for (name, term) in [ + ("mobility", mobility::fold(&board)), + ("shelter", shelter::fold(&board)), + ("pawn structure", pawn_structure::fold(&board)), + ] { + assert_ne!(term, 0, "{} is level here, so it says nothing", name); + } + let leaf = mobility::fold(&board) + shelter::fold(&board) + pawn_structure::fold(&board); + assert_eq!(eval(&board), board.eval.score(board.active_color, leaf)); + } +} diff --git a/arche-core/src/eval/pawn_structure.rs b/arche-core/src/eval/pawn_structure.rs new file mode 100644 index 00000000..744105c9 --- /dev/null +++ b/arche-core/src/eval/pawn_structure.rs @@ -0,0 +1,869 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +// Copyright (C) 2022-2026 Andrew Wright + +//! What each side's pawns stand as, and what that is worth. +//! +//! The term owns its counts, the masks they are read off, its weights, its +//! fold and the memo the search hands it. A pawn that moves changes which of +//! the pawns behind and beside it are passed, isolated or doubled, so there is +//! nothing here for `Accumulator::count` to add and take away a pawn at a +//! time. What makes that cheap anyway is that only a pawn move changes it at +//! all, which is what [`Cache`] is built on. + +use crate::board::Board; +use crate::misc::Color; +use crate::psqt::pack; + +/// How many ranks the passed pawn count is split over: a pawn's relative +/// second through its relative seventh, which is every rank one can stand on. +/// +/// A table by rank rather than one weight times the rank. What a passer is +/// worth is convex in how far it has come, and a ramp cannot say so; six +/// numbers can, and the tables already spend sixty four on where a pawn +/// stands. +const PASSED_RANKS: usize = 6; + +/// How many counts the pawn structure is measured in, in the order +/// [`PAWN_STRUCTURE`] and [`counts_of`] are indexed by: the passed pawns by +/// relative rank, then the isolated pawns, then the doubled ones. +pub(crate) const COUNTS: usize = PASSED_RANKS + 2; + +/// A pawn's own file and the files beside it, as a bit per file. +/// +/// The shelter's `king_files` steps the middle file in at the two edges so +/// that a king always names three; this does not, because the two are asking +/// different questions. A white pawn on a4 is passed while black has no pawn +/// on the a file or the b file, and a black pawn on the c file has nothing to +/// say about it. Stepping in would let that pawn stop it. +const fn pawn_files(square: u8) -> u8 { + let own = 1u8 << (square % 8); + // a shift off either end of the byte drops the bit, which is the edge + // case: the a file has no file to its left and the h file none to its + // right + own | (own << 1) | (own >> 1) +} + +/// The squares on `files` on every rank strictly ahead of `square`, ahead +/// meaning the direction `forward` pushes. +const fn span_ahead(square: u8, forward: i8, files: u8) -> u64 { + let mut mask = 0; + let mut rank = (square / 8) as i8 + forward; + while rank >= 0 && rank < 8 { + mask |= (files as u64) << (rank * 8); + rank += forward; + } + mask +} + +/// What stands in a pawn's way, as two masks a square names. +/// +/// `front_span` is the pawn's file and the two beside it, on every rank ahead +/// of it. A pawn of ours is passed when no pawn of theirs stands anywhere in +/// it, which is the whole of the standard definition bar one clause. +/// `file_ahead` is the same span without the neighbouring files, and it +/// answers that clause: whether a pawn of ours is already in front of this +/// one. It is kept as its own table rather than masked out of the first at +/// the leaf, because a leaf term pays for every instruction it adds and a +/// table costs half a kilobyte. +/// +/// Indexed by `Color`'s discriminant and then the square, the way the +/// shelter's masks are indexed and for the same reason. A white pawn is read +/// up the board and a black one down it. +struct Masks { + front_span: [[u64; 64]; 2], + file_ahead: [[u64; 64]; 2], +} + +impl Masks { + /// Built at compile time, the way the shelter's are. + const fn new() -> Self { + let mut masks = Masks { + front_span: [[0; 64]; 2], + file_ahead: [[0; 64]; 2], + }; + let mut square = 0u8; + while square < 64 { + let i = square as usize; + let own = 1u8 << (square % 8); + let beside = pawn_files(square); + masks.front_span[Color::White as usize][i] = span_ahead(square, 1, beside); + masks.front_span[Color::Black as usize][i] = span_ahead(square, -1, beside); + masks.file_ahead[Color::White as usize][i] = span_ahead(square, 1, own); + masks.file_ahead[Color::Black as usize][i] = span_ahead(square, -1, own); + square += 1; + } + masks + } +} + +static MASKS: Masks = Masks::new(); + +/// A set of files put back on the board: every square on every file the byte +/// names. What [`files_of`] undoes, and one multiply rather than eight +/// shifts, since a byte times the a file's eight squares lands a copy of the +/// byte on each rank. +const fn spread(files: u8) -> u64 { + (files as u64) * 0x0101_0101_0101_0101 +} + +/// Every square strictly in front of one of `pawns`, on that pawn's own file. +/// +/// The pawns shifted one rank on and then doubled three times, which carries +/// them the seven ranks a board has. Seeded with the shift rather than with +/// the pawns, so a pawn is never in its own fill, which is what leaves a lone +/// pawn undoubled. +const fn ahead_of(pawns: u64, color: Color) -> u64 { + match color { + Color::White => { + let mut filled = pawns << 8; + filled |= filled << 8; + filled |= filled << 16; + filled |= filled << 32; + filled + } + Color::Black => { + let mut filled = pawns >> 8; + filled |= filled >> 8; + filled |= filled >> 16; + filled |= filled >> 32; + filled + } + } +} + +/// Which files a set of pawns stands on, as a bit per file. +/// +/// The board folded in half three times, so the answer is three shifts, three +/// ors and a narrowing rather than eight masked tests. Nothing here says how +/// many pawns a file holds, which is all the counts that ask want to know. +/// +/// The shelter reads it too, for the king's files that hold no pawn of ours. +/// It lives here because [`spread`] is its inverse and the pair is easier to +/// check side by side than apart. +pub(super) const fn files_of(pawns: u64) -> u8 { + let folded = pawns | (pawns >> 32); + let folded = folded | (folded >> 16); + let folded = folded | (folded >> 8); + folded as u8 +} + +/// What one of those eight counts is worth, as the packed pairs the taper is +/// read from. +/// +/// Fitted 2026-09-12 by `scripts/tune.py` over the whole archived strength +/// run: 164 artifacts across 37 runs, 34,175 games, whose 4,428,569 post-book +/// plies gave 4,196,989 positions and 1,922,548 quiet rows, extracted by +/// `arche terms` at 39da0fe. The corpus is sha256 `f9ff326c` and the rows are +/// sha256 `35fd184d`. K was held at 1.1959 and the games split by pair: 10,696 +/// pairs trained, 3,638 chose the ridge and 2,750 were sealed. The 768 table +/// entries, the six material values, the eight mobility weights and the +/// fourteen shelter ones were all held where they stand, so these sixteen are +/// the only thing that moved. Every number here is of the rounded vector that +/// ships. +/// +/// The selection group scores 0.090580 at zero and 0.089555 at these, a +/// paired difference of -0.001025 against a standard error of 0.000128 over +/// its 7,263 games, at a design factor of 4.8. That is eight standard errors +/// outside its interval, and larger than the king safety fit read on a corpus +/// four fifths this size. By phase it is -0.001274 where six or fewer pieces +/// are left, -0.000671 from seven to twelve, and +0.000135 at thirteen or +/// more: the term pays in the ending, pays a little in the middlegame and +/// costs a little in the opening, which is the shape a pawn structure term +/// should have. +/// +/// The sealed group was opened once, after the weights were frozen, over +/// 2,750 pairs and 5,480 games and 312,012 positions that no fit and no +/// ridge choice had read. It scores 0.084236 at zero and 0.083240 at these, +/// a paired difference of -0.000995 against a standard error of 0.000146 at +/// a design factor of 5.0. That is outside its interval, and 0.15 standard +/// errors from the selection group's reading, so the two agree more closely +/// than the king safety fit's two did. +/// +/// It also answers the one thing the selection group said against the term. +/// There the opening bucket got worse by 0.000135; on the sealed games it +/// improves by 0.000249, so that was noise rather than a cost. By phase the +/// sealed reading is -0.001107 at six pieces or fewer, -0.000867 from seven +/// to twelve and -0.000249 at thirteen or more, which is the same shape +/// without the sting in its tail. +/// +/// The ridge is 1e-8, chosen on the selection group from a grid the zero was +/// taken out of. The grid as it stands includes no regularisation at all and +/// ranked it first, and the vector it produced put 239 on a passed pawn's +/// seventh rank in the midgame and -29 in the ending. `PAWNS_END` already +/// pays a pawn on the seventh 78 to 83, so that says a passed pawn one square +/// from queening is worth less than a blockaded one in the phase that decides +/// it. The seventh rank count carries the fewest rows of the eight (5.31%) +/// and the smallest summed midgame coefficient of all sixteen slots, a +/// twenty seventh of the isolated count's, so it is the direction the corpus +/// constrains least and the one an unregularised fit put its error in. +/// Overruling the grid cost 0.000132 of selection loss against a standard +/// error of 0.00015, and bought a largest weight of 53 rather than 239. +/// +/// The selection group did not catch that, which is the part worth knowing: +/// zero scored better there, on games no fit had read. A held-out group +/// detects a vector that has memorised its training games and does not detect +/// one that has learned a real but unrepresentative regularity, and the quiet +/// filter manufactures exactly that wherever a feature's interesting cases +/// are tactical. +/// +/// Three of the sixteen are not what the term was built to say, and they are +/// left as the fit gave them rather than tidied. A passed pawn on the seventh +/// is worth less than one on the sixth at both ends, and that survives every +/// ridge on the grid, so it is the corpus and not the regularisation: a +/// position with a passer on the seventh is rarely quiet unless the pawn is +/// blockaded or about to be lost, so the rows that reach the fit are the ones +/// where it is not winning. Passed pawns on the second through the fourth are +/// a midgame penalty. A doubled pawn is worth 8 in the midgame. The isolated +/// count, which carries a coefficient in 56.98% of the rows and has by far +/// the most evidence behind it, reads -11 and -6, which is what a player +/// would have guessed. +/// +/// A side's eight pawns can fill the eight counts twenty four times over +/// between them, since one pawn can be passed and isolated and doubled at +/// once, but no count can exceed eight and the six passed counts share the +/// eight between them. So the most this term adds to one half of the packed +/// pair is 776 in the midgame and 440 in the ending, both sides counted, +/// against the 32,767 a half has to stay inside. `bounds_hold` charges eight +/// of every count rather than eight across them, which is looser again and +/// is the screen rather than the arithmetic: it puts the whole vector's +/// boardful at 8,622, of which this term is at most 3,024. +static PAWN_STRUCTURE: [i32; COUNTS] = [ + pack(-18, 18), + pack(-25, 15), + pack(-22, 28), + pack(6, 30), + pack(53, 39), + pack(46, 13), + pack(-11, -6), + pack(8, -10), +]; + +/// The weight of one of the eight counts, as the packed pair. The tuner's +/// seam asks through [`super::TERMS`], so that a slot names the live weight +/// rather than a copy of it, the way it reads the tables. +pub(crate) fn weight(index: usize) -> i32 { + PAWN_STRUCTURE[index] +} + +/// What this side's pawns stand as, in the eight counts [`COUNTS`] names: its +/// passed pawns by relative rank, the second through the seventh, then its +/// isolated pawns and its doubled ones. +/// +/// A pawn of ours is passed when no pawn of theirs stands on its file or +/// either file beside it on any rank ahead of it, and no pawn of ours +/// stands ahead of it on its own file. The second clause is what leaves +/// the rear of a doubled pair out: the front pawn is the runner, and the +/// one behind it is going nowhere the front one has not gone first. What +/// stands on the square in front of the pawn is not read, so a passer a +/// knight has blockaded is counted as a passer. That is on purpose and it +/// is the first thing this term leaves out: the stop square reads the +/// pieces, and a term that reads the pieces cannot sit behind a key over +/// the pawns. +/// +/// A pawn is isolated when no pawn of ours stands on either file beside +/// it, and doubled when a pawn of ours stands behind it on its own file. +/// Both are counted per pawn rather than per file, so an isolated pair on +/// one file pays the isolated weight twice and a tripled file is doubled +/// two. Per pawn is what one coefficient can state; per file would want a +/// second table to say how many. +/// +/// Relative rank is the rank a pawn has come, so a white pawn's is its +/// rank and a black pawn's is nine less. The relative second is a real +/// bucket and not a rounding of the others: a pawn still at home is +/// passed the moment the enemy pawns on its three files are gone. +/// +/// Pawns and nothing else is read here, which is the property the pawn +/// hash rests on. Neither king, no piece and not the side to move: two +/// positions whose pawns agree agree on all eight counts, and +/// `Board::pawn_key` already stands for that agreement. +/// +/// Both the evaluation and the tuner's walk read this, so the identity +/// between them cannot see a wrong count here, at the fitted weights or at +/// zero. What pins it is the hand counts in the tests below, the way the +/// shelter counts and the mobility counts are pinned. +#[inline] +pub(crate) fn counts_of(board: &Board, color: Color) -> [i32; COUNTS] { + let masks = &MASKS; + let side = color as usize; + let (ours, theirs) = board.sides(color); + let pawns = board.pawns(); + let (our_pawns, their_pawns) = (pawns & ours, pawns & theirs); + let mut counts = [0; COUNTS]; + let mut remaining = our_pawns; + while remaining != 0 { + let square = remaining.trailing_zeros() as usize; + remaining &= remaining - 1; + let relative = match color { + Color::White => square / 8, + Color::Black => 7 - square / 8, + }; + // from_fen accepts a pawn on either back rank, knowingly, and the + // search has to survive one. Such a pawn has come no ranks or all + // eight and so names none of the six counted; it is left out of + // the passed count rather than folded into the nearest bucket, + // and it still counts toward the two below, which read its file + // and not its rank + if !(1..PASSED_RANKS + 1).contains(&relative) { + continue; + } + if their_pawns & masks.front_span[side][square] == 0 + && our_pawns & masks.file_ahead[side][square] == 0 + { + counts[relative - 1] += 1; + } + } + let files = files_of(our_pawns); + let beside = (files << 1) | (files >> 1); + counts[PASSED_RANKS] = (our_pawns & spread(files & !beside)).count_ones() as i32; + counts[PASSED_RANKS + 1] = (our_pawns & ahead_of(our_pawns, color)).count_ones() as i32; + counts +} + +/// The eight counts, written into `into`, which is what [`super::TERMS`] hands +/// the tuner's walk. +pub(crate) fn counts(board: &Board, color: Color, into: &mut [i32]) { + into.copy_from_slice(&counts_of(board, color)); +} + +/// What white's pawn structure stands ahead by, as a packed pair on the scale +/// the piece square pair is on. +#[inline] +pub(crate) fn fold(board: &Board) -> i32 { + fold_with(board, &PAWN_STRUCTURE) +} + +/// The same fold against weights named by the caller. +/// +/// The live weights are the fit's now, and no two of the eight pairs agree, +/// so the sign of this term, the order of the eight counts and the packing +/// all show in an evaluation the engine prints. The tests still supply weights +/// of their own, because what they pin is the fold rather than the fit: a +/// permuted [`PAWN_STRUCTURE`] would be a different evaluation and not a wrong +/// one. +#[inline] +fn fold_with(board: &Board, weights: &[i32; COUNTS]) -> i32 { + let white = counts_of(board, Color::White); + let black = counts_of(board, Color::Black); + let mut packed = 0; + for ((weight, white), black) in weights.iter().zip(white).zip(black) { + packed += weight * (white - black); + } + packed +} + +/// How many pawn structure scores the cache holds. A power of two, so the +/// index is a mask rather than a remainder. +/// +/// Four thousand entries at sixteen bytes is sixty four kilobytes, half what +/// the shelter's table takes. The size was measured rather than reasoned +/// about, the way the shelter's bits were, and the measurement was retaken +/// on the fitted tree when the weights arrived, since a different tree is a +/// different working set. Callgrind over the bench with the cache simulated, +/// at eleven, twelve, thirteen and fourteen bits, reads 4,098,418,313, +/// 4,092,752,759, 4,089,023,904 and 4,086,576,985 instructions against last +/// level misses of 287,010, 287,113, 290,125 and 294,220. Each bit buys +/// fewer instructions than the one before it, and the misses are flat from +/// eleven to twelve and then turn up by three thousand at thirteen and four +/// thousand more at fourteen. So twelve is the last size the memory does not +/// notice, and the whole range is inside three tenths of a percent of +/// instructions: the constant is not load bearing and a later working set +/// can move it. The sweep at zero weights, before the fit, put the turn in +/// the same place. +/// +/// Half the shelter's table is what this term's key predicted before either +/// sweep was run. The same pawns under two different pairs of king squares +/// are two entries there and one entry here, so the working set behind this +/// key is the smaller of the two. +/// +/// The bench counts the same 4,066,438 nodes at all four sizes, which is +/// what says the cache changes how a score is arrived at and not what it is. +const CACHE_BITS: usize = 12; +pub(super) const CACHE_SLOTS: usize = 1 << CACHE_BITS; + +/// One remembered pawn structure score, under the key that decides it. The +/// whole key, for the reason the shelter's entry keeps the whole of its own. +#[derive(Copy, Clone)] +struct Entry { + key: u64, + packed: i32, +} + +/// The pawn structure, remembered by what it depends on. +/// +/// The term reads the two pawn boards and nothing else, so its key is +/// `Board::pawn_key` with nothing folded in. That is the difference between +/// this cache and the shelter's, and it is the whole reason this term could +/// be cached in the commit that introduced it: the shelter's key carries both +/// kings and so misses on every king move, and this one does not. What misses +/// here is a pawn move and the capture of a pawn, which is every way the two +/// pawn boards change and nothing else. +/// +/// Direct mapped and never cleared, on the terms the shelter's cache sets +/// out. An empty entry is key zero holding zero, and here that is exact +/// rather than a coincidence to be argued about: a board with no pawns has a +/// pawn key of zero, which `a_board_with_no_pawns_has_no_key` pins, and the +/// structure of no pawns is eight zero counts. The one position that reads an +/// empty entry as its own reads the right answer from it. +pub(super) struct Cache { + entries: Box<[Entry]>, +} + +impl Default for Cache { + fn default() -> Self { + Cache { + entries: vec![Entry { key: 0, packed: 0 }; CACHE_SLOTS].into_boxed_slice(), + } + } +} + +impl Cache { + /// What white's pawn structure stands ahead by, remembered or computed. + #[inline] + pub(super) fn get(&mut self, board: &Board) -> i32 { + let key = board.pawn_key; + let slot = (key as usize) & (CACHE_SLOTS - 1); + let entry = &mut self.entries[slot]; + if entry.key == key { + return entry.packed; + } + let packed = fold(board); + *entry = Entry { key, packed }; + packed + } +} + +#[cfg(test)] +mod tests { + use super::{ + Board, CACHE_SLOTS, COUNTS, Cache, Color, MASKS, ahead_of, counts_of, files_of, fold, + pawn_files, spread, + }; + use crate::board::fens; + use crate::psqt::{eg_value, mg_value, pack}; + use pretty_assertions::assert_eq; + + /// A remembered score is read back rather than recomputed, under the pawn + /// key rather than that key with the two kings folded in. + /// + /// What this reads is the key as much as the score: an entry written + /// under the position's pawn key is the entry the next position with + /// those pawns is handed. The score is now the fit's rather than zero, so + /// the entry holding it says something too. + #[test] + fn a_pawn_structure_is_remembered_under_the_pawn_key() { + let board = Board::from_fen(fens::MIDDLEGAME).unwrap(); + let mut cache = Cache::default(); + let key = board.pawn_key; + assert_ne!(key, 0, "an empty entry would answer this one correctly"); + let slot = (key as usize) & (CACHE_SLOTS - 1); + assert_eq!(cache.entries[slot].key, 0, "the slot starts empty"); + let first = cache.get(&board); + assert_eq!(cache.entries[slot].key, key); + assert_eq!(cache.entries[slot].packed, fold(&board)); + assert_eq!(cache.get(&board), first); + } + + /// The counts by hand, position by position, because nothing else pins + /// them. The tuner's identity folds a row against the live weights, and + /// `eval` and the tuner's walk read this same helper, so the two sides of + /// the identity move together whatever it answers. These cases are the + /// only check this term has. + /// + /// Each case names the eight counts in the order the helper returns them: + /// the passed pawns on the relative second through the relative seventh, + /// then the isolated pawns, then the doubled ones. The other king stands + /// out of the way. + #[test] + fn pawns_count_as_a_hand_count_says_they_do() { + for (fen, counts, why) in [ + // a pawn with nothing in front of it anywhere is passed, and a + // pawn with no pawn beside it is isolated. The lone pawn is both + ( + "4k3/8/8/8/4P3/8/8/4K3 w - - 0 1", + [0, 0, 1, 0, 0, 0, 1, 0], + "a white pawn on e4 and no black pawn", + ), + // an enemy pawn on an adjacent file ahead of it stops it + ( + "4k3/8/8/3p4/4P3/8/8/4K3 w - - 0 1", + [0, 0, 0, 0, 0, 0, 1, 0], + "the same against a black pawn on d5", + ), + // level is not ahead. A black pawn beside the white one has + // already been passed + ( + "4k3/8/8/8/3pP3/8/8/4K3 w - - 0 1", + [0, 0, 1, 0, 0, 0, 1, 0], + "the same against a black pawn on d4", + ), + // the whole file ahead is read and not the next rank or two + ( + "4k3/5p2/8/8/4P3/8/8/4K3 w - - 0 1", + [0, 0, 0, 0, 0, 0, 1, 0], + "the same against a black pawn on f7", + ), + // a pawn of ours in front of it stops it too, which is what + // leaves the rear of a doubled pair out of the passed count + ( + "4k3/8/8/4P3/4P3/8/8/4K3 w - - 0 1", + [0, 0, 0, 1, 0, 0, 2, 1], + "white pawns on e4 and e5", + ), + // a tripled file is two doubled pawns and not one or three + ( + "4k3/8/8/8/4P3/4P3/4P3/4K3 w - - 0 1", + [0, 0, 1, 0, 0, 0, 3, 2], + "white pawns on e2, e3 and e4", + ), + // the relative second is a real bucket. A pawn still at home is + // passed once the enemy pawns on its three files are gone + ( + "4k3/8/8/8/8/8/4P3/4K3 w - - 0 1", + [1, 0, 0, 0, 0, 0, 1, 0], + "a white pawn on e2 with no black pawn", + ), + ( + "4k3/4P3/8/8/8/8/8/4K3 w - - 0 1", + [0, 0, 0, 0, 0, 1, 1, 0], + "a white pawn on e7", + ), + // the stop square is not read, so a blockaded passer is a passer. + // On purpose: the stop square is the first thing this term leaves + // out, and a later arm has to be able to find the place it goes + ( + "4k3/4n3/4P3/8/8/8/8/4K3 w - - 0 1", + [0, 0, 0, 0, 1, 0, 1, 0], + "a white pawn on e6 behind a black knight on e7", + ), + // a pawn two files away is no company + ( + "4k3/8/8/8/8/8/P1P5/4K3 w - - 0 1", + [2, 0, 0, 0, 0, 0, 2, 0], + "white pawns on a2 and c2", + ), + ( + "4k3/8/8/8/8/8/PP6/4K3 w - - 0 1", + [2, 0, 0, 0, 0, 0, 0, 0], + "white pawns on a2 and b2", + ), + // the two edge files at once. The file mask is a shift and not a + // rotate, so the a file has no neighbour off the left of the byte + // and the h file none off the right; a rotate would make each of + // these the other's neighbour and leave both counted as company + ( + "4k3/8/8/8/8/8/P6P/4K3 w - - 0 1", + [2, 0, 0, 0, 0, 0, 2, 0], + "white pawns on a2 and h2", + ), + // the edge cases the file mask decides. A pawn on the a file is + // read against the a and b files and not against the c file, so + // stepping the mask in the way the shelter's `king_files` steps + // it would let the pawn on c5 stop this one + ( + "4k3/8/8/1p6/P7/8/8/4K3 w - - 0 1", + [0, 0, 0, 0, 0, 0, 1, 0], + "a white pawn on a4 against a black pawn on b5", + ), + ( + "4k3/8/8/2p5/P7/8/8/4K3 w - - 0 1", + [0, 0, 1, 0, 0, 0, 1, 0], + "a white pawn on a4 against a black pawn on c5", + ), + ( + "4k3/8/8/6p1/7P/8/8/4K3 w - - 0 1", + [0, 0, 0, 0, 0, 0, 1, 0], + "a white pawn on h4 against a black pawn on g5", + ), + // from_fen accepts a pawn on either back rank and the search has + // to survive one. It has come no ranks or all eight, so it names + // none of the six passed buckets, and it still counts toward the + // two that read its file + ( + "4k3/8/8/8/8/8/8/3K1P2 w - - 0 1", + [0, 0, 0, 0, 0, 0, 1, 0], + "a white pawn on f1", + ), + ( + "4k1P1/8/8/8/8/8/8/4K3 w - - 0 1", + [0, 0, 0, 0, 0, 0, 1, 0], + "a white pawn on g8", + ), + ] { + let board = Board::from_fen(fen).unwrap(); + assert_eq!(counts_of(&board, Color::White), counts, "{}", why); + } + } + + /// The same reading for black, whose pawns are measured down the board + /// rather than up it. Each case is the reflection of one above, so a + /// relative rank read the wrong way up shows as a count in the wrong + /// bucket rather than as no count at all. + #[test] + fn a_black_pawn_is_measured_down_the_board() { + for (fen, counts, why) in [ + ( + "4k3/8/8/4p3/8/8/8/4K3 w - - 0 1", + [0, 0, 1, 0, 0, 0, 1, 0], + "a black pawn on e5 and no white pawn", + ), + ( + "4k3/8/8/4p3/3P4/8/8/4K3 w - - 0 1", + [0, 0, 0, 0, 0, 0, 1, 0], + "the same against a white pawn on d4", + ), + ( + "4k3/8/8/4p3/4p3/8/8/4K3 w - - 0 1", + [0, 0, 0, 1, 0, 0, 2, 1], + "black pawns on e4 and e5", + ), + ( + "4k3/8/8/8/8/8/4p3/4K3 w - - 0 1", + [0, 0, 0, 0, 0, 1, 1, 0], + "a black pawn on e2", + ), + ( + "4k3/4p3/8/8/8/8/8/4K3 w - - 0 1", + [1, 0, 0, 0, 0, 0, 1, 0], + "a black pawn on e7", + ), + ] { + let board = Board::from_fen(fen).unwrap(); + assert_eq!(counts_of(&board, Color::Black), counts, "{}", why); + } + } + + /// Black's count of a position is white's count of its reflection, so the + /// two colours are read the same way round. + #[test] + fn the_two_colours_count_the_same_way() { + let white = Board::from_fen("4k3/P4p2/8/3P2p1/3P4/PP2p2p/1P6/4K3 w - - 0 1").unwrap(); + let black = Board::from_fen("4k3/1p6/pp2P2P/3p4/3p2P1/8/p4P2/4K3 w - - 0 1").unwrap(); + assert_eq!( + counts_of(&white, Color::White), + counts_of(&black, Color::Black) + ); + assert_eq!( + counts_of(&white, Color::Black), + counts_of(&black, Color::White) + ); + } + + /// Nothing but the pawns decides the counts, which is what [`Cache`] is + /// keyed on. The same pawns behind different pieces, and with the two + /// kings somewhere else, count the same. + #[test] + fn nothing_but_the_pawns_is_counted() { + let bare = Board::from_fen("4k3/pp3ppp/8/8/8/8/PPP2PP1/4K3 w - - 0 1").unwrap(); + let full = + Board::from_fen("r1bq1rk1/pp3ppp/2n5/8/8/2N5/PPP2PP1/R1BQK2R w KQ - 0 1").unwrap(); + for color in [Color::White, Color::Black] { + assert_eq!( + counts_of(&bare, color), + counts_of(&full, color), + "{:?}", + color + ); + } + } + + /// A pawn's own file and the files beside it, and no more than that. The + /// edges are the case: two files there and not three, and not three with + /// the middle one stepped in the way the shelter's `king_files` steps it. + #[test] + fn a_pawn_names_its_own_file_and_the_ones_beside_it() { + for square in 0..64u8 { + let file = u32::from(square % 8); + let files = pawn_files(square); + let expected = if file == 0 || file == 7 { 2 } else { 3 }; + assert_eq!(files.count_ones(), expected, "the files of {}", square); + assert_eq!( + files & (1 << file), + 1 << file, + "not its own file: {}", + square + ); + assert_eq!( + files.trailing_zeros() + expected - 1, + 7 - files.leading_zeros(), + "the files of {} are not in a row", + square + ); + } + } + + /// Each front span holds every rank ahead of the pawn and no rank level + /// with it or behind it, on the files the pawn names and no others. + #[test] + fn the_front_span_is_the_files_beside_the_pawn_on_the_ranks_ahead() { + for square in 0..64u8 { + let rank = i32::from(square / 8); + for (side, forward) in [(Color::White, 1), (Color::Black, -1)] { + let span = MASKS.front_span[side as usize][square as usize]; + let ranks = if forward == 1 { 7 - rank } else { rank }; + assert_eq!( + span.count_ones(), + pawn_files(square).count_ones() * ranks as u32, + "{:?} on {}", + side, + square + ); + if span != 0 { + assert_eq!( + files_of(span), + pawn_files(square), + "{:?} on {} covers other files", + side, + square + ); + } + for step in 0..8i32 { + let on_rank = span & (0xffu64 << (step * 8)); + let ahead = (step - rank) * forward > 0; + assert_eq!( + on_rank != 0, + ahead, + "{:?} on {} holds rank {}", + side, + square, + step + ); + } + } + } + } + + /// The file ahead is the front span with the neighbouring files taken + /// off, which is the relationship the two tables are built to have. Kept + /// as its own table and not worked out at the leaf, so this is what says + /// the two agree. + #[test] + fn the_file_ahead_is_the_front_span_on_the_pawns_own_file() { + for square in 0..64u8 { + let own = spread(1u8 << (square % 8)); + for side in [Color::White, Color::Black] { + let i = side as usize; + assert_eq!( + MASKS.file_ahead[i][square as usize], + MASKS.front_span[i][square as usize] & own, + "{:?} on {}", + side, + square + ); + } + } + } + + /// The forward fill holds every square in front of a pawn and never the + /// pawn itself, which is what leaves a lone pawn undoubled. + #[test] + fn the_fill_starts_one_rank_in_front_of_the_pawn() { + for square in 0..64u8 { + let pawn = 1u64 << square; + for side in [Color::White, Color::Black] { + let filled = ahead_of(pawn, side); + assert_eq!( + filled & pawn, + 0, + "{:?} on {} is in its own fill", + side, + square + ); + assert_eq!( + filled, MASKS.file_ahead[side as usize][square as usize], + "{:?} on {}", + side, square + ); + } + } + } + + /// The fold down to a file a bit answers a walk of the squares. What the + /// shelter borrows from here is pinned at the same time. + #[test] + fn the_file_fold_answers_a_walk_of_the_squares() { + for square in 0..64u8 { + assert_eq!(files_of(1u64 << square), 1 << (square % 8), "{}", square); + } + let board = Board::from_fen("k7/8/8/5p2/8/6P1/7P/6K1 w - - 0 1").unwrap(); + let (white, black) = board.sides(Color::White); + // white's pawns stand on g3 and h2, and black's on f5 + assert_eq!(files_of(board.pawns() & white), 0b1100_0000); + assert_eq!(files_of(board.pawns() & black), 0b0010_0000); + } + + /// The position the fold test below is read against, and what each side + /// counts in it, worked out by hand rather than read back off + /// [`counts_of`]. + /// + /// The two sides are kept to files that do not meet, white on a, b and d + /// and black on e, f, g and h, so no pawn of one colour stands in any + /// pawn of the other's way and every passer here is passed for a reason + /// a reader can check in one line. + /// + /// White has a7, a3, b3, b2, d5 and d4. a7 is passed on the seventh and + /// a3 is not, because a7 is ahead of it on the file; b3 is passed on the + /// third and b2 is not; d5 is passed on the fifth and d4 is not. The d + /// file has no white pawn beside it, so d5 and d4 are the two isolated + /// pawns. a7, b3 and d5 each have a pawn of their own behind them on + /// their file, which is the three doubled. + /// + /// Black has f7, g5, e3 and h3, one to a file and all four passed: f7 on + /// its relative second, g5 on its relative fourth, and e3 and h3 on its + /// relative sixth. The four files run together, so nothing is isolated, + /// and no file holds two, so nothing is doubled. + /// + /// The eight differences are -1, 1, -1, 1, -2, 1, 2 and 3. None is zero, + /// so every slot does work in the assertion below. They are not all + /// distinct, so this alone would not tell the first count from the third; + /// what tells those apart is + /// `every_pawn_count_writes_both_ends_of_the_taper` in tune.rs, which + /// reads the coefficients bucket by bucket. + const STRUCTURED: &str = "3k4/P4p2/8/3P2p1/3P4/PP2p2p/1P6/6K1 w - - 0 1"; + const WHITE_STRUCTURE: [i32; COUNTS] = [0, 1, 0, 1, 0, 1, 2, 3]; + const BLACK_STRUCTURE: [i32; COUNTS] = [1, 0, 1, 0, 2, 0, 0, 0]; + + /// Eight weights that differ from each other at both ends of the taper, + /// so that a pair read into the wrong count's slot lands on a different + /// number. The eight differences between the halves are 38, 20, -24, -30, + /// 36, 25, -26 and -28, which differ from each other too, so a + /// permutation of either array shows. + const TRIAL: [i32; COUNTS] = [ + pack(3, 41), + pack(-7, 13), + pack(29, 5), + pack(11, -19), + pack(17, 53), + pack(-23, 2), + pack(-5, -31), + pack(37, 9), + ]; + + /// What the fold does with weights that are not the shipped ones. + /// + /// The shipped weights would do here now that they are not zero. Weights + /// of this test's own are kept anyway, because the shipped ones are the + /// fit's and will move again: a pin written against them would have to be + /// rewritten by every refit, and what it is pinning is the fold. So this + /// hands the fold eight pairs that differ from each other at both ends + /// and asserts the packed pair against the arithmetic: white's count less + /// black's, count by count, each half of the pair summed on its own. + #[test] + fn the_pawn_structure_fold_reads_white_less_black_count_by_count() { + let board = Board::from_fen(STRUCTURED).unwrap(); + assert_eq!(counts_of(&board, Color::White), WHITE_STRUCTURE); + assert_eq!(counts_of(&board, Color::Black), BLACK_STRUCTURE); + let midgame: i32 = (0..COUNTS) + .map(|i| mg_value(TRIAL[i]) * (WHITE_STRUCTURE[i] - BLACK_STRUCTURE[i])) + .sum(); + let endgame: i32 = (0..COUNTS) + .map(|i| eg_value(TRIAL[i]) * (WHITE_STRUCTURE[i] - BLACK_STRUCTURE[i])) + .sum(); + assert_ne!( + midgame, endgame, + "the two halves would not tell a swap apart" + ); + let packed = super::fold_with(&board, &TRIAL); + assert_ne!(midgame, 0, "black less white would answer the same here"); + assert_eq!((mg_value(packed), eg_value(packed)), (midgame, endgame)); + } +} diff --git a/arche-core/src/eval/shelter.rs b/arche-core/src/eval/shelter.rs new file mode 100644 index 00000000..2a0ee718 --- /dev/null +++ b/arche-core/src/eval/shelter.rs @@ -0,0 +1,700 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +// Copyright (C) 2022-2026 Andrew Wright + +//! What stands between each king and the board, and what that is worth. +//! +//! The term owns its counts, the masks they are read off, its weights, its +//! fold and the memo the search hands it. All seven counts are read off the +//! king's square, so a king move rewrites the side's whole reading and a pawn +//! move changes it wherever the pawn stood. There is nothing here for +//! `Accumulator::count` to add and take away a piece at a time. + +use super::pawn_structure::files_of; +use crate::board::{Board, ZOBRIST}; +use crate::misc::{Color, Piece}; +use crate::psqt::pack; + +/// How many ranks in front of the king are masked. Three, which is as far as +/// an enemy pawn is counted: for a king at home that is the rank its own pawns +/// start on and the two beyond it. +const RANKS_AHEAD: usize = 3; + +/// How many counts the king's shelter is measured in, and so how many weights +/// it carries at each end of the taper. [`counts_of`] prints them in this +/// order: this side's pawns one rank in front of its king, its pawns two ranks +/// in front, the king's files with no pawn of either colour on them, the +/// king's files holding an enemy pawn and none of ours, and then the enemy +/// pawns one, two and three ranks in front of the king. +pub(crate) const COUNTS: usize = 7; + +/// The three files a king on `square` stands behind, as a bit per file. +/// +/// A king on the a file or the h file is read against three files rather than +/// two, by stepping the middle one in: the b file and the g file are the +/// centres a corner king keeps. So every king square names three files and +/// the counts off them are on one scale wherever the king stands. +const fn king_files(square: u8) -> u8 { + let file = square % 8; + let centre = match file { + 0 => 1, + 7 => 6, + file => file, + }; + 0b111 << (centre - 1) +} + +/// The three squares `ahead` ranks in front of a king on `square`, on the +/// files [`king_files`] names, and empty where that rank is off the board. +/// +/// `forward` is the direction the side's pawns push, so a white king is read +/// up the board and a black king down it. A king that has walked far enough +/// up is left with nothing in front of it, which is the answer rather than a +/// case to rule out: a king off its own back ranks has no shelter, and what +/// that is worth is for the weights to say. +const fn shelter_rank(square: u8, forward: i8, ahead: i8) -> u64 { + let rank = (square / 8) as i8 + forward * ahead; + if rank < 0 || rank > 7 { + return 0; + } + // a file bit and a square index share their low three bits, so the byte + // shifted to the rank is the three squares on it + (king_files(square) as u64) << (rank * 8) +} + +/// The squares in front of each king, and which files it stands behind. +/// +/// `ahead` is indexed by how many ranks forward, then by `Color`'s +/// discriminant the way the accumulator's material is, then by the king's +/// square. The colour is an index because the ranks in front of a king run +/// opposite ways for the two; the files do not, and one entry serves both. +/// +/// The same three masks serve both sides of the term. Read against this side's +/// pawns they are the cover the king has, and against the other side's they +/// are the pawns coming for it. +struct Masks { + ahead: [[[u64; 64]; 2]; RANKS_AHEAD], + files: [u8; 64], +} + +impl Masks { + /// Built at compile time, the way the attack masks in `board` are. + const fn new() -> Self { + let mut masks = Masks { + ahead: [[[0; 64]; 2]; RANKS_AHEAD], + files: [0; 64], + }; + let mut square = 0u8; + while square < 64 { + let i = square as usize; + masks.files[i] = king_files(square); + let mut rank = 0; + while rank < RANKS_AHEAD { + let ahead = rank as i8 + 1; + masks.ahead[rank][Color::White as usize][i] = shelter_rank(square, 1, ahead); + masks.ahead[rank][Color::Black as usize][i] = shelter_rank(square, -1, ahead); + rank += 1; + } + square += 1; + } + masks + } +} + +static MASKS: Masks = Masks::new(); + +/// What one of those seven counts is worth, as the packed pairs the taper is +/// read from. +/// +/// Fitted 2026-09-12 by `scripts/tune.py` over the whole archived strength +/// run: 28,675 games (27,133 at 10+0.1, 1,500 at 30+0.3 and 42 at 2+0.02), +/// whose 3,711,074 post-book plies gave 3,536,193 positions and 1,623,149 +/// quiet rows in 28,618 of them, extracted by `arche terms` at 4e5bd7d, with +/// K held at 1.2071 and the games split 17,192 that trained, 5,654 that chose +/// the ridge of 1e-8 and 5,772 that were sealed. The 768 table entries, the +/// six material values and the eight mobility weights were all held where +/// they stand, so these fourteen are the only thing that moved. Every number +/// here is of the rounded vector that ships. The selection group scores +/// 0.090395 at zero and 0.089746 at these, a paired difference of -0.000649 +/// against a standard error of 0.000116 over its 5,654 games, at a design +/// factor of 3.9. That is outside its interval, which the mobility fit's +/// reading was not. +/// +/// The sealed group was opened once, after the games had accepted this +/// vector, over 5,772 games and 324,273 positions no fit and no ridge choice +/// had read. It scores 0.090914 at zero and 0.090377 at these, a paired +/// difference of -0.000537 against a standard error of 0.000118 at a design +/// factor of 3.9. That is outside its interval, and 0.68 standard errors +/// from the selection group's reading, so the two groups agree. The piece +/// square fit's two did not. +/// +/// Not one of the fourteen rounded to nothing, so every count is priced and +/// none can be left uncounted at the leaf the way `mobility::SCORED_KINDS` +/// leaves a mobility kind. [`Cache`] is what pays for the seven instead. +/// +/// Two of the storm's three signs are not what the term was named for. An +/// enemy pawn one rank in front of the king reads 11 and 35, and three ranks +/// out reads 14, so the fit likes the near storm where the term expected it +/// to fear it, and only the middle rank is negative. Our own cover is worth +/// 21 in the midgame and -26 in the ending, which reads as a king that wants +/// to be active rather than covered. The corpus is the first place to look +/// and not the term: 66.4% of its appearances have six or fewer pieces left +/// on the board and 6.0% have thirteen or more of the fourteen, so the +/// midgame half of the taper, which is the half king safety is about, is +/// fitted on the thinnest slice of the games. The count with the oddest +/// weight is also the thinnest supported: an enemy pawn one rank in front of +/// a king carries a coefficient in 4.65% of the rows against 47.83% for the +/// near cover, because a king usually takes such a pawn and the position is +/// then not quiet. The loss says these fourteen score the corpus better than +/// zero did, and that is all it says. +/// +/// A side's seven counts come to eighteen at the very most. Five of them are +/// at most three pawns each, and the other two share three files between them +/// rather than reaching three each. Against these weights the largest total +/// any legal set of counts reaches on one side is 168 in the midgame half and +/// -207 in the ending half, so a boardful of both colours leaves the sixteen +/// bits `pack` gives each half a long way off. Six of the fourteen are past +/// single figures, which the paragraph here said they would not be while they +/// were all zero. +static SHELTER: [i32; COUNTS] = [ + pack(10, -11), + pack(21, -26), + pack(-11, -26), + pack(-8, -1), + pack(11, 35), + pack(-18, 2), + pack(14, -6), +]; + +/// The weight of one of the seven counts, as the packed pair. The tuner's +/// seam asks through [`super::TERMS`], so that a slot names the live weight +/// rather than a copy of it, the way it reads the tables. +pub(crate) fn weight(index: usize) -> i32 { + SHELTER[index] +} + +/// What this term depends on and nothing else: both sides' pawns and both +/// kings' squares. +/// +/// [`counts_of`] reads the pawn boards and the two king squares and nothing +/// else, so two positions whose pawns and kings agree agree on every count of +/// it whatever else has moved. This key stands in for that agreement rather +/// than being it: two positions can share it and differ, which takes a +/// collision across the whole sixty four bits. The pawn key already hashes the +/// pawns of both colours and is kept in step move by move, so this is that key +/// with the two kings folded in, from the same zobrist table the position key +/// uses. +/// +/// Composed here rather than maintained beside `Board::pawn_key`, because a +/// king move would then have to write it and the cost of this is two loads and +/// two xors at the one place that asks. +#[inline] +pub(crate) fn key(board: &Board) -> u64 { + board.pawn_key + ^ ZOBRIST.get_piece_key(board.king_index(Color::White), Piece::King, Color::White) + ^ ZOBRIST.get_piece_key(board.king_index(Color::Black), Piece::King, Color::Black) +} + +/// What stands between this side's king and the board, as seven counts in the +/// order [`COUNTS`] names them: this side's pawns one rank in front of the +/// king and two ranks in front, how many of the king's three files hold no +/// pawn of either colour, how many hold an enemy pawn and none of this side's, +/// and then the enemy pawns one, two and three ranks in front of the king. +/// +/// All seven are read off the three files the king stands behind, which +/// [`king_files`] steps in at the two corners so that the counts mean the +/// same thing on every square. Pawns and nothing else: a piece in front of +/// the king shelters it too, but a term that pays for one pays a piece to +/// sit still, and the piece square tables already hold an opinion about +/// where a piece belongs. A pawn is the part of the cover the king cannot +/// get back. +/// +/// The last three are the storm, and they are the same masks read against +/// the other side's pawns. A pawn of ours on g3 is cover and a pawn of +/// theirs on g3 is not the absence of cover, it is a lever, so the two are +/// counted apart and each rank apart from the next: how far the storm has +/// come is most of what it is worth, and the weights are where that is +/// said. Three ranks is as far as it is followed, which for a king at home +/// reaches the fourth rank. +/// +/// The two file counts overlap the two pawn counts, since a file with no +/// pawn of ours on it adds nothing to either of those. They are kept apart +/// because they are different knowledge: a missing g pawn and a g pawn +/// pushed to g4 both leave the near count short, and only the first opens +/// the file to a rook. +/// +/// Nothing here is gated on the king standing at home. A king that has +/// walked up the board has no rank in front of it inside the masks and +/// counts nothing, so the term fades rather than falling off a cliff the +/// search could step over. +/// +/// Both the evaluation and the tuner's walk read this, so the identity +/// between them cannot see a wrong count here. What pins it is the hand counts +/// in the tests below, the way the mobility counts are pinned. +#[inline] +pub(crate) fn counts_of(board: &Board, color: Color) -> [i32; COUNTS] { + let masks = &MASKS; + let square = board.king_index(color) as usize; + let side = color as usize; + let (ours, theirs) = board.sides(color); + let pawns = board.pawns(); + let (our_pawns, their_pawns) = (pawns & ours, pawns & theirs); + let ahead = + |pawns: u64, rank: usize| (pawns & masks.ahead[rank][side][square]).count_ones() as i32; + // the king's files with no pawn of ours on them, split by whether the + // other side has one there + let files = masks.files[square]; + let bare = files & !files_of(our_pawns); + let theirs_on = files_of(their_pawns); + let open = (bare & !theirs_on).count_ones() as i32; + let half_open = (bare & theirs_on).count_ones() as i32; + [ + ahead(our_pawns, 0), + ahead(our_pawns, 1), + open, + half_open, + ahead(their_pawns, 0), + ahead(their_pawns, 1), + ahead(their_pawns, 2), + ] +} + +/// The seven counts, written into `into`, which is what [`super::TERMS`] hands +/// the tuner's walk. +pub(crate) fn counts(board: &Board, color: Color, into: &mut [i32]) { + into.copy_from_slice(&counts_of(board, color)); +} + +/// What white's king shelter stands ahead by, as a packed pair on the scale +/// the piece square pair is on. +#[inline] +pub(crate) fn fold(board: &Board) -> i32 { + fold_with(board, &SHELTER) +} + +/// The same fold against weights named by the caller. +/// +/// The live weights are the fit's now, and the two halves of every one of +/// them differ, so the sign of this term, the order of the seven counts and +/// the packing all show in an evaluation the engine prints and in the rows +/// the tuner's walk states. The tests still supply weights of their own +/// through here, because what they pin is the fold rather than the fit: a +/// permuted [`SHELTER`] would be a different evaluation and not a wrong one. +#[inline] +fn fold_with(board: &Board, weights: &[i32; COUNTS]) -> i32 { + let white = counts_of(board, Color::White); + let black = counts_of(board, Color::Black); + let mut packed = 0; + for ((weight, white), black) in weights.iter().zip(white).zip(black) { + packed += weight * (white - black); + } + packed +} + +/// How many shelter scores the cache holds. A power of two, so the index is +/// a mask rather than a remainder. +/// +/// Eight thousand entries at sixteen bytes is a hundred and twenty eight +/// kilobytes, which is past the first level cache and inside the second. +/// The size was measured rather than reasoned about. Callgrind over the +/// bench with the cache simulated, at eleven, twelve, thirteen and fourteen +/// bits, reads 3,968,905,639, 3,958,897,319, 3,950,063,019 and 3,943,552,801 +/// instructions against last level misses of 284,996, 285,117, 286,191 and +/// 292,291. Each bit buys fewer instructions than the one before it and the +/// misses are flat until fourteen, where they turn up by six thousand. So +/// this is the last size the memory does not notice, and the whole range is +/// within two thirds of a percent of instructions: the constant is not +/// load bearing and a later working set can move it. +const CACHE_BITS: usize = 13; +pub(super) const CACHE_SLOTS: usize = 1 << CACHE_BITS; + +/// One remembered shelter score, under the key that decides it. +/// +/// The whole key is kept rather than the bits the index does not use, so a +/// hit is a hit on the position's pawns and kings and not on a tag that +/// happens to agree. A wrong hit here would be a silently wrong evaluation, +/// which is the one error a search does not report, and sixty four bits of +/// key costs four bytes against the alternative. +#[derive(Copy, Clone)] +struct Entry { + key: u64, + packed: i32, +} + +/// The king shelter, remembered by what it depends on. +/// +/// The term reads both sides' pawns and the two king squares and nothing +/// else, so a position that agrees with a remembered one on those scores the +/// same however its pieces stand. Most moves in a search are piece moves, +/// which leave every one of those alone, so the score computed at one leaf +/// answers a great many of the leaves after it. +/// +/// Direct mapped and never cleared. An entry is only ever read against the +/// key that wrote it, so a stale one is a miss rather than a wrong answer, +/// and a search that begins with the last search's entries begins with a warm +/// cache. That is also what leaves the node counts alone: the cache changes +/// how a score is arrived at and not what it is, so the counts are the ones +/// the weights alone produce. +/// +/// Owned by the searcher rather than by the board, because it is scratch and +/// not position. A board carries what it would take to undo a move and +/// compares equal to another board holding the same position; a cache does +/// neither, and one thread's cache is its own. +pub(super) struct Cache { + entries: Box<[Entry]>, +} + +impl Default for Cache { + fn default() -> Self { + // an empty entry is key zero holding a score of zero, so a position + // whose pawns and kings xor to nothing would read it as its own and + // take nothing from it. That is a sixty four bit coincidence, which + // is the same one a wrong hit needs anywhere else in the table + Cache { + entries: vec![Entry { key: 0, packed: 0 }; CACHE_SLOTS].into_boxed_slice(), + } + } +} + +impl Cache { + /// What white's shelter stands ahead by, remembered or computed. + #[inline] + pub(super) fn get(&mut self, board: &Board) -> i32 { + let key = key(board); + let slot = (key as usize) & (CACHE_SLOTS - 1); + let entry = &mut self.entries[slot]; + if entry.key == key { + return entry.packed; + } + let packed = fold(board); + *entry = Entry { key, packed }; + packed + } +} + +#[cfg(test)] +mod tests { + use super::{ + Board, CACHE_SLOTS, COUNTS, Cache, Color, MASKS, RANKS_AHEAD, counts_of, files_of, fold, + key, king_files, + }; + use crate::board::fens; + use crate::psqt::{eg_value, mg_value, pack}; + use pretty_assertions::assert_eq; + + /// A remembered score is read back rather than recomputed, which is the + /// whole point of the cache and is not visible in what it answers. + /// + /// Asked twice for one position, the second answer comes from the entry + /// the first wrote. There is no counter to read, so this says it the way + /// a caller could: the entry the key lands on holds the score after the + /// first call, and holds it under that key and no other. + #[test] + fn a_score_is_remembered_under_the_key_that_wrote_it() { + let board = Board::from_fen(fens::MIDDLEGAME).unwrap(); + let mut cache = Cache::default(); + let key = key(&board); + let slot = (key as usize) & (CACHE_SLOTS - 1); + assert_eq!(cache.entries[slot].key, 0, "the slot starts empty"); + let first = cache.get(&board); + assert_eq!(cache.entries[slot].key, key); + assert_eq!(cache.entries[slot].packed, fold(&board)); + assert_eq!(cache.get(&board), first); + } + + /// The key is the pawn key with the two kings folded in, which is what + /// the cache is keyed on. What the term reads is the pawns and the two + /// king squares, so a piece that is neither has to leave it alone and + /// either king moving has to move it. A key that missed a king would hand + /// one position's shelter to another. + #[test] + fn the_shelter_key_follows_the_pawns_and_the_two_kings() { + let bare = Board::from_fen("4k3/pppppppp/8/8/8/8/PPPPPPPP/4K3 w - - 0 1").unwrap(); + + // the same pawns and the same two kings behind a boardful of other + // pieces, which the shelter does not read + let pieced = + Board::from_fen("rnbqk1nr/pppppppp/8/8/8/8/PPPPPPPP/RNBQK1NR w - - 0 1").unwrap(); + assert_eq!(key(&bare), key(&pieced)); + + // either king one square along + let ours = Board::from_fen("4k3/pppppppp/8/8/8/8/PPPPPPPP/5K2 w - - 0 1").unwrap(); + let theirs = Board::from_fen("5k2/pppppppp/8/8/8/8/PPPPPPPP/4K3 w - - 0 1").unwrap(); + assert_ne!(key(&bare), key(&ours)); + assert_ne!(key(&bare), key(&theirs)); + assert_ne!(key(&ours), key(&theirs)); + + // and one pawn pushed, the pawn key being the rest of it + let pushed = Board::from_fen("4k3/pppppppp/8/8/8/7P/PPPPPPP1/4K3 w - - 0 1").unwrap(); + assert_ne!(key(&bare), key(&pushed)); + } + + /// The counts by hand, square by square, because nothing else pins them. + /// The tuner's identity folds a row against the live weights, and `eval` + /// and the tuner's walk read the same helper, so the two sides of the + /// identity move together whatever the helper answers. These cases are the + /// only check this term has. + /// + /// Each case names what the count is made of, in the order the helper + /// returns them: our pawns one rank ahead and two, the king's files that + /// hold no pawn at all, the ones that hold an enemy pawn and none of ours, + /// and then the enemy pawns one, two and three ranks ahead. The other king + /// stands out of the way. + #[test] + fn a_king_shelters_behind_what_a_hand_count_says_it_does() { + for (fen, counts, why) in [ + // three pawns where a castled king wants them + ( + "k7/8/8/8/8/8/5PPP/6K1 w - - 0 1", + [3, 0, 0, 0, 0, 0, 0], + "a king on g1 behind f2, g2 and h2", + ), + // the g pawn one square further on is the far rank rather than + // the near one + ( + "k7/8/8/8/8/6P1/5P1P/6K1 w - - 0 1", + [2, 1, 0, 0, 0, 0, 0], + "a king on g1 with the g pawn on g3", + ), + // the g file holds no pawn of either colour + ( + "k7/8/8/8/8/8/5P1P/6K1 w - - 0 1", + [2, 0, 1, 0, 0, 0, 0], + "a king on g1 with no g pawn", + ), + // the same file with a black pawn on it is half open rather than + // open. The pawn is on g7, which is past the three ranks the + // storm is followed over, so it is a file and not a storm + ( + "k7/6p1/8/8/8/8/5P1P/6K1 w - - 0 1", + [2, 0, 0, 1, 0, 0, 0], + "a king on g1 with a black pawn on g7", + ), + // h2 near, g3 far, and the f file holding a black pawn on f5, + // which is a rank further out than the storm reaches + ( + "k7/8/8/5p2/8/6P1/7P/6K1 w - - 0 1", + [1, 1, 0, 1, 0, 0, 0], + "a king on g1 with the f file gone", + ), + // an enemy pawn standing on a rank in front of the king is not + // cover. It is the storm, counted by the rank it has reached + ( + "k7/8/8/8/8/8/5PpP/6K1 w - - 0 1", + [2, 0, 0, 1, 1, 0, 0], + "a king on g1 with a black pawn on g2", + ), + ( + "k7/8/8/8/8/6p1/5P1P/6K1 w - - 0 1", + [2, 0, 0, 1, 0, 1, 0], + "a king on g1 with a black pawn on g3", + ), + ( + "k7/8/8/8/6p1/8/5P1P/6K1 w - - 0 1", + [2, 0, 0, 1, 0, 0, 1], + "a king on g1 with a black pawn on g4", + ), + // two ranks of storm at once, on two files + ( + "k7/8/8/8/7p/6p1/5P1P/6K1 w - - 0 1", + [2, 0, 0, 1, 0, 1, 1], + "a king on g1 against pawns on g3 and h4", + ), + // a king in the corner is read against three files, so the f file + // it does not stand beside is still counted. Two files would + // leave this at nothing + ( + "k7/8/8/8/8/8/6PP/7K w - - 0 1", + [2, 0, 1, 0, 0, 0, 0], + "a king on h1 behind g2 and h2", + ), + ( + "k7/8/8/8/8/8/PPP5/K7 w - - 0 1", + [3, 0, 0, 0, 0, 0, 0], + "a king on a1 behind a2, b2 and c2", + ), + // a king off its own ranks has nothing on the masked ranks, so + // the pawns it left behind are not shelter + ( + "k7/8/8/4K3/8/8/3PPP2/8 w - - 0 1", + [0, 0, 0, 0, 0, 0, 0], + "a king on e5 with its pawns at home", + ), + // a pawn on the file stops it counting as open wherever on the + // file it stands, so a passed pawn up the board is not a hole + // behind the king it left + ( + "k7/6P1/8/8/8/8/8/6K1 w - - 0 1", + [0, 0, 2, 0, 0, 0, 0], + "a king on g1 whose only pawn is on g7", + ), + // the ranks in front of a king on the eighth are off the board + // rather than round the other side of it + ( + "4K3/8/8/8/8/8/8/k7 w - - 0 1", + [0, 0, 3, 0, 0, 0, 0], + "a king on e8 with no pawns anywhere", + ), + ] { + let board = Board::from_fen(fen).unwrap(); + assert_eq!(counts_of(&board, Color::White), counts, "{}", why); + } + } + + /// The same reading for black, whose king is measured down the board + /// rather than up it. + #[test] + fn a_black_king_is_measured_down_the_board() { + // the rank in front of a king on the first is off the board, and the + // three files all hold a white pawn and no black one + let board = Board::from_fen("7K/8/8/8/8/8/3PPP2/4k3 b - - 0 1").unwrap(); + assert_eq!(counts_of(&board, Color::Black), [0, 0, 0, 3, 0, 0, 0]); + // and the storm the other way up: the white pawn on g6 stands two + // ranks in front of a black king on g8, and is not its cover + let stormed = Board::from_fen("6k1/5ppp/6P1/8/8/8/8/6K1 b - - 0 1").unwrap(); + assert_eq!(counts_of(&stormed, Color::Black), [3, 0, 0, 0, 0, 1, 0]); + } + + /// Black's count of a position is white's count of its reflection, so the + /// two colours are read the same way round. + #[test] + fn the_two_colours_count_the_same_squares() { + let white = Board::from_fen("k7/8/8/5p2/8/6P1/7P/6K1 w - - 0 1").unwrap(); + let black = Board::from_fen("6k1/7p/6p1/8/5P2/8/8/K7 b - - 0 1").unwrap(); + assert_eq!( + counts_of(&white, Color::White), + counts_of(&black, Color::Black) + ); + // and the storm half of it, which the pair above leaves at zero + let stormed = Board::from_fen("k7/8/8/8/7p/6p1/5P1P/6K1 w - - 0 1").unwrap(); + let mirrored = Board::from_fen("6k1/5p1p/6P1/7P/8/8/8/K7 b - - 0 1").unwrap(); + assert_eq!( + counts_of(&stormed, Color::White), + counts_of(&mirrored, Color::Black) + ); + } + + /// Every square names three files, the two corners included, and the + /// three are the king's own file and its neighbours wherever there is + /// room for them. + #[test] + fn every_king_square_names_three_files() { + for square in 0..64u8 { + let files = king_files(square); + assert_eq!(files.count_ones(), 3, "the files of {}", square); + assert_eq!( + files & (1 << (square % 8)), + 1 << (square % 8), + "the king's own file is not among the files of {}", + square + ); + assert_eq!( + files.trailing_zeros() + 2, + 7 - files.leading_zeros(), + "the files of {} are not three in a row", + square + ); + } + } + + /// Each mask holds the rank its index names, on those same three files, + /// and nothing where the board has run out. + #[test] + fn the_masks_hold_the_ranks_in_front_of_the_king() { + for square in 0..64u8 { + let rank = i32::from(square / 8); + for (side, forward) in [(Color::White, 1), (Color::Black, -1)] { + let i = side as usize; + for step in 0..RANKS_AHEAD { + let mask = MASKS.ahead[step][i][square as usize]; + let target = rank + forward * (step as i32 + 1); + if !(0..8).contains(&target) { + assert_eq!(mask, 0, "{:?} on {} at {} ahead", side, square, step + 1); + continue; + } + assert_eq!(mask.count_ones(), 3, "{:?} on {}", side, square); + assert_eq!( + files_of(mask), + king_files(square), + "{:?} on {} covers other files", + side, + square + ); + let rank_mask = 0xffu64 << (target * 8); + assert_eq!( + mask & rank_mask, + mask, + "{:?} on {} is off its rank", + side, + square + ); + } + } + } + } + + /// The position the fold test below is read against, and what each side + /// counts in it, worked out by hand rather than read back off + /// [`counts_of`]. + /// + /// White's king on g1 stands behind the f, g and h files. It has f2 and h2 + /// one rank ahead and g3 two, and all three files hold a pawn of its own, + /// so neither file count fires. Coming the other way it faces g2 one rank + /// ahead, f3 and h3 two, and f4, g4 and h4 three. + /// + /// Black's king on b8 stands behind the a, b and c files with nothing on + /// either rank in front of it and no white pawn within three ranks, so its + /// storm is empty. White's pawn on a4 leaves that file half open and the b + /// and c files hold no pawn at all. + /// + /// The seven differences are 2, 1, -2, -1, 1, 2 and 3. None is zero, so + /// every slot is doing work in the assertion below, which a position with + /// an empty storm would not manage. + const SHELTERED: &str = "1k6/8/8/8/P4ppp/5pPp/5PpP/6K1 w - - 0 1"; + const WHITE_SHELTERS: [i32; COUNTS] = [2, 1, 0, 0, 1, 2, 3]; + const BLACK_SHELTERS: [i32; COUNTS] = [0, 0, 2, 1, 0, 0, 0]; + + /// Seven weights that differ from each other at both ends of the taper, so + /// that a pair read into the wrong count's slot lands on a different + /// number. The seven differences between the halves are 9, -20, 8, -12, + /// 20, -29 and -14, which differ from each other too, so a permutation of + /// either array shows. + const TRIAL: [i32; COUNTS] = [ + pack(11, 2), + pack(-7, 13), + pack(3, -5), + pack(29, 41), + pack(17, -3), + pack(-23, 6), + pack(5, 19), + ]; + + /// What the fold does with weights that are not the shipped ones. + /// + /// The shipped weights would do here now that they are not zero. Weights + /// of this test's own are kept anyway, because the shipped ones are the + /// fit's and will move again: a pin written against them would have to + /// be rewritten by every refit, and what it is pinning is the fold. So + /// this hands the fold seven pairs that differ from each other at both + /// ends and asserts the packed pair against the arithmetic: white's count + /// less black's, count by count, each half of the pair summed on its + /// own. + #[test] + fn the_shelter_fold_reads_white_less_black_count_by_count() { + let board = Board::from_fen(SHELTERED).unwrap(); + assert_eq!(counts_of(&board, Color::White), WHITE_SHELTERS); + assert_eq!(counts_of(&board, Color::Black), BLACK_SHELTERS); + let midgame: i32 = (0..COUNTS) + .map(|i| mg_value(TRIAL[i]) * (WHITE_SHELTERS[i] - BLACK_SHELTERS[i])) + .sum(); + let endgame: i32 = (0..COUNTS) + .map(|i| eg_value(TRIAL[i]) * (WHITE_SHELTERS[i] - BLACK_SHELTERS[i])) + .sum(); + assert_ne!( + midgame, endgame, + "the two halves would not tell a swap apart" + ); + let packed = super::fold_with(&board, &TRIAL); + assert_ne!(midgame, 0, "black less white would answer the same here"); + assert_eq!((mg_value(packed), eg_value(packed)), (midgame, endgame)); + } +} diff --git a/arche-core/src/tune.rs b/arche-core/src/tune.rs index f7957109..ceb3fc7a 100644 --- a/arche-core/src/tune.rs +++ b/arche-core/src/tune.rs @@ -28,18 +28,23 @@ //! is asserted on every row a run prints rather than only in a test. //! //! The walk is a third pass over the board, beside `Accumulator::count` and -//! `Accumulator::recomputed`. eval.rs defends that duplication on the second +//! `Accumulator::recomputed`. eval/mod.rs defends that duplication on the second //! and the same terms hold here: two implementations that agree are a check, //! and a helper shared between them is not. The accumulator the search keeps //! is neither read nor duplicated. //! //! The three leaf terms are the exception, and it is deliberate. The walk -//! asks `Board::mobility_counts`, `Board::shelter_counts` and -//! `Board::pawn_structure_counts` for their counts and so does `eval`, so the -//! identity cannot see a wrong count at any weights, fitted or zero. A second -//! count here would be a second chance to be wrong about a term that is read -//! at every leaf rather than a check on the first, so what pins them is the -//! hand counts beside each helper in board.rs. +//! asks each term in `eval::TERMS` for its counts through the descriptor, and +//! what it reaches is the function `eval` reads, so the identity cannot see a +//! wrong count at any weights, fitted or zero. A second count here would be a +//! second chance to be wrong about a term that is read at every leaf rather +//! than a check on the first, so what pins them is the hand counts beside each +//! term in `eval/`. +//! +//! The layout is that list too. Where a term stands in the vector, what its +//! weights are and how many counts it carries all come off `eval::TERMS`, so a +//! term added there is laid out, priced and walked here without an edit, and +//! the run's layout line states the result for `scripts/tune.py` to read. //! //! On mobility the two may ask for different kinds. The walk always asks for //! all four, because it is offline and a coefficient for a kind worth nothing @@ -82,44 +87,39 @@ pub const ENDGAME_SLOTS: usize = 6 * 64; /// the tables. pub const MATERIAL_SLOT: usize = MIDGAME_SLOTS + ENDGAME_SLOTS; -/// Where the mobility weights stand, after the material block: four midgame -/// weights, one for each of `eval::MOBILE_PIECES`, then the same four at the -/// endgame end. After the material rather than beside the tables, so that -/// adding them moved no slot a fit has already been written against. -pub const MOBILITY_SLOT: usize = MATERIAL_SLOT + 6; - -/// How many pieces carry a mobility weight, which is how far apart a piece's -/// two mobility weights are. -const MOBILITY_SLOTS: usize = eval::MOBILE_PIECES.len(); - -/// Where the shelter weights stand, after the mobility block and laid out the -/// same way: seven midgame weights, one for each of the counts -/// `eval::SHELTER_TERMS` names, then the same seven at the endgame end. Each -/// term appended rather than inserted, so that adding one moves no slot a fit -/// has already been written against. Growing one in place is not that: the -/// storm took this block from eight weights to fourteen, and the endgame half -/// moved with it. -pub const SHELTER_SLOT: usize = MOBILITY_SLOT + 2 * MOBILITY_SLOTS; +/// How many material values there are, one per piece. +const MATERIAL_SLOTS: usize = 6; -/// How many counts the shelter is measured in, which is how far apart a -/// count's two weights are. -const SHELTER_SLOTS: usize = eval::SHELTER_TERMS; - -/// Where the pawn structure weights stand, after the shelter block and laid -/// out the same way: eight midgame weights, one for each of the counts -/// `eval::PAWN_TERMS` names, then the same eight at the endgame end. -/// Appended rather than inserted, on the rule the two blocks above it follow, -/// so no slot a fit has already been written against has moved. -pub const PAWN_SLOT: usize = SHELTER_SLOT + 2 * SHELTER_SLOTS; +/// Where the leaf terms start, after the material block. After it rather than +/// beside the tables, so that adding the first of them moved no slot a fit +/// had already been written against. +/// +/// Each term takes twice its width from here on, its midgame half first, in +/// `eval::TERMS` order. A term is appended rather than inserted for the same +/// reason the block is: mobility stands where it did when it was the only one +/// here, and the shelter and the pawn structure were each added after it. +/// Growing one in place is not that, and the storm is what says so: it took +/// the shelter from eight weights to fourteen and moved the endgame half of +/// the vector with it. +pub const TERM_SLOT: usize = MATERIAL_SLOT + MATERIAL_SLOTS; -/// How many counts the pawn structure is measured in, which is how far apart -/// a count's two weights are. -const PAWN_SLOTS: usize = eval::PAWN_TERMS; +/// Where the term at `index` in `eval::TERMS` starts, which is the sum of the +/// widths before it, doubled. +const fn term_slot(index: usize) -> usize { + let mut slot = TERM_SLOT; + let mut before = 0; + while before < index { + slot += 2 * eval::TERMS[before].width; + before += 1; + } + slot +} /// The whole weight vector: 384 midgame entries, 384 endgame ones, the six -/// material values, the eight mobility weights, the fourteen shelter ones and -/// the sixteen pawn structure ones. -pub const SLOTS: usize = PAWN_SLOT + 2 * PAWN_SLOTS; +/// material values, and then each leaf term's widths twice over. The tables +/// and the material are written here; everything after them follows from the +/// term list, so a term added there moves this without a second edit. +pub const SLOTS: usize = term_slot(eval::TERMS.len()); /// The weight a slot names. /// @@ -142,32 +142,33 @@ pub fn weight(slot: usize) -> i32 { PieceSquareTables::TABLES.get_value(entry, piece, Color::Black) }; if slot < MIDGAME_SLOTS { - mg_value(packed(Piece::PIECES[slot / 64], slot % 64)) - } else if slot < MATERIAL_SLOT { + return mg_value(packed(Piece::PIECES[slot / 64], slot % 64)); + } + if slot < MATERIAL_SLOT { let entry = slot - MIDGAME_SLOTS; - eg_value(packed(Piece::PIECES[entry / 64], entry % 64)) - } else if slot < MOBILITY_SLOT { - eval::material(Piece::PIECES[slot - MATERIAL_SLOT]) as i32 - } else if slot < MOBILITY_SLOT + MOBILITY_SLOTS { - mg_value(eval::mobility_weight(slot - MOBILITY_SLOT)) - } else if slot < SHELTER_SLOT { - eg_value(eval::mobility_weight(slot - MOBILITY_SLOT - MOBILITY_SLOTS)) - } else if slot < SHELTER_SLOT + SHELTER_SLOTS { - mg_value(eval::shelter_weight(slot - SHELTER_SLOT)) - } else if slot < PAWN_SLOT { - eg_value(eval::shelter_weight(slot - SHELTER_SLOT - SHELTER_SLOTS)) - } else if slot < PAWN_SLOT + PAWN_SLOTS { - mg_value(eval::pawn_weight(slot - PAWN_SLOT)) - } else { - eg_value(eval::pawn_weight(slot - PAWN_SLOT - PAWN_SLOTS)) + return eg_value(packed(Piece::PIECES[entry / 64], entry % 64)); + } + if slot < TERM_SLOT { + return eval::material(Piece::PIECES[slot - MATERIAL_SLOT]) as i32; } + let mut index = slot - TERM_SLOT; + for term in eval::TERMS { + if index < term.width { + return mg_value((term.weight)(index)); + } + if index < 2 * term.width { + return eg_value((term.weight)(index - term.width)); + } + index -= 2 * term.width; + } + panic!("slot {} is past the {} the vector holds", slot, SLOTS) } /// Whether a slot is one of the material values, which are the only weights -/// added outside the taper's divide. Everything else is inside it, mobility -/// and the shelter included. +/// added outside the taper's divide. Everything else is inside it, the leaf +/// terms included. fn is_material(slot: usize) -> bool { - (MATERIAL_SLOT..MOBILITY_SLOT).contains(&slot) + (MATERIAL_SLOT..TERM_SLOT).contains(&slot) } /// One position's evaluation, decomposed over the weight vector. @@ -231,25 +232,20 @@ impl Terms { coefficients[midgame] += sign * phase; coefficients[MIDGAME_SLOTS + midgame] += sign * (TOTAL_PHASE - phase); } - // the three leaf terms are per position rather than per square, so - // their counts come off the board whole rather than out of the walk - // above. Tapered the way a square is, and so two slots per count + // the leaf terms are per position rather than per square, so their + // counts come off the board whole rather than out of the walk above. + // Tapered the way a square is, and so two slots per count + let mut counts = [0; eval::WIDEST]; for (color, sign) in [(Color::White, mover), (Color::Black, -mover)] { - let counts = board.mobility_counts::<{ eval::ALL_KINDS }>(color); - for (index, count) in counts.into_iter().enumerate() { - coefficients[MOBILITY_SLOT + index] += sign * count * phase; - coefficients[MOBILITY_SLOT + MOBILITY_SLOTS + index] += - sign * count * (TOTAL_PHASE - phase); - } - for (index, count) in board.shelter_counts(color).into_iter().enumerate() { - coefficients[SHELTER_SLOT + index] += sign * count * phase; - coefficients[SHELTER_SLOT + SHELTER_SLOTS + index] += - sign * count * (TOTAL_PHASE - phase); - } - for (index, count) in board.pawn_structure_counts(color).into_iter().enumerate() { - coefficients[PAWN_SLOT + index] += sign * count * phase; - coefficients[PAWN_SLOT + PAWN_SLOTS + index] += - sign * count * (TOTAL_PHASE - phase); + let mut slot = TERM_SLOT; + for term in eval::TERMS { + let counts = &mut counts[..term.width]; + (term.counts)(board, color, counts); + for (index, count) in counts.iter().enumerate() { + coefficients[slot + index] += sign * count * phase; + coefficients[slot + term.width + index] += sign * count * (TOTAL_PHASE - phase); + } + slot += 2 * term.width; } } Self { @@ -449,7 +445,18 @@ pub fn run(positions: &[Position], suite: Option<&str>) -> Report { } /// The report as the command prints it: a header naming the suite and what -/// the filter did with it, the weight vector, then a row a position. +/// the filter did with it, the layout the vector is in, the weight vector, +/// then a row a position. +/// +/// The layout line is `layout midgame 384 endgame 384 material 6` and then a +/// term and its width for each of `eval::TERMS`, a width being the counts a +/// term is measured in per side and per half of the taper. A term takes twice +/// that in slots, its midgame half first, in the order printed. It is spelled +/// off the table rather than written out, so what reads these rows learns the +/// layout from the run that printed them instead of holding a copy of it: +/// `scripts/tune.py` derives its slots from this line and refuses a run whose +/// terms it has no bounds for, where a copy would have read the numbers on to +/// the wrong weights and fitted them. /// /// A row is `id eval phase n slot:coefficient... fen`, whitespace separated, /// and both ends of it can hold spaces. A fen is six fields, and an id is @@ -494,6 +501,15 @@ impl fmt::Display for Report { self.drawn, self.rows.len(), )?; + write!( + f, + "layout midgame {} endgame {} material {}", + MIDGAME_SLOTS, ENDGAME_SLOTS, MATERIAL_SLOTS + )?; + for term in eval::TERMS { + write!(f, " {} {}", term.name, term.width)?; + } + writeln!(f)?; write!(f, "weights {}", SLOTS)?; for slot in 0..SLOTS { write!(f, " {}", weight(slot))?; @@ -527,6 +543,20 @@ mod tests { use crate::{bench, strategy}; use pretty_assertions::assert_eq; + /// Where a named term's block starts and how wide it is, which is what a + /// test needs to read one term's coefficients. Read off the same table the + /// layout is, so a test names a term rather than a slot number. + fn term(name: &str) -> (usize, usize) { + let mut slot = TERM_SLOT; + for term in eval::TERMS { + if term.name == name { + return (slot, term.width); + } + slot += 2 * term.width; + } + panic!("no term is called {}", name) + } + /// Every position three suites of different shapes hold: the shared /// fens, the bench's eighteen and the strategic suite's fifteen hundred. fn every_shape() -> Vec { @@ -665,14 +695,15 @@ mod tests { // way. The pawn on e2 blocks the diagonal beside that one (3, 10, "queen"), ] { + let (start, width) = term("mobility"); assert_eq!( - coefficient(MOBILITY_SLOT + index), + coefficient(start + index), count * terms.phase, "{} midgame", why ); assert_eq!( - coefficient(MOBILITY_SLOT + MOBILITY_SLOTS + index), + coefficient(start + width + index), count * (TOTAL_PHASE - terms.phase), "{} endgame", why @@ -696,7 +727,7 @@ mod tests { terms .coefficients .iter() - .any(|(slot, _)| usize::from(*slot) >= MOBILITY_SLOT), + .any(|(slot, _)| usize::from(*slot) >= term("mobility").0), "no mobility coefficient here, so this test says nothing about one" ); assert_eq!(terms, Terms::of(&black)); @@ -883,16 +914,17 @@ mod tests { midgame += largest(mg_value); endgame += largest(eg_value); } - let shelter = |half: fn(i32) -> i32| { - (0..SHELTER_SLOTS) - .map(|index| half(eval::shelter_weight(index)).abs()) - .sum::() - }; - let pawns = |half: fn(i32) -> i32| { - (0..PAWN_SLOTS) - .map(|index| half(eval::pawn_weight(index)).abs()) + let charged = |name: &str, half: fn(i32) -> i32| { + let named = eval::TERMS + .iter() + .find(|term| term.name == name) + .expect(name); + (0..named.width) + .map(|index| half((named.weight)(index)).abs()) .sum::() }; + let shelter = |half: fn(i32) -> i32| charged("shelter", half); + let pawns = |half: fn(i32) -> i32| charged("pawn_structure", half); // both sides at once, which is what the accumulator carries let worst = 2 * midgame.max(endgame) + 2 * MAX_SHELTER * shelter(mg_value).max(shelter(eg_value)) @@ -1010,8 +1042,44 @@ mod tests { "{}", text.lines().next().unwrap_or_default() ); - // the header, the weights and a row a position - assert_eq!(text.lines().count(), report.rows.len() + 2); + // the header, the layout, the weights and a row a position + assert_eq!(text.lines().count(), report.rows.len() + 3); + } + + /// The layout line states where every block of the vector stands, and + /// states it off the same table the slots are laid out from. + /// + /// What reads these rows has no copy of the layout, so this is the whole + /// of what it is told. A term renamed, dropped or given a different width + /// shows here, and what reads the line refuses a term it cannot price + /// rather than reading the numbers into a layout of its own. + #[test] + fn a_run_prints_the_layout_the_slots_are_laid_out_from() { + let text = run(&bench::positions()[..1], None).to_string(); + let line = text.lines().nth(1).expect("a layout line"); + let mut words = line.split(' '); + assert_eq!(words.next(), Some("layout")); + let mut named: Vec<(String, usize)> = Vec::new(); + while let Some(name) = words.next() { + let width = words.next().expect("a width").parse().expect("a width"); + named.push((name.to_string(), width)); + } + let mut expected: Vec<(String, usize)> = vec![ + ("midgame".to_string(), MIDGAME_SLOTS), + ("endgame".to_string(), ENDGAME_SLOTS), + ("material".to_string(), MATERIAL_SLOTS), + ]; + expected.extend( + eval::TERMS + .iter() + .map(|term| (term.name.to_string(), term.width)), + ); + assert_eq!(named, expected); + // the three blocks are counted whole and a term twice over, which is + // what the line leaves a reader to work out + let counted: usize = named[..3].iter().map(|(_, width)| width).sum::() + + 2 * named[3..].iter().map(|(_, width)| width).sum::(); + assert_eq!(counted, SLOTS); } /// The weight vector is printed beside the coefficients, so that nothing @@ -1021,7 +1089,7 @@ mod tests { fn a_run_prints_the_weights_the_coefficients_are_read_against() { let report = run(&bench::positions()[..1], None); let text = report.to_string(); - let line = text.lines().nth(1).expect("a weights line"); + let line = text.lines().nth(2).expect("a weights line"); let mut words = line.split(' '); assert_eq!(words.next(), Some("weights")); assert_eq!(words.next(), Some(SLOTS.to_string().as_str())); @@ -1069,7 +1137,7 @@ mod tests { }], }; let text = report.to_string(); - let row = text.lines().nth(2).expect("a row"); + let row = text.lines().nth(3).expect("a row"); let words: Vec<&str> = row.split(' ').collect(); // the fen is the last six fields let (head, last_six) = words.split_at(words.len() - 6); @@ -1112,19 +1180,24 @@ mod tests { slot ); } - for slot in MATERIAL_SLOT..MOBILITY_SLOT { + for slot in MATERIAL_SLOT..TERM_SLOT { assert!(is_material(slot), "the material slot {} is", slot); } - for slot in MOBILITY_SLOT..SLOTS { + for slot in TERM_SLOT..SLOTS { assert!( !is_material(slot), "the leaf term slot {} is inside the divide", slot ); } - assert_eq!(SHELTER_SLOT, MOBILITY_SLOT + 2 * MOBILITY_SLOTS); - assert_eq!(PAWN_SLOT, SHELTER_SLOT + 2 * SHELTER_SLOTS); - assert_eq!(SLOTS, PAWN_SLOT + 2 * PAWN_SLOTS); + // and the blocks after the material are each term's width twice over, + // one after the other, which is what the layout line states + let mut slot = TERM_SLOT; + for named in eval::TERMS { + assert_eq!(term(named.name), (slot, named.width), "{}", named.name); + slot += 2 * named.width; + } + assert_eq!(slot, SLOTS); } /// Every shelter count writes both ends of the taper too, and the counts @@ -1181,14 +1254,15 @@ mod tests { // f4, g4 and h4 (6, 3, "the storm three ranks ahead"), ] { + let (start, width) = term("shelter"); assert_eq!( - coefficient(SHELTER_SLOT + index), + coefficient(start + index), count * terms.phase, "{} midgame", why ); assert_eq!( - coefficient(SHELTER_SLOT + SHELTER_SLOTS + index), + coefficient(start + width + index), count * (TOTAL_PHASE - terms.phase), "{} endgame", why @@ -1250,14 +1324,15 @@ mod tests { // white's a7, b3 and d5 each have one behind them (7, 3, "the doubled pawns"), ] { + let (start, width) = term("pawn_structure"); assert_eq!( - coefficient(PAWN_SLOT + index), + coefficient(start + index), count * terms.phase, "{} midgame", why ); assert_eq!( - coefficient(PAWN_SLOT + PAWN_SLOTS + index), + coefficient(start + width + index), count * (TOTAL_PHASE - terms.phase), "{} endgame", why @@ -1277,7 +1352,7 @@ mod tests { terms .coefficients .iter() - .any(|(slot, _)| usize::from(*slot) >= PAWN_SLOT), + .any(|(slot, _)| usize::from(*slot) >= term("pawn_structure").0), "no pawn structure coefficient here, so this test says nothing about one" ); assert_eq!(terms, Terms::of(&black)); @@ -1300,7 +1375,7 @@ mod tests { terms .coefficients .iter() - .any(|(slot, _)| usize::from(*slot) >= SHELTER_SLOT), + .any(|(slot, _)| usize::from(*slot) >= term("shelter").0), "no shelter coefficient here, so this test says nothing about one" ); assert_eq!(terms, Terms::of(&black)); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 70e710db..ac25565f 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -41,6 +41,9 @@ ones. All three are fitted to the engine's own archived games. derived state in sync. Debug builds recompute the derived state from scratch after every move and assert it matches, so a bug in an incremental update fails tests instead of misevaluating quietly. + It names no evaluation term: each one reads the boards it needs through + `pub(crate)` accessors and keeps its own counts and masks beside its + weights. Move generation also lives here. It is pseudo-legal: moves are generated without checking whether they leave the king in check, and `make_move` rejects the ones that do. When already in check the list is first @@ -100,18 +103,31 @@ ones. All three are fitted to the engine's own archived games. by about 45 elo, so the error is carried knowingly and the bench prints the taint counters on every run. A `reference` configuration keeps the cautious search as a baseline for classifying future changes. -- **eval.rs**: What a position scores. The board hosts an accumulator and - tells it about every piece placed, removed and moved, so material and the - piece square score are carried rather than counted; anything too dear to - keep in step is computed at the leaf instead. The king shelter is computed - at the leaf and then remembered under the pawns and the two king squares it - is a function of, in a small table the searcher owns, because a king move - rewrites a whole side's reading and there is nothing there to keep in step. - The pawn structure is remembered the same way in a table of its own, under - the pawn key alone: it reads neither king, so what misses is a pawn move - and the capture of a pawn, where the shelter's key misses on a king move - as well. Material that cannot mate is answered with a hard zero before any - of that, which is the one place the score is not a sum over the weights. +- **eval/**: What a position scores, a file per leaf term and one for what + they share. + - **mod.rs**: The material values, the phase weights the taper is read at, + the accumulator, and the sum the search asks for. The board hosts the + accumulator and tells it about every piece placed, removed and moved, so + material and the piece square score are carried rather than counted; + anything too dear to keep in step is computed at the leaf instead. + Material that cannot mate is answered with a hard zero before any of + that, which is the one place the score is not a sum over the weights. + Also here is `TERMS`, a descriptor per leaf term (its name, how many + counts it is measured in, its weights and its counts), which is what the + tuner lays its slot vector out from. + - **mobility.rs**: How many squares each side's pieces cover. Read off the + board at every leaf and not remembered: a piece that moves changes what + every slider looking through its square sees. + - **shelter.rs**: What stands between each king and the board, its own + pawns and the enemy pawns coming for it. Computed at the leaf and then + remembered under the pawns and the two king squares it is a function of, + in a small table the searcher owns, because a king move rewrites a whole + side's reading and there is nothing there to keep in step. + - **pawn_structure.rs**: Each side's passed pawns by rank, its isolated + pawns and its doubled ones. Remembered the same way in a table of its + own, under the pawn key alone: it reads neither king, so what misses is a + pawn move and the capture of a pawn, where the shelter's key misses on a + king move as well. - **psqt.rs**: The piece square tables. Every piece has a second table for the endgame; both phases are packed into one integer so the taper costs one multiply. @@ -146,7 +162,10 @@ ones. All three are fitted to the engine's own archived games. which is asserted on every row printed as well as over three suites in a test. A position drawn by material is turned away and counted in the header rather than fitted, because its score does not read the weights at all. - Driven by the `terms` argument, and read by `scripts/tune.py`. + Where the leaf terms stand in the vector comes from `eval::TERMS` rather + than from constants here, and the run prints that layout on a line of its + own so that what reads the rows keeps no copy of it. Driven by the `terms` + argument, and read by `scripts/tune.py`. - **tactics.rs**: 300 tactical positions with a pinned pass count, gated in CI. - **strategy.rs**: 1500 quiet positions, each move graded out of a diff --git a/docs/INSTRUMENTS.md b/docs/INSTRUMENTS.md index aeeeaac3..1a94da91 100644 --- a/docs/INSTRUMENTS.md +++ b/docs/INSTRUMENTS.md @@ -408,7 +408,25 @@ those changes, and any vector fitted against them, are refused rather than read: every slot they name exists in the layout that replaced them, so reading them would put the numbers on the wrong weights. -The line after the header is `weights 812 ...`, the vector itself as +The line after the header states that layout, so that what reads these rows +holds no copy of it: + +``` +layout midgame 384 endgame 384 material 6 mobility 4 shelter 7 pawn_structure 8 +``` + +The first three are runs of slots. The names after them are the leaf terms, in +the order the vector holds them, and a term's number is the counts it is +measured in per side and per half of the taper, so it takes twice that in +slots, its midgame half first. The line is spelled off the same term list the +engine lays its slots out from, so a term added there is named here without a +second edit. `scripts/tune.py` reads its slots from this line and refuses a run +naming a term it has no bounds for, since a term nobody has priced cannot be +screened against the sixteen bits each half of a packed pair has to stay +inside. A run that prints no layout line at all was printed by an engine older +than the line, and is refused too. + +The line after that is `weights 812 ...`, the vector itself as the live tables hold it, so that nothing reading these rows transcribes psqt.rs. A transcription is the same failure as a reimplemented evaluation and quieter: a table copied out and left behind fits weights against a position it diff --git a/scripts/tests/test_tune.py b/scripts/tests/test_tune.py index 712fff09..d7c5157b 100644 --- a/scripts/tests/test_tune.py +++ b/scripts/tests/test_tune.py @@ -26,6 +26,16 @@ import pytest import tune +# The layout line an `arche terms` run prints today, which is what the rows +# and the vectors below are built in. Written out here rather than asked of +# tune.py, because what these tests are about is a file that reads its layout +# off the run instead of holding one: a fixture that took the layout from +# tune.py would agree with it whatever either of them said. +LAYOUT_LINE = ( + "layout midgame 384 endgame 384 material 6 mobility 4 shelter 7 pawn_structure 8" +) +LAYOUT = tune.Layout.of(LAYOUT_LINE) + # A pawn, a knight, a bishop, a rook, a queen and a king, which is what the # material slots hold today. MATERIAL = [100, 310, 320, 500, 900, 10000] @@ -54,8 +64,8 @@ def labels_of(mapping): def weights(entries=None): """A weight vector in the engine's layout, tables at nothing unless the caller names an entry.""" - vector = [0] * tune.SLOTS - vector[tune.MATERIAL_SLOT : tune.MOBILITY_SLOT] = MATERIAL + vector = [0] * LAYOUT.slots + vector[LAYOUT.block("material")] = MATERIAL for slot, value in (entries or {}).items(): vector[slot] = value return vector @@ -67,7 +77,7 @@ def row( """One row in the shape `arche terms` prints it, with the evaluation it states worked out from the weights the same way the engine does. The fen is six fields, which is what the row's reader counts back from.""" - evaluation = tune.reconstruct(coefficients, vector) + evaluation = tune.reconstruct(coefficients, vector, LAYOUT) terms = " ".join(f"{slot}:{coefficient}" for slot, coefficient in coefficients) return f"{identifier} {evaluation} {phase} {len(coefficients)} {terms} {fen}" @@ -76,8 +86,10 @@ def extraction(rows, vector): header = ( f"terms positions {len(rows)} in_check 0 unsettled 0 drawn 0 kept {len(rows)}" ) - line = "weights {} {}".format(tune.SLOTS, " ".join(str(value) for value in vector)) - return [header, line, *rows] + line = "weights {} {}".format( + LAYOUT.slots, " ".join(str(value) for value in vector) + ) + return [header, LAYOUT_LINE, line, *rows] def test_the_divide_truncates_toward_zero(): @@ -98,10 +110,12 @@ def test_material_is_added_outside_the_divide(): vector = weights({0: -1}) # a knight and a pawn's worth of material, and a piece square numerator of # minus one, which does not divide by the taper - coefficients = [(0, 1), (tune.MATERIAL_SLOT, 1)] - assert tune.reconstruct(coefficients, vector) == 100 + tune.trunc_div(-1, 24) + coefficients = [(0, 1), (LAYOUT.start["material"], 1)] + assert tune.reconstruct(coefficients, vector, LAYOUT) == 100 + tune.trunc_div( + -1, 24 + ) folded = (100 * 24 - 1) // 24 - assert folded != tune.reconstruct(coefficients, vector) + assert folded != tune.reconstruct(coefficients, vector, LAYOUT) def test_a_row_that_does_not_rebuild_stops_the_run(): @@ -109,8 +123,8 @@ def test_a_row_that_does_not_rebuild_stops_the_run(): row. A row that does not rebuild means they have parted company, so it raises rather than being dropped and fitted around.""" vector = weights({0: 30}) - good = row("a", [(0, 24), (tune.MATERIAL_SLOT, 1)], vector) - _, rows = tune.parse_terms(extraction([good], vector)) + good = row("a", [(0, 24), (LAYOUT.start["material"], 1)], vector) + _, _, rows = tune.parse_terms(extraction([good], vector)) assert len(rows) == 1 words = good.split(" ") words[1] = str(int(words[1]) + 1) @@ -130,13 +144,13 @@ def test_a_row_reads_from_the_right_and_an_id_can_hold_spaces(): walk over.""" vector = weights({5: 12, 400: -7}) fen = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1" - coefficients = [(5, 24), (400, -3), (tune.MATERIAL_SLOT + 4, -1)] + coefficients = [(5, 24), (400, -3), (LAYOUT.start["material"] + 4, -1)] lines = [ row("ruy lopez", coefficients, vector, 18, fen), row(fen, coefficients, vector, 18, fen), row("start", [], vector, 24, fen), ] - _, rows = tune.parse_terms(extraction(lines, vector)) + _, _, rows = tune.parse_terms(extraction(lines, vector)) assert [parsed.id for parsed in rows] == ["ruy lopez", fen, "start"] assert [parsed.fen for parsed in rows] == [fen, fen, fen] assert rows[0].phase == 18 @@ -145,12 +159,17 @@ def test_a_row_reads_from_the_right_and_an_id_can_hold_spaces(): def test_a_weights_line_of_the_wrong_length_is_refused(): - """The layout is a contract between two files, and a vector of another - length means one of them has changed and the other has not.""" + """The run states its layout and then its vector, and a vector of another + length means the two do not describe the same engine.""" with pytest.raises(ValueError, match="weights line"): - tune.parse_terms(["weights 3 1 2 3"]) + tune.parse_terms([LAYOUT_LINE, "weights 3 1 2 3"]) with pytest.raises(ValueError, match="no weights line"): - tune.parse_terms(["terms positions 0 in_check 0 unsettled 0 drawn 0 kept 0"]) + tune.parse_terms( + [ + "terms positions 0 in_check 0 unsettled 0 drawn 0 kept 0", + LAYOUT_LINE, + ] + ) def test_a_header_without_the_drawn_count_is_refused(): @@ -164,8 +183,8 @@ def test_a_header_without_the_drawn_count_is_refused(): right only for as long as nobody edited either copy. So the count is what is read, and a header without it refuses the whole run.""" vector = weights() - lines = extraction([row("a", [(tune.MATERIAL_SLOT, 1)], vector)], vector) - _, rows = tune.parse_terms(lines) + lines = extraction([row("a", [(LAYOUT.start["material"], 1)], vector)], vector) + _, _, rows = tune.parse_terms(lines) assert len(rows) == 1 for header in [ "terms positions 1 in_check 0 unsettled 0 kept 1", @@ -175,36 +194,66 @@ def test_a_header_without_the_drawn_count_is_refused(): tune.parse_terms([header, *lines[1:]]) -@pytest.mark.parametrize( - ("count", "message"), - [ - (518, "given an endgame table"), - (774, "before mobility"), - (782, "before the king's shelter"), - (790, "before the pawn storm"), - (796, "before the pawn structure"), - ], -) -def test_a_vector_of_an_earlier_layout_is_refused(tmp_path, count, message): - """518, 774, 782, 790 and 796 are the wrong lengths that would otherwise - read as right ones: every slot any of them names exists in the layout that - replaced it, so their numbers would land on the wrong weights rather than - failing to parse. Both doors a vector comes through say what changed.""" - assert tune.SHARED_TABLE_SLOTS == 518 - assert tune.NO_MOBILITY_SLOTS == 774 - assert tune.NO_SHELTER_SLOTS == 782 - assert tune.NO_STORM_SLOTS == 790 - assert tune.NO_PAWN_SLOTS == 796 - assert tune.SLOTS == 812 +@pytest.mark.parametrize("count", [518, 774, 782, 790, 796]) +def test_a_vector_of_an_earlier_layout_is_refused(tmp_path, count): + """518, 774, 782, 790 and 796 are the lengths the vector had before a + knight, a bishop, a rook and a queen were given an endgame table, before + mobility, before the king's shelter, before the pawn storm joined it and + before the pawn structure. Every slot any of them names exists in the + layout that replaced it, so their numbers would land on the wrong weights + rather than failing to parse. Both doors a vector comes through refuse + them.""" + assert LAYOUT.slots == 812 old = [0] * count - with pytest.raises(ValueError, match=message): + with pytest.raises(ValueError, match=f"of {count}, expected 812"): tune.parse_terms( - ["weights {} {}".format(len(old), " ".join(str(w) for w in old))] + [ + LAYOUT_LINE, + "weights {} {}".format(len(old), " ".join(str(w) for w in old)), + ] ) written = tmp_path / "fitted.json" written.write_text(json.dumps(old), encoding="utf-8") - with pytest.raises(ValueError, match=message): - tune.read_weights(written) + with pytest.raises(ValueError, match=f"of {count}, expected 812"): + tune.read_weights(written, LAYOUT) + + +def test_a_header_without_a_layout_line_is_refused(): + """The layout line is where this file learns its slots, so a run that + prints none leaves it nothing to read the numbers with. + + Before the line the layout was written out here as well as in the engine, + and the two were edited together by hand. An extraction from an engine + older than the line is exactly the case that copy was wrong for: it parses + and every slot it names exists, so a layout assumed for it would be wrong + with nothing saying so. The run is refused instead, whether the line is + missing or comes after the vector it describes.""" + vector = weights() + lines = extraction([row("a", [(LAYOUT.start["material"], 1)], vector)], vector) + assert lines[1] == LAYOUT_LINE + with pytest.raises(ValueError, match="states no layout"): + tune.parse_terms([lines[0], *lines[2:]]) + with pytest.raises(ValueError, match="states no layout"): + tune.parse_terms([lines[0], lines[2], lines[1], *lines[3:]]) + + +def test_a_term_the_layout_names_and_nothing_prices_is_refused(): + """A term this file has no bound for cannot be screened against the + packed halves, so a run that names one is refused rather than fitted. + + A new term is priced here before it is fitted, which is one line and a + reason for it. The alternative is a fit that runs, prints a boardful that + leaves the new weights out of the figure, and says nothing about the one + thing nobody has checked yet. The same goes for a term whose width has + moved: the bound is per count, so a count added arrives with a width this + file does not cover.""" + with pytest.raises(ValueError, match="no bounds for"): + tune.Layout.of(LAYOUT_LINE + " king_tropism 4") + with pytest.raises(ValueError, match="different number of counts"): + tune.Layout.of(LAYOUT_LINE.replace("shelter 7", "shelter 9")) + # and the terms it does price are priced count by count + for name in LAYOUT.terms: + assert len(tune.BOUNDS[name]) == LAYOUT.widths[name] def test_a_row_whose_id_opens_with_the_header_word_is_kept(): @@ -215,16 +264,28 @@ def test_a_row_whose_id_opens_with_the_header_word_is_kept(): it.""" vector = weights() lines = extraction( - [row("terms of the endgame", [(tune.MATERIAL_SLOT, 1)], vector)], vector + [row("terms of the endgame", [(LAYOUT.start["material"], 1)], vector)], vector ) - _, rows = tune.parse_terms(lines) + _, _, rows = tune.parse_terms(lines) assert [parsed.id for parsed in rows] == ["terms of the endgame"] # and both shapes of the header are still skipped rather than read as rows header = "terms epd corpus.epd positions 1 in_check 0 unsettled 0 drawn 0 kept 1" - _, rows = tune.parse_terms([header, *lines]) + _, _, rows = tune.parse_terms([header, *lines]) assert [parsed.id for parsed in rows] == ["terms of the endgame"] +def test_a_row_whose_id_opens_with_the_layout_word_is_kept(): + """The layout line is the second word the reader claims, and it is claimed + with the space after it rather than on the six letters. An id that opens + with those letters and runs on is a row, so it is read as one.""" + vector = weights() + lines = extraction( + [row("layouts of the endgame", [(LAYOUT.start["material"], 1)], vector)], vector + ) + _, _, rows = tune.parse_terms(lines) + assert [parsed.id for parsed in rows] == ["layouts of the endgame"] + + def test_a_corpus_line_is_read_the_way_the_engine_reads_epd(): """Four fields and then operations, so the id is not looked for among the words of the position.""" @@ -277,8 +338,8 @@ def test_the_groups_are_assigned_from_the_pair_and_not_the_game(): "g00001p020": tune.Label(1.0, 1, key(1, TRAIN), key(9, CALIBRATION)), "g00002p020": tune.Label(0.0, 1, key(2, CALIBRATION), key(8, TRAIN)), } - shipped, parsed = tune.parse_terms(extraction(rows, vector)) - corpus = tune.Corpus(shipped, parsed, labels) + _, shipped, parsed = tune.parse_terms(extraction(rows, vector)) + corpus = tune.Corpus(LAYOUT, shipped, parsed, labels) assert [r.id for r in corpus.rows] == ["g00002p020"] assert corpus.sealed.positions == 1 assert corpus.sealed.pairs == 1 @@ -292,8 +353,8 @@ def test_a_corpus_that_repeats_a_position_across_games_is_refused(): vector = weights({0: 7}) fen = "4k3/8/8/8/8/8/8/4K3 w - - 0 1" rows = [ - row("g00001p020", [(0, 24), (tune.MATERIAL_SLOT, 1)], vector, 24, fen), - row("g00002p031", [(0, 24), (tune.MATERIAL_SLOT, 1)], vector, 24, fen), + row("g00001p020", [(0, 24), (LAYOUT.start["material"], 1)], vector, 24, fen), + row("g00002p031", [(0, 24), (LAYOUT.start["material"], 1)], vector, 24, fen), ] labels = { "g00001p020": (1.0, 1, key(1, TRAIN)), @@ -312,7 +373,9 @@ def games_and_labels(vector, games, plies=4): for ply in range(plies): name = f"g{index:05d}p{ply:03d}" fen = f"4k3/8/8/8/8/{index}p/{ply}p/4K3 w - - 0 1" - rows.append(row(name, [(0, 24), (tune.MATERIAL_SLOT, 1)], vector, 24, fen)) + rows.append( + row(name, [(0, 24), (LAYOUT.start["material"], 1)], vector, 24, fen) + ) labels[name] = (float(index % 2), 1, key(index, slice_)) return rows, labels @@ -350,8 +413,8 @@ def test_a_position_is_bucketed_by_what_is_left_on_the_board(): def corpus_of(rows, vector, labels): - _, parsed = tune.parse_terms(extraction(rows, vector)) - return tune.Corpus(vector, parsed, labels_of(labels)) + _, _, parsed = tune.parse_terms(extraction(rows, vector)) + return tune.Corpus(LAYOUT, vector, parsed, labels_of(labels)) def test_the_integer_score_is_the_evaluation_the_engine_gave(): @@ -361,11 +424,15 @@ def test_the_integer_score_is_the_evaluation_the_engine_gave(): vector = weights({0: -30, 64: 17, 400: 5}) rows = [ row( - "a", [(0, 7), (64, -13), (tune.MATERIAL_SLOT, 1)], vector, 7, "a w - - 0 1" + "a", + [(0, 7), (64, -13), (LAYOUT.start["material"], 1)], + vector, + 7, + "a w - - 0 1", ), row( "b", - [(0, -11), (400, 3), (tune.MATERIAL_SLOT + 3, -2)], + [(0, -11), (400, 3), (LAYOUT.start["material"] + 3, -2)], vector, 11, "b w - - 0 1", @@ -390,8 +457,8 @@ def test_the_occurrence_count_weights_the_loss(): pulls the loss towards its own result.""" vector = weights({0: 20}) rows = [ - row("a", [(0, 24), (tune.MATERIAL_SLOT, 1)], vector, 24, "a w - - 0 1"), - row("b", [(0, 24), (tune.MATERIAL_SLOT, 1)], vector, 24, "b w - - 0 1"), + row("a", [(0, 24), (LAYOUT.start["material"], 1)], vector, 24, "a w - - 0 1"), + row("b", [(0, 24), (LAYOUT.start["material"], 1)], vector, 24, "b w - - 0 1"), ] # two positions scored alike and labelled oppositely, so the only thing a # weight can move is which of the two the loss listens to @@ -417,8 +484,8 @@ def test_a_slots_support_is_how_many_rows_it_appears_in(): """A weight the corpus barely constrains says so before it ships.""" vector = weights({0: 5}) rows = [ - row("a", [(0, 24), (tune.MATERIAL_SLOT, 1)], vector, 24, "a w - - 0 1"), - row("b", [(tune.MATERIAL_SLOT, 1)], vector, 24, "b w - - 0 1"), + row("a", [(0, 24), (LAYOUT.start["material"], 1)], vector, 24, "a w - - 0 1"), + row("b", [(LAYOUT.start["material"], 1)], vector, 24, "b w - - 0 1"), ] corpus = corpus_of( rows, @@ -427,7 +494,7 @@ def test_a_slots_support_is_how_many_rows_it_appears_in(): ) support = corpus.support() assert support[0] == 1 - assert support[tune.MATERIAL_SLOT] == 2 + assert support[LAYOUT.start["material"]] == 2 assert support[1] == 0 @@ -496,13 +563,13 @@ def test_the_optimiser_finds_the_bottom_of_a_bowl(): longer than one, so a search that only backtracked would stall, and the bowl below is scaled to say that.""" - centre = np.arange(tune.SLOTS, dtype=float) + centre = np.arange(LAYOUT.slots, dtype=float) def objective(x): slack = (x - centre) * 1e-4 return float(slack @ slack), 2e-8 * (x - centre) - found, value = tune.lbfgs(objective, np.zeros(tune.SLOTS)) + found, value = tune.lbfgs(objective, np.zeros(LAYOUT.slots)) assert value < 1e-12 assert np.max(np.abs(found - centre)) < 1e-3 @@ -511,10 +578,10 @@ def test_a_vector_the_engine_could_not_carry_is_refused(): """Each half of a packed pair is an `i16` and a boardful of them is summed into one, so a fit that grew the tables past that is no candidate whatever it scores.""" - inside, worst = tune.bounds_hold(np.array(weights())) + inside, worst = tune.bounds_hold(np.array(weights()), LAYOUT) assert inside and worst == 0 - huge = np.array(weights({slot: 400 for slot in range(tune.MATERIAL_SLOT)})) - inside, worst = tune.bounds_hold(huge) + huge = np.array(weights({slot: 400 for slot in range(LAYOUT.start["material"])})) + inside, worst = tune.bounds_hold(huge, LAYOUT) assert not inside assert worst == 2 * 64 * 400 @@ -524,14 +591,17 @@ def test_the_mobility_weights_are_priced_too(): it would be a vector priced at 774 of its 790 slots. The range stops at the shelter block, which the test below prices on its own.""" one_each = weights( - {slot: 1 for slot in range(tune.MOBILITY_SLOT, tune.SHELTER_SLOT)} + {slot: 1 for slot in range(LAYOUT.start["mobility"], LAYOUT.start["shelter"])} ) - _, worst = tune.bounds_hold(np.array(one_each)) - assert worst == 2 * int(tune.MAX_COUNT.sum()) + _, worst = tune.bounds_hold(np.array(one_each), LAYOUT) + assert worst == 2 * int(np.array(tune.BOUNDS["mobility"]).sum()) huge = weights( - {slot: 5000 for slot in range(tune.MOBILITY_SLOT, tune.SHELTER_SLOT)} + { + slot: 5000 + for slot in range(LAYOUT.start["mobility"], LAYOUT.start["shelter"]) + } ) - assert not tune.bounds_hold(np.array(huge))[0] + assert not tune.bounds_hold(np.array(huge), LAYOUT)[0] def test_the_shelter_weights_are_priced_too(): @@ -539,17 +609,25 @@ def test_the_shelter_weights_are_priced_too(): colours, at the larger of the two halves, which here is the endgame one at ten a count.""" vector = weights( - {tune.SHELTER_SLOT + index: 5 for index in range(tune.SHELTER_SLOTS)} + { + LAYOUT.start["shelter"] + index: 5 + for index in range(LAYOUT.widths["shelter"]) + } | { - tune.SHELTER_SLOT + tune.SHELTER_SLOTS + index: 10 - for index in range(tune.SHELTER_SLOTS) + LAYOUT.start["shelter"] + LAYOUT.widths["shelter"] + index: 10 + for index in range(LAYOUT.widths["shelter"]) } ) - inside, worst = tune.bounds_hold(np.array(vector)) + inside, worst = tune.bounds_hold(np.array(vector), LAYOUT) assert inside - assert worst == 2 * tune.MAX_SHELTER * tune.SHELTER_SLOTS * 10 - huge = weights({slot: 5000 for slot in range(tune.SHELTER_SLOT, tune.PAWN_SLOT)}) - assert not tune.bounds_hold(np.array(huge))[0] + assert worst == 2 * tune.BOUNDS["shelter"][0] * LAYOUT.widths["shelter"] * 10 + huge = weights( + { + slot: 5000 + for slot in range(LAYOUT.start["shelter"], LAYOUT.start["pawn_structure"]) + } + ) + assert not tune.bounds_hold(np.array(huge), LAYOUT)[0] def test_the_pawn_structure_weights_are_priced_too(): @@ -557,17 +635,25 @@ def test_the_pawn_structure_weights_are_priced_too(): at the larger of the two halves, which here is the endgame one at four a count.""" vector = weights( - {tune.PAWN_SLOT + index: 3 for index in range(tune.PAWN_SLOTS)} + { + LAYOUT.start["pawn_structure"] + index: 3 + for index in range(LAYOUT.widths["pawn_structure"]) + } | { - tune.PAWN_SLOT + tune.PAWN_SLOTS + index: 4 - for index in range(tune.PAWN_SLOTS) + LAYOUT.start["pawn_structure"] + LAYOUT.widths["pawn_structure"] + index: 4 + for index in range(LAYOUT.widths["pawn_structure"]) } ) - inside, worst = tune.bounds_hold(np.array(vector)) + inside, worst = tune.bounds_hold(np.array(vector), LAYOUT) assert inside - assert worst == 2 * tune.MAX_PAWNS * tune.PAWN_SLOTS * 4 - huge = weights({slot: 5000 for slot in range(tune.PAWN_SLOT, tune.SLOTS)}) - assert not tune.bounds_hold(np.array(huge))[0] + assert ( + worst + == 2 * tune.BOUNDS["pawn_structure"][0] * LAYOUT.widths["pawn_structure"] * 4 + ) + huge = weights( + {slot: 5000 for slot in range(LAYOUT.start["pawn_structure"], LAYOUT.slots)} + ) + assert not tune.bounds_hold(np.array(huge), LAYOUT)[0] def test_the_material_block_is_the_only_thing_outside_the_divide(): @@ -583,16 +669,17 @@ def test_the_material_block_is_the_only_thing_outside_the_divide(): still ship at zero and theirs would not. The weights here are the fixture's for that reason, so the two answers differ whatever a fit holds.""" - material = [slot for slot in range(tune.SLOTS) if tune.is_material(slot)] - assert material == list(range(tune.MATERIAL_SLOT, tune.MOBILITY_SLOT)) + material = [slot for slot in range(LAYOUT.slots) if LAYOUT.is_material(slot)] + assert material == list(range(LAYOUT.start["material"], LAYOUT.start["mobility"])) assert not any( - tune.is_material(slot) for slot in range(tune.MOBILITY_SLOT, tune.SLOTS) + LAYOUT.is_material(slot) + for slot in range(LAYOUT.start["mobility"], LAYOUT.slots) ) - vector = weights({tune.SHELTER_SLOT: 30}) - vector[tune.MATERIAL_SLOT] = 100 + vector = weights({LAYOUT.start["shelter"]: 30}) + vector[LAYOUT.start["material"]] = 100 # a pawn outside the divide, and a shelter count of a full phase inside it - coefficients = [(tune.SHELTER_SLOT, 24), (tune.MATERIAL_SLOT, 1)] - assert tune.reconstruct(coefficients, vector) == 130 + coefficients = [(LAYOUT.start["shelter"], 24), (LAYOUT.start["material"], 1)] + assert tune.reconstruct(coefficients, vector, LAYOUT) == 130 def test_a_fit_is_free_to_move_the_leaf_terms_weights(): @@ -600,11 +687,11 @@ def test_a_fit_is_free_to_move_the_leaf_terms_weights(): material block's end instead, which is what it was before mobility, the twenty two weights of the two leaf terms would sit at zero through the fit and the arm would report a null result with nothing saying why.""" - frozen = tune.frozen_slots(False) - assert frozen[tune.MATERIAL_SLOT : tune.MOBILITY_SLOT].all() - assert not frozen[tune.MOBILITY_SLOT :].any() - assert not frozen[: tune.MATERIAL_SLOT].any() - assert not tune.frozen_slots(True).any() + frozen = tune.frozen_slots(LAYOUT, False) + assert frozen[LAYOUT.start["material"] : LAYOUT.start["mobility"]].all() + assert not frozen[LAYOUT.start["mobility"] :].any() + assert not frozen[: LAYOUT.start["material"]].any() + assert not tune.frozen_slots(LAYOUT, True).any() def test_a_term_is_fitted_with_every_earlier_term_held(): @@ -613,14 +700,14 @@ def test_a_term_is_fitted_with_every_earlier_term_held(): shelter fit that passed only that would have refitted mobility beside the shelter and called the pair king safety. The same again one term on: two holds leave the shelter free, and a pawn structure fit wants three.""" - tables = tune.frozen_slots(False, True) - assert not tables[tune.MOBILITY_SLOT : tune.SHELTER_SLOT].any() - both = tune.frozen_slots(False, True, True) - assert both[: tune.SHELTER_SLOT].all() - assert not both[tune.SHELTER_SLOT : tune.PAWN_SLOT].any() - three = tune.frozen_slots(False, True, True, True) - assert three[: tune.PAWN_SLOT].all() - assert not three[tune.PAWN_SLOT :].any() + tables = tune.frozen_slots(LAYOUT, False, True) + assert not tables[LAYOUT.start["mobility"] : LAYOUT.start["shelter"]].any() + both = tune.frozen_slots(LAYOUT, False, True, True) + assert both[: LAYOUT.start["shelter"]].all() + assert not both[LAYOUT.start["shelter"] : LAYOUT.start["pawn_structure"]].any() + three = tune.frozen_slots(LAYOUT, False, True, True, True) + assert three[: LAYOUT.start["pawn_structure"]].all() + assert not three[LAYOUT.start["pawn_structure"] :].any() def test_a_refit_holds_the_terms_above_it_as_well_as_the_ones_below(): @@ -630,10 +717,10 @@ def test_a_refit_holds_the_terms_above_it_as_well_as_the_ones_below(): the eight weights are the only thing that moves and a match reads them alone. Without the last hold the sixteen pawn weights move too, which is the confound `1b0862a` found the first time a hold was missing.""" - refit = tune.frozen_slots(False, True, False, True, True) - assert refit[: tune.MOBILITY_SLOT].all() - assert not refit[tune.MOBILITY_SLOT : tune.SHELTER_SLOT].any() - assert refit[tune.SHELTER_SLOT :].all() + refit = tune.frozen_slots(LAYOUT, False, True, False, True, True) + assert refit[: LAYOUT.start["mobility"]].all() + assert not refit[LAYOUT.start["mobility"] : LAYOUT.start["shelter"]].any() + assert refit[LAYOUT.start["shelter"] :].all() def test_quantizing_rounds_to_nearest(): @@ -658,7 +745,7 @@ def sample(vector, count=30, plies=4): rows.append( row( name, - [(0, 24), (tune.MATERIAL_SLOT, 1 if index % 2 else -1)], + [(0, 24), (LAYOUT.start["material"], 1 if index % 2 else -1)], vector, 24, fen, @@ -766,8 +853,8 @@ def test_a_fit_holds_the_material_values_unless_it_is_told_not_to(tmp_path, caps == 0 ) fitted = json.loads(out.read_text(encoding="utf-8")) - assert fitted[tune.MATERIAL_SLOT : tune.MOBILITY_SLOT] == MATERIAL - assert len(fitted) == tune.SLOTS + assert fitted[LAYOUT.start["material"] : LAYOUT.start["mobility"]] == MATERIAL + assert len(fitted) == LAYOUT.slots def test_a_ridge_is_chosen_on_the_selection_games(tmp_path, capsys): @@ -986,7 +1073,9 @@ def test_final_scores_the_sealed_rows_and_only_those(tmp_path, capsys): scores = np.array( [ tune.reconstruct( - [(0, 24), (tune.MATERIAL_SLOT, 1 if result == 1.0 else -1)], vector + [(0, 24), (LAYOUT.start["material"], 1 if result == 1.0 else -1)], + vector, + LAYOUT, ) for _, result in sealed ], @@ -1229,14 +1318,14 @@ def test_a_named_seal_holds_out_the_games_it_names_and_no_others(): set.""" vector = weights() rows, raw = sample(vector) - _, parsed = tune.parse_terms(extraction(rows, vector)) + _, _, parsed = tune.parse_terms(extraction(rows, vector)) labels = labels_of(raw) named = {min(label.pair for label in labels.values())} - corpus = tune.Corpus(vector, parsed, labels, named) + corpus = tune.Corpus(LAYOUT, vector, parsed, labels, named) assert corpus.sealed.checksum() == sealed_checksum(named) assert named.isdisjoint(corpus.pairs) assert set(corpus.groups) == {"train", "selection"} - drawn = tune.Corpus(vector, parsed, labels) + drawn = tune.Corpus(LAYOUT, vector, parsed, labels) assert drawn.sealed.checksum() != sealed_checksum(named) @@ -1246,9 +1335,9 @@ def test_a_sealed_pair_no_game_of_the_corpus_holds_is_refused(): That is worse than no reading at all, so it is refused.""" vector = weights() rows, raw = sample(vector) - _, parsed = tune.parse_terms(extraction(rows, vector)) + _, _, parsed = tune.parse_terms(extraction(rows, vector)) with pytest.raises(ValueError, match="in no game of this corpus"): - tune.Corpus(vector, parsed, labels_of(raw), {"f" * 64}) + tune.Corpus(LAYOUT, vector, parsed, labels_of(raw), {"f" * 64}) def test_a_sealed_pair_with_no_row_is_counted_rather_than_refused(): @@ -1267,7 +1356,7 @@ def test_a_sealed_pair_with_no_row_is_counted_rather_than_refused(): dropped = {name for name, label in labels.items() if label.pair == lonely} kept = [row for row in rows if row.split(" ", 1)[0] not in dropped] assert len(kept) < len(rows), "the fixture shares no pair" - _, parsed = tune.parse_terms(extraction(kept, vector)) - corpus = tune.Corpus(vector, parsed, labels, {lonely}) + _, _, parsed = tune.parse_terms(extraction(kept, vector)) + corpus = tune.Corpus(LAYOUT, vector, parsed, labels, {lonely}) assert corpus.sealed_without_rows == 1 assert corpus.sealed.positions == 0 diff --git a/scripts/tune.py b/scripts/tune.py index 486b9bbf..2bbef28e 100755 --- a/scripts/tune.py +++ b/scripts/tune.py @@ -72,6 +72,13 @@ That is checked on every row read, so a corpus this cannot rebuild stops the run rather than being fitted. +Nor does it know where the weights stand. The run prints a layout line naming +every block of the vector and how wide it is, and `Layout` reads its slots off +that, so an extraction from an engine whose terms this file has no bounds for +is refused by name rather than fitted on to the wrong weights. What stays here +is the price of each term: a term the run names and `BOUNDS` does not is one +nobody has screened against the packed halves yet. + Three details of that arithmetic are the ones a python reader gets wrong. The divide truncates toward zero where `//` floors, and on a negative numerator that does not divide evenly the two differ by a centipawn. The material is @@ -96,67 +103,121 @@ import numpy as np from groups import CALIBRATION, group_of, sealed_pairs -# The weight vector, as `arche-core/src/tune.rs` lays it out: 384 midgame table -# entries, then 384 endgame ones in the same order, then the six material -# values, then four midgame mobility weights and the same four at the endgame -# end, then the seven shelter weights the same way, then the eight pawn -# structure weights the same way again. A square's two weights are -# MIDGAME_SLOTS apart, a piece kind's two mobility weights MOBILITY_SLOTS -# apart, a shelter count's two SHELTER_SLOTS apart and a pawn count's two -# PAWN_SLOTS apart. -MIDGAME_SLOTS = 6 * 64 -ENDGAME_SLOTS = 6 * 64 -MATERIAL_SLOT = MIDGAME_SLOTS + ENDGAME_SLOTS -MOBILITY_SLOT = MATERIAL_SLOT + 6 -MOBILITY_SLOTS = 4 -SHELTER_SLOT = MOBILITY_SLOT + 2 * MOBILITY_SLOTS -SHELTER_SLOTS = 7 -PAWN_SLOT = SHELTER_SLOT + 2 * SHELTER_SLOTS -PAWN_SLOTS = 8 -SLOTS = PAWN_SLOT + 2 * PAWN_SLOTS - -# The layout before a knight, a bishop, a rook and a queen were given an -# endgame table of their own: the same 384 midgame entries, the pawn's endgame -# table and the king's, and the six material values. It is named so that a row -# or a fitted vector written against it is turned away by what it is rather -# than by its length alone. -SHARED_TABLE_SLOTS = 6 * 64 + 2 * 64 + 6 - -# The layout before mobility: both halves of the tables and the material, and -# nothing after them. Named for the same reason SHARED_TABLE_SLOTS is, since -# every slot it holds still exists here and holds the same weight. -NO_MOBILITY_SLOTS = MOBILITY_SLOT - -# The layout after mobility and before the king's shelter, and the layout -# before the shelter grew the pawn storm. Both are nearer mistakes than the two -# above: they are one term or half a term back rather than two, and every slot -# either holds still means here what it meant there. -NO_SHELTER_SLOTS = SHELTER_SLOT -NO_STORM_SLOTS = SHELTER_SLOT + 2 * 4 - -# The layout after the shelter and before the pawn structure, which is the -# most recent mistake of the five and so the likeliest: a row extracted or a -# vector fitted one term back, when every slot it holds still means here what -# it meant there. -NO_PAWN_SLOTS = PAWN_SLOT - -# The most one knight, one bishop, one rook and one queen can each cover, which -# is what a mobility weight is priced against in bounds_hold. A queen in the -# middle of an empty board is the twenty seven. -MAX_COUNT = np.array([8, 13, 14, 27]) - -# The most of each shelter count one side can show, which is what a shelter -# weight is priced against in bounds_hold. Three pawns on each of the five -# ranks counted, its own two and the storm's three, and three files, which the -# open and half open counts share rather than reach each. -MAX_SHELTER = 3 - -# The most of each pawn count one side can show, which is what a pawn -# structure weight is priced against in bounds_hold. Eight, since that is how -# many pawns a side has. Charging all eight counts at eight is far past any -# position, since eight pawns cannot fill sixty four counts between them, and -# the looseness is on the safe side. -MAX_PAWNS = 8 +# The three blocks of the vector that are not leaf terms, in the order the +# layout line prints them. Each is a run of slots and not a width per half of +# the taper: the midgame table entries, the endgame ones, then the material +# values. +FIXED_BLOCKS = ("midgame", "endgame", "material") + +# What one side can show of each of a term's counts, which is what a weight is +# priced against in bounds_hold. A term the layout names and this does not is +# refused rather than fitted: a new term has to be priced before a fit can say +# whether its weights stay inside the packed halves. +# +# Mobility is the most one knight, one bishop, one rook and one queen can each +# cover, the twenty seven being a queen in the middle of an empty board. The +# shelter is three pawns on each of the five ranks counted and three files, +# which the open and half open counts share rather than reach each. The pawn +# structure is eight, since that is how many pawns a side has; charging all +# eight counts at eight is far past any position, and the looseness is on the +# safe side. +# +# One entry per count rather than one per term, so a term that grows a count +# arrives here with a width this file does not cover and is refused too. +BOUNDS = { + "mobility": (8, 13, 14, 27), + "shelter": (3,) * 7, + "pawn_structure": (8,) * 8, +} + + +class Layout: + """Where every block of the weight vector stands, read off the layout line + the run printed rather than written out here. + + The engine lays its slots out from one list of terms and states the result + on that line, so this file learns the layout from the extraction it is + reading rather than keeping one of its own. A layout kept here is the + failure this seam is here to prevent. Such a run parses and every slot it + names exists, so nothing about it looks wrong while its coefficients are + priced against weights that stand somewhere else. + + A term's width is the counts it is measured in per side and per half of + the taper, so it takes twice that in slots, its midgame half first. The + three fixed blocks are stated as slots because neither half of the tables + is a half of anything. + """ + + def __init__(self, widths): + missing = [name for name in FIXED_BLOCKS if name not in widths] + if missing or list(widths)[: len(FIXED_BLOCKS)] != list(FIXED_BLOCKS): + raise ValueError( + "a layout naming {}, where this file expects {} and then the " + "terms".format(" ".join(widths), " ".join(FIXED_BLOCKS)) + ) + self.widths = dict(widths) + self.terms = [name for name in widths if name not in FIXED_BLOCKS] + unpriced = [name for name in self.terms if name not in BOUNDS] + if unpriced: + raise ValueError( + "the run names {}, which this file has no bounds for: a term " + "has to be priced here before a fit can say whether its " + "weights stay inside the packed halves".format(" ".join(unpriced)) + ) + mispriced = [name for name in self.terms if len(BOUNDS[name]) != widths[name]] + if mispriced: + raise ValueError( + "the run measures {} in a different number of counts than this " + "file prices it in, so the bounds are to redo".format( + " ".join(mispriced) + ) + ) + self.start = {} + slot = 0 + for name, width in widths.items(): + self.start[name] = slot + slot += width if name in FIXED_BLOCKS else 2 * width + self.slots = slot + + @classmethod + def of(cls, line): + """The layout a run's `layout` line states.""" + words = line.split()[1:] + if len(words) % 2: + raise ValueError(f"a layout line with a name and no width: {line}") + widths = {} + for name, width in zip(words[::2], words[1::2]): + if name in widths: + raise ValueError(f"a layout line naming {name} twice") + widths[name] = int(width) + return cls(widths) + + def is_material(self, slot): + """Whether a slot is one of the material values, which are the only + weights added outside the taper's divide. Everything else is inside + it, the tables and every leaf term alike, which is what + `Accumulator::score` does with them.""" + start = self.start["material"] + return start <= slot < start + self.widths["material"] + + def block(self, name): + """The slots a named block holds, as a slice.""" + width = self.widths[name] + size = width if name in FIXED_BLOCKS else 2 * width + return slice(self.start[name], self.start[name] + size) + + def check(self, count, what): + """Refuse a vector of any length but this run's, and say so. + + The layout is the run's rather than this file's now, so a length that + does not match is a vector from a different engine or a different + extraction. Which one it was is no longer guessable from the length, + and it does not need to be: the line names its terms, so a run this + file cannot fit has already been refused by name. + """ + if count != self.slots: + raise ValueError(f"{what} of {count}, expected {self.slots}") + # What the opening's pieces come to on the scale the taper is read at, which is # what the piece square half of a row divides by. @@ -196,6 +257,10 @@ # seam forbids, so it refuses the extraction instead. HEADER_COUNTS = ("positions", "in_check", "unsettled", "drawn", "kept") +# What the layout line opens with. An extraction that carries no such line is +# refused, and `no_layout` says why. +LAYOUT = "layout " + def check_header(line): """Refuse a header this engine did not print, and say what it counted.""" @@ -212,47 +277,19 @@ def check_header(line): ) -def check_layout(count, what): - """Refuse a vector of any length but this file's, and say what changed when - it is a length the layout had before. +def no_layout(why): + """Why an extraction with no layout line is refused rather than read. - The layout is a contract between this file and `arche-core/src/tune.rs`, - and the two are edited together. A vector of an earlier length is the wrong - length a bare count would not explain: it parses, every slot it names - exists here, and its numbers land on weights they were not fitted for. + The header says what the run turned away and the layout line says where + its weights stand. A run that prints the first and not the second was + printed by an engine older than the line, so this file would have to + assume a layout for it, which is the duplication the line was added to end. """ - if count == SLOTS: - return - if count == SHARED_TABLE_SLOTS: - raise ValueError( - f"{what} of {count}, which is the layout from before a knight, a " - f"bishop, a rook and a queen were given an endgame table. The " - f"vector is {SLOTS} now, so extract the rows again and refit" - ) - if count == NO_MOBILITY_SLOTS: - raise ValueError( - f"{what} of {count}, which is the layout from before mobility. The " - f"vector is {SLOTS} now, so extract the rows again and refit" - ) - if count == NO_SHELTER_SLOTS: - raise ValueError( - f"{what} of {count}, which is the layout from before the king's " - f"shelter was measured. The vector is {SLOTS} now, so extract the " - f"rows again and refit" - ) - if count == NO_STORM_SLOTS: - raise ValueError( - f"{what} of {count}, which is the layout from before the pawn " - f"storm joined the king's shelter. The vector is {SLOTS} now, so " - f"extract the rows again and refit" - ) - if count == NO_PAWN_SLOTS: - raise ValueError( - f"{what} of {count}, which is the layout from before the pawn " - f"structure was measured. The vector is {SLOTS} now, so extract " - f"the rows again and refit" - ) - raise ValueError(f"{what} of {count}, expected {SLOTS}") + return ( + f"a terms run whose header carries no layout line ({why}): it was printed " + "by an engine that states no layout, so where its weights stand " + "cannot be read here and the extraction is to redo" + ) def trunc_div(numerator, denominator): @@ -263,15 +300,7 @@ def trunc_div(numerator, denominator): return quotient if numerator >= 0 else -quotient -def is_material(slot): - """Whether a slot is one of the six material values, which are the only - weights added outside the taper's divide. Everything else is inside it, - the tables and both leaf terms alike, which is what `Accumulator::score` - does with them.""" - return MATERIAL_SLOT <= slot < MOBILITY_SLOT - - -def reconstruct(coefficients, weights): +def reconstruct(coefficients, weights, layout): """The evaluation a row states, folded back against the weights. The material is added outside the divide and not scaled into it. Folding it @@ -282,7 +311,7 @@ def reconstruct(coefficients, weights): numerator = 0 for slot, coefficient in coefficients: product = coefficient * weights[slot] - if is_material(slot): + if layout.is_material(slot): material += product else: numerator += product @@ -372,13 +401,14 @@ def __init__(self, identifier, evaluation, phase, coefficients, fen): def parse_terms(lines): - """The weights and the rows of an `arche terms` run. + """The layout, the weights and the rows of an `arche terms` run. Every row is rebuilt from the weights and held against the evaluation it states. A row that does not rebuild means this file's arithmetic and the engine's have parted company, which is the one failure the seam exists to catch, so it raises rather than dropping the row. """ + layout = None weights = None rows = [] for line in lines: @@ -388,28 +418,35 @@ def parse_terms(lines): if line.startswith(HEADERS): check_header(line) continue + if line.startswith(LAYOUT): + layout = Layout.of(line) + continue words = line.split() if words[0] == "weights": + if layout is None: + raise ValueError(no_layout("the weights line comes first")) count = int(words[1]) weights = [int(word) for word in words[2 : 2 + count]] if len(weights) != count: raise ValueError( f"a weights line saying {count} with {len(weights)} on it" ) - check_layout(count, "a weights line") + layout.check(count, "a weights line") continue if weights is None: raise ValueError("a row arrived before the weights line") identifier, evaluation, phase, coefficients, fen = split_row(words) - rebuilt = reconstruct(coefficients, weights) + rebuilt = reconstruct(coefficients, weights, layout) if rebuilt != evaluation: raise ValueError( f"{identifier} rebuilds to {rebuilt} and the engine says {evaluation}" ) rows.append(Row(identifier, evaluation, phase, coefficients, fen)) + if layout is None: + raise ValueError(no_layout("there is none")) if weights is None: raise ValueError("no weights line") - return weights, rows + return layout, weights, rows class Label: @@ -487,7 +524,8 @@ class Sealed: appearances, which `build_corpus.py` counts. """ - def __init__(self, rows, labels, weights): + def __init__(self, layout, rows, labels, weights): + self._layout = layout self._rows = rows self._labels = labels self._weights = weights @@ -520,6 +558,7 @@ def unseal(self): scored, for the arm whose weights are final.""" opened = Corpus.__new__(Corpus) opened.sealed = None + opened.layout = self._layout opened._load( self._weights, list(self._rows), @@ -551,7 +590,8 @@ class Corpus: apart and scores them for `final` alone. """ - def __init__(self, weights, rows, labels, sealed=None): + def __init__(self, layout, weights, rows, labels, sealed=None): + self.layout = layout joined = [row for row in rows if row.id in labels] # a position appears in exactly one row, because build_corpus.py # deduplicates by fen before it labels and gives the row the lowest key @@ -593,7 +633,10 @@ def __init__(self, weights, rows, labels, sealed=None): sealed - {labels[row.id].pair for row in joined} ) self.sealed = Sealed( - [row for row in joined if groups[row.id] == CALIBRATION], labels, weights + layout, + [row for row in joined if groups[row.id] == CALIBRATION], + labels, + weights, ) kept = [row for row in joined if groups[row.id] != CALIBRATION] self._load(weights, kept, labels, groups) @@ -619,7 +662,7 @@ def _load(self, weights, kept, labels, groups): tapered, material = [], [] for index, row in enumerate(kept): for slot, coefficient in row.coefficients: - (material if is_material(slot) else tapered).append( + (material if self.layout.is_material(slot) else tapered).append( (index, slot, coefficient) ) self.tapered = self._arrays(tapered) @@ -680,19 +723,22 @@ def scatter(self, per_row): """A per-row quantity spread back over the slots, which is the gradient of anything that reads the corpus through `scores`.""" rows, slots, values = self.tapered + slot_count = self.layout.slots gradient = np.bincount( - slots, values * per_row[rows] / TOTAL_PHASE, minlength=SLOTS + slots, values * per_row[rows] / TOTAL_PHASE, minlength=slot_count ) rows, slots, values = self.material - return gradient + np.bincount(slots, values * per_row[rows], minlength=SLOTS) + return gradient + np.bincount( + slots, values * per_row[rows], minlength=slot_count + ) def support(self, mask=None): """How many of the rows each slot appears in. A weight the corpus barely constrains says so here rather than after it has shipped.""" - counts = np.zeros(SLOTS, dtype=np.int64) + counts = np.zeros(self.layout.slots, dtype=np.int64) for rows, slots, _ in (self.tapered, self.material): picked = slots if mask is None else slots[mask[rows]] - counts += np.bincount(picked, minlength=SLOTS).astype(np.int64) + counts += np.bincount(picked, minlength=self.layout.slots).astype(np.int64) return counts @@ -900,6 +946,7 @@ def objective_for(corpus, mask, k, start, penalty, frozen): counts = corpus.counts[mask] total = np.sum(counts) scale = k * math.log(10.0) / 400.0 + slot_count = corpus.layout.slots rows, slots, values = corpus.tapered material_rows, material_slots, material_values = corpus.material picked = mask[rows] @@ -935,11 +982,11 @@ def objective(weights): gradient = np.bincount( tapered_part[1], tapered_part[2] * per_row[tapered_part[0]] / TOTAL_PHASE, - minlength=SLOTS, + minlength=slot_count, ) + np.bincount( material_part[1], material_part[2] * per_row[material_part[0]], - minlength=SLOTS, + minlength=slot_count, ) gradient += 2.0 * penalty * np.where(frozen, 0.0, slack) gradient[frozen] = 0.0 @@ -955,7 +1002,7 @@ def quantize(weights): return np.rint(np.asarray(weights)).astype(np.int64) -def bounds_hold(weights): +def bounds_hold(weights, layout): """Whether a quantized vector stays inside what the engine's arithmetic can carry: each half of a packed pair is an `i16`, a boardful of them is summed into one, and an evaluation over the mate threshold would be read as @@ -964,41 +1011,38 @@ def bounds_hold(weights): A one-sided boardful, both colours, against the sixteen bits the halves have to stay inside. - All three leaf terms are in the same sum, so they are priced here too - rather than left out of a figure that reads as the whole vector. A piece of - each kind at its widest is the mobility price, three of each of its seven - counts is the shelter's, and eight of each of its eight is the pawn - structure's. All are screens rather than proofs: a side that promoted could - cover more, the worst a board can be arranged into comes to 313 squares - against the 62 mobility charges, a side's open and half open files come to - three between them rather than three each, and eight pawns cannot fill the - pawn term's sixty four charges between them. Over - the 1,809 positions of the three suites the largest one-sided mobility - difference was 37, so at the centipawn weights a fit produces none of the - figures is near the sixteen bits. What this catches is a vector that has - gone somewhere else entirely. + Every leaf term is in the same sum, so each is priced here too rather than + left out of a figure that reads as the whole vector, and what one side can + show of each of its counts is what `BOUNDS` names. All are screens rather + than proofs: a side that promoted could cover more, the worst a board can + be arranged into comes to 313 squares against the 62 mobility charges, a + side's open and half open files come to three between them rather than + three each, and eight pawns cannot fill the pawn term's sixty four charges + between them. Over the 1,809 positions of the three suites the largest + one-sided mobility difference was 37, so at the centipawn weights a fit + produces none of the figures is near the sixteen bits. What this catches is + a vector that has gone somewhere else entirely. """ weights = np.abs(np.asarray(weights)) - tables = weights[:MATERIAL_SLOT] - midgame = tables[:MIDGAME_SLOTS].reshape(6, 64) - endgame = tables[MIDGAME_SLOTS:MATERIAL_SLOT].reshape(6, 64) + tables = weights[: layout.start["material"]] + midgame = tables[: layout.widths["midgame"]].reshape(6, 64) + endgame = tables[layout.widths["midgame"] :].reshape(6, 64) worst = 2 * max(int(midgame.max(axis=0).sum()), int(endgame.max(axis=0).sum())) - mobility = weights[MOBILITY_SLOT:SHELTER_SLOT].reshape(2, MOBILITY_SLOTS) - worst += 2 * int((mobility * MAX_COUNT).sum(axis=1).max()) - shelter = weights[SHELTER_SLOT:PAWN_SLOT].reshape(2, SHELTER_SLOTS) - worst += 2 * MAX_SHELTER * int(shelter.sum(axis=1).max()) - pawns = weights[PAWN_SLOT:].reshape(2, PAWN_SLOTS) - worst += 2 * MAX_PAWNS * int(pawns.sum(axis=1).max()) + for name in layout.terms: + # a term's block is its midgame half and then its endgame one, and + # each half is charged the counts a side can show of it + halves = weights[layout.block(name)].reshape(2, layout.widths[name]) + worst += 2 * int((halves * np.array(BOUNDS[name])).sum(axis=1).max()) return worst < 32767, worst -def read_weights(path): +def read_weights(path, layout): weights = json.loads(Path(path).read_text(encoding="utf-8")) - check_layout(len(weights), f"{path} holds a vector") + layout.check(len(weights), f"{path} holds a vector") return np.array(weights, dtype=np.float64) -def table_scale(weights, shipped): +def table_scale(weights, shipped, layout): """How much larger the table entries have grown, as the ratio of their root mean squares. @@ -1009,8 +1053,8 @@ def table_scale(weights, shipped): overall scale are one degree of freedom, and a fit given enough licence will spend the loss on the scale rather than on the shape. """ - fitted = np.asarray(weights)[:MATERIAL_SLOT] - before = np.asarray(shipped)[:MATERIAL_SLOT] + fitted = np.asarray(weights)[: layout.start["material"]] + before = np.asarray(shipped)[: layout.start["material"]] return float( math.sqrt(float(fitted @ fitted)) / max(math.sqrt(float(before @ before)), 1e-12) @@ -1105,7 +1149,8 @@ def report(corpus, named, k, out=None): corpus.counts[train], ) print( - f"{name} scale {table_scale(weights, corpus.weights):.3f} own_k {own:.4f} " + f"{name} scale {table_scale(weights, corpus.weights, corpus.layout):.3f} " + f"own_k {own:.4f} " f"selection mse at own_k " f"{mean_squared_error(corpus.scores(np.asarray(weights, dtype=np.float64))[selection], corpus.results[selection], corpus.counts[selection], own):.6f}", file=out, @@ -1209,7 +1254,7 @@ def cross_validate(corpus, penalties, start, frozen, iterations, out=None): start, iterations, ) - if not bounds_hold(quantize(fitted))[0]: + if not bounds_hold(quantize(fitted), corpus.layout)[0]: outside.add(penalty) errors[penalty][held] = squared_errors( corpus.scores(fitted)[held], results, k @@ -1298,7 +1343,7 @@ def choose_penalty( mean, error, naive, design = paired_difference( shipped, errors, counts, corpus.games[corpus.selection] ) - refused = not bounds_hold(quantize(fitted))[0] + refused = not bounds_hold(quantize(fitted), corpus.layout)[0] print( f"penalty {penalty:g} selection mse {loss:.6f} vs shipped {mean:+.6f} " f"se {error:.6f} per position {naive:.6f} design {design:.1f}" @@ -1493,12 +1538,12 @@ def learning_curve( def load(args): - weights, rows = parse_terms( + layout, weights, rows = parse_terms( Path(args.terms).read_text(encoding="utf-8").splitlines() ) labels = parse_corpus(Path(args.corpus).read_text(encoding="utf-8").splitlines()) sealed = sealed_pairs(args.sealed) if getattr(args, "sealed", None) else None - corpus = Corpus(weights, rows, labels, sealed) + corpus = Corpus(layout, weights, rows, labels, sealed) if not len(corpus): raise SystemExit("tune.py: no row of the extraction is in the corpus") for name, mask in (("train", corpus.train), ("selection", corpus.selection)): @@ -1516,12 +1561,13 @@ def command_loss(args): ) named = [("shipped", corpus.weights)] for path in args.weights or []: - named.append((Path(path).stem, read_weights(path))) + named.append((Path(path).stem, read_weights(path, corpus.layout))) report(corpus, named, k) return 0 def frozen_slots( + layout, free_material, held_tables=False, held_mobility=False, @@ -1562,17 +1608,19 @@ def frozen_slots( arm passes follows from the one term it means to move and not from where that term sits in the vector. """ - frozen = np.zeros(SLOTS, dtype=bool) + frozen = np.zeros(layout.slots, dtype=bool) if not free_material: - frozen[MATERIAL_SLOT:MOBILITY_SLOT] = True + frozen[layout.block("material")] = True if held_tables: - frozen[:MATERIAL_SLOT] = True - if held_mobility: - frozen[MOBILITY_SLOT:SHELTER_SLOT] = True - if held_shelter: - frozen[SHELTER_SLOT:PAWN_SLOT] = True - if held_pawn: - frozen[PAWN_SLOT:] = True + frozen[layout.block("midgame")] = True + frozen[layout.block("endgame")] = True + for held, name in ( + (held_mobility, "mobility"), + (held_shelter, "shelter"), + (held_pawn, "pawn_structure"), + ): + if held: + frozen[layout.block(name)] = True return frozen @@ -1580,6 +1628,7 @@ def command_cv(args): corpus = load(args) start = corpus.weights.copy() frozen = frozen_slots( + corpus.layout, args.free_material, args.hold_tables, args.hold_mobility, @@ -1606,6 +1655,7 @@ def command_fit(args): corpus = load(args) start = corpus.weights.copy() frozen = frozen_slots( + corpus.layout, args.free_material, args.hold_tables, args.hold_mobility, @@ -1627,7 +1677,7 @@ def command_fit(args): raise SystemExit("tune.py: every penalty on the grid left the tables too large") print(f"chose penalty {penalty:g}") rounded = quantize(fitted) - _, worst = bounds_hold(rounded) + _, worst = bounds_hold(rounded, corpus.layout) if args.out: Path(args.out).write_text( json.dumps([int(value) for value in rounded]), @@ -1669,6 +1719,7 @@ def command_curve(args): shares = curve_shares(args.shares, args.draws) start = corpus.weights.copy() frozen = frozen_slots( + corpus.layout, args.free_material, args.hold_tables, args.hold_mobility, @@ -1722,14 +1773,14 @@ def command_final(args): whatever file they arrive in. """ corpus = load(args) - frozen = read_weights(args.weights) + frozen = read_weights(args.weights, corpus.layout) if np.any(frozen != np.rint(frozen)): raise SystemExit( "tune.py: the frozen vector is not the integers that would ship; " "quantize it first" ) frozen = frozen.astype(np.int64) - holds, worst = bounds_hold(frozen) + holds, worst = bounds_hold(frozen, corpus.layout) if not holds: raise SystemExit(f"tune.py: the frozen vector's boardful is {worst} of 32767") corpus_sha = sha256_of(args.corpus)