Run feature checks on every database seed-forum supports

.github/workflows/ci.yml

@@ -19,7 +19,7 @@ jobs:
           persist-credentials: false
 
       - name: ShellCheck
-        run: shellcheck -x git-extensions.sh git-release-style.sh git-release-langpack.sh lib/github-release.sh lib/release-source.sh tests/test-release-scripts.sh tests/test-release-api.sh phpbb-test-board/setup-board.sh phpbb-test-board/run-unit-tests.sh phpbb-test-board/php-lint.sh tests/test-phpbb-test-board.sh tests/test-pr-wait.sh
+        run: shellcheck -x git-extensions.sh git-release-style.sh git-release-langpack.sh lib/github-release.sh lib/release-source.sh tests/test-release-scripts.sh tests/test-release-api.sh phpbb-test-board/setup-board.sh phpbb-test-board/run-unit-tests.sh phpbb-test-board/php-lint.sh phpbb-test-board/db-matrix.sh tests/test-phpbb-test-board.sh tests/test-pr-wait.sh
 
       - name: Tests
         run: tests/test-release-scripts.sh

phpbb-test-board/README.md

@@ -91,6 +91,20 @@ phpbb-test-board/feature_checks.py path/to/extension[@REF] [more extensions...]
 
 Exercises the main feature of each phpbbmodders extension that has a check. One example: a moderator can't warn a user in a group that isn't ticked. `--help` lists the extensions with checks. To add one, write a `check_<name>()` function and add it to the `CHECKS` table, keyed by the extension's composer name.
 
+**On another database:** `--seed-board BOARD_ROOT --port PORT` runs the same checks against a board that [seed-forum](https://github.com/phpbbmodders/seed-forum) built, on whichever database it uses: SQLite, MySQL, MariaDB, PostgreSQL or MSSQL. The extensions must already be enabled there, and `--port` is the board's own `server_port`. Nothing is installed or reset. After its check, each extension is disabled, purged and enabled again through phpBB's CLI, which runs its migrations backwards and forwards on that database. SQL and PHP go through seed-forum's `bin/board-sql.php` and `bin/board-php.py`; `--seed-forum DIR` points at the checkout (default: `seed-forum` next to this repository). A check should build route URLs with `b.route("path")`, which gives `app.php/path` on phpBB 3.3 and `index.php/path` on 4.0, and compare numbers from `b.sql()` as strings, since most database drivers return them that way.
+
+### `db-matrix.sh`: every phpBB branch and database
+
+```bash
+phpbb-test-board/db-matrix.sh path/to/extension [-b 3.3,master] [-d sqlite,mysql,mariadb,postgres,mssql]
+```
+
+For each phpBB branch and database, builds a seed-forum board with the extension enabled (bind-mounted from the checkout as it is on disk), runs `feature_checks.py --seed-board` against it, and stops that board's database container. It prints a line per board and a summary. Boards and logs go in `--work-dir` (default `~/.cache/phpbb-release-tools/db-matrix`) and are rebuilt on every run; ports start at `--base-port` (default 8130). Oracle isn't covered: phpBB is dropping it.
+
+The MSSQL container accepts Microsoft's SQL Server Developer licence; leave `mssql` out of `-d` if that hasn't been agreed. Needs Docker and a seed-forum checkout with `board.database` support.
+
+Exit status: 0 when every board passed, 1 when one failed, 2 for bad arguments.
+
 ### `screenshots.py`: documentation screenshots
 
 ```bash

phpbb-test-board/_board.py

@@ -9,6 +9,7 @@
 import os
 import re
 import shutil
+import socket
 import sqlite3
 import subprocess
 import time
@@ -238,6 +239,119 @@ def add_user(self, name: str, password: str) -> int:
                        "'user_type'=>USER_NORMAL,'user_regdate'=>time()]);")
         return int(out.strip().splitlines()[-1])
 
+    def route(self, path: str) -> str:
+        """Return the board-relative URL of an extension route (app.php/PATH on phpBB 3.3)."""
+        return f"app.php/{path}"
+
+
+class SeedBoard(Board):
+    """An installed seed-forum board, on any database seed-forum supports.
+
+    seed-forum builds and seeds the board, with its extensions enabled; this
+    only drives it. PHP, SQL and phpBB's CLI run through seed-forum's
+    bin/board-php.py (host PHP, or a driver container for MSSQL) and
+    bin/board-sql.php (phpBB's own database layer), so the same checks work on
+    SQLite, MySQL, MariaDB, PostgreSQL and MSSQL. The admin login is
+    seed-forum's fixture admin. Nothing is installed or reset here: rebuild the
+    board with seed-forum between runs that change it.
+    """
+
+    ADMIN_USER = "admin"
+    ADMIN_PASSWORD = "KbTest1234!"
+
+    def __init__(self, root: str, port: int, seed_forum: str) -> None:
+        self.root = Path(root).expanduser().resolve()
+        if not (self.root / "config.php").is_file():
+            raise SystemExit(f"error: no installed board at {self.root}; build it with seed-forum first")
+        self.seed_forum = Path(seed_forum).expanduser().resolve()
+        if not (self.seed_forum / "bin/board-php.py").is_file():
+            raise SystemExit(f"error: {self.seed_forum} has no bin/board-php.py; pass a seed-forum checkout "
+                             "with board.database support")
+        self.admin_user = self.ADMIN_USER
+        self.admin_password = self.ADMIN_PASSWORD
+        self.port = port
+        self.base = f"http://localhost:{port}"
+        # Inside the board, so a driver container can write it too.
+        self.php_log = self.root / ".seed-forum-php-errors.log"
+        self.installed = []
+        constants = (self.root / "includes/constants.php").read_text()
+        version = re.search(r"define\('PHPBB_VERSION', '([^']+)'\)", constants)
+        self.version = version.group(1) if version else ""
+
+    def _php(self, *args: str, check: bool = True) -> subprocess.CompletedProcess:
+        """Run PHP for this board through seed-forum's board-php.py."""
+        r = subprocess.run(["python3", str(self.seed_forum / "bin/board-php.py"), str(self.root), *args],
+                           cwd=self.root, capture_output=True, text=True, timeout=600)
+        if check and r.returncode != 0:
+            raise RuntimeError(f"PHP failed: {(r.stderr or r.stdout).strip()[-400:]}")
+        return r
+
+    def cli(self, *args: str) -> str:
+        """Run phpBB's CLI and return its combined output."""
+        r = self._php("bin/phpbbcli.php", *args, "--no-interaction", check=False)
+        return r.stdout + r.stderr
+
+    def purge_cache(self) -> None:
+        """Purge the board's cache through phpBB's CLI."""
+        out = self.cli("cache:purge")
+        if "Purged" not in out and "purged" not in out:
+            raise RuntimeError(f"cache purge failed: {out.strip()[-300:]}")
+
+    def sql(self, query: str, args: tuple = ()) -> list:
+        """Run one SQL statement through phpBB's database layer; return rows as tuples.
+
+        Values come back as phpBB's driver returns them, which on most
+        databases means strings, so compare numbers as strings.
+        """
+        r = self._php(str(self.seed_forum / "bin/board-sql.php"), str(self.root), query, json.dumps(list(args)))
+        result = json.loads(r.stdout.strip().splitlines()[-1])
+        return [tuple(row) for row in result] if isinstance(result, list) else []
+
+    def php(self, code: str, check: bool = False) -> str:
+        """Run PHP inside the board (common.php and functions_user.php loaded); return its output."""
+        boot = ("define('IN_PHPBB',1);$phpbb_root_path='./';$phpEx='php';require 'common.php';"
+                "require 'includes/functions_user.php';$user->session_begin();$auth->acl($user->data);$user->setup();")
+        return self._php("-r", boot + code, check=check).stdout
+
+    def reset(self) -> None:
+        """Not available: rebuild a seed-forum board with seed-forum --reset."""
+        raise RuntimeError("a seed-forum board is rebuilt with seed-forum --reset, not by feature checks")
+
+    @contextmanager
+    def serve(self) -> Iterator[None]:
+        """Serve the board on its own port, with PHP error logging on.
+
+        Refuses to start when something already answers on the port, so the
+        checks never run against a leftover server for another board.
+        """
+        with socket.socket() as probe:
+            if probe.connect_ex(("127.0.0.1", self.port)) == 0:
+                raise RuntimeError(f"port {self.port} is already in use; stop whatever serves it first")
+        self.php_log.write_text("")
+        server = subprocess.Popen(["python3", str(self.seed_forum / "bin/board-php.py"), str(self.root),
+                                   "-d", "opcache.enable=0", "-d", "log_errors=1",
+                                   "-d", f"error_log={self.php_log}", "-S", f"127.0.0.1:{self.port}",
+                                   "-t", str(self.root)],
+                                  cwd=self.root, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        try:
+            for _ in range(300):
+                try:
+                    requests.head(f"{self.base}/images/index.htm", timeout=2)
+                    break
+                except requests.ConnectionError:
+                    time.sleep(0.1)
+            yield
+        finally:
+            server.terminate()
+            try:
+                server.wait(timeout=20)
+            except subprocess.TimeoutExpired:
+                server.kill()
+
+    def route(self, path: str) -> str:
+        """Return the board-relative URL of an extension route (index.php/PATH on phpBB 4.0)."""
+        return f"index.php/{path}" if self.version.startswith("4.") else f"app.php/{path}"
+
 
 # Composer packages that are never phpBB extensions.
 NOT_EXTENSIONS = re.compile(r"^(php|ext-.*|lib-.*|composer/.*|phpbb/phpbb)$")

phpbb-test-board/db-matrix.sh

@@ -0,0 +1,152 @@
+#!/usr/bin/bash
+#
+# Run extensions' feature checks on every phpBB branch and database seed-forum
+# supports: for each branch and database, build a seed-forum board with the
+# extensions enabled (bind-mounted from their checkouts, as they are on disk),
+# run feature_checks.py --seed-board against it (checks plus a
+# disable/purge/enable round trip), then unmount its extensions and stop its
+# database container.
+# Prints one result line per board and a summary.
+#
+# Needs a seed-forum checkout with board.database support, Docker, and what
+# seed-forum and feature_checks.py need. MSSQL accepts Microsoft's SQL Server
+# Developer licence (ACCEPT_EULA=Y) in its container; leave mssql out of -d if
+# that hasn't been agreed. Oracle isn't covered: phpBB is dropping it.
+#
+# Without arguments it shows help.
+# Exit status: 0 when every board passed, 1 when one failed, 2 for bad arguments.
+
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BRANCHES="3.3,master"
+DATABASES="sqlite,mysql,mariadb,postgres,mssql"
+WORK_DIR="$HOME/.cache/phpbb-release-tools/db-matrix"
+SEED_FORUM="$(cd "$SCRIPT_DIR/../.." && pwd)/seed-forum"
+BASE_PORT=8130
+KEEP=0
+
+usage() {
+	cat <<'EOF'
+Usage: db-matrix.sh [options] EXTENSION_CHECKOUT...
+
+Run each extension's feature checks on every phpBB branch and database.
+
+Options:
+  -b, --branches LIST     phpBB branches, comma-separated (default: 3.3,master)
+  -d, --databases LIST    databases, comma-separated (default: sqlite,mysql,mariadb,postgres,mssql)
+  -w, --work-dir DIR      where boards and their configs go (default: ~/.cache/phpbb-release-tools/db-matrix)
+  -s, --seed-forum DIR    seed-forum checkout (default: seed-forum next to this repository)
+  -p, --base-port PORT    first port for the boards; each board gets the next one (default: 8130)
+  -k, --keep              leave the extensions mounted and the database containers running after the checks
+  -h, --help              show this help and exit
+
+Boards are kept in the work dir between runs, so phpBB is only downloaded once
+per branch and database. Each run rebuilds them from scratch.
+EOF
+}
+
+die() {
+	echo "Error: $*" >&2
+	exit 2
+}
+
+EXTENSIONS=()
+while [ $# -gt 0 ]; do
+	case "$1" in
+		-b|--branches) BRANCHES="${2:?}"; shift 2 ;;
+		-d|--databases) DATABASES="${2:?}"; shift 2 ;;
+		-w|--work-dir) WORK_DIR="${2:?}"; shift 2 ;;
+		-s|--seed-forum) SEED_FORUM="${2:?}"; shift 2 ;;
+		-p|--base-port) BASE_PORT="${2:?}"; shift 2 ;;
+		-k|--keep) KEEP=1; shift ;;
+		-h|--help) usage; exit 0 ;;
+		-*) die "unknown option $1 (see --help)" ;;
+		*) EXTENSIONS+=("$(cd "$1" && pwd)"); shift ;;
+	esac
+done
+if [ ${#EXTENSIONS[@]} -eq 0 ]; then
+	usage
+	exit 0
+fi
+[[ "$BASE_PORT" =~ ^[0-9]+$ ]] || die "--base-port must be a number"
+[ -f "$SEED_FORUM/bin/db-container.py" ] || die "$SEED_FORUM has no board.database support (bin/db-container.py)"
+for ext in "${EXTENSIONS[@]}"; do
+	[ -f "$ext/composer.json" ] || die "$ext has no composer.json"
+done
+
+mkdir -p "$WORK_DIR"
+IFS=, read -r -a branch_list <<< "$BRANCHES"
+IFS=, read -r -a database_list <<< "$DATABASES"
+
+# One YAML config per board, listing the extensions by a label from their names.
+write_config() {
+	local file="$1" root="$2" branch="$3" database="$4" port="$5"
+	{
+		echo "# Written by phpbb-release-tools db-matrix.sh; rewritten on every run."
+		echo "board:"
+		echo "  root: '$root'"
+		echo "  phpbb_branch: '$branch'"
+		echo "  database: $database"
+		echo "  server_name: localhost"
+		echo "  server_port: '$port'"
+		echo "  seeds:"
+		echo "    - bin/seed-standard-fixtures.php"
+		echo "styles: {}"
+		echo "extensions:"
+		for ext in "${EXTENSIONS[@]}"; do
+			echo "  $(jq -r '.name' "$ext/composer.json" | tr '/' '_'):"
+			echo "    source: '$ext'"
+		done
+	} > "$file"
+}
+
+summary=()
+failed=0
+port="$BASE_PORT"
+for branch in "${branch_list[@]}"; do
+	for database in "${database_list[@]}"; do
+		name="${branch//./}-$database"
+		root="$WORK_DIR/boards/$name"
+		config="$WORK_DIR/$name.yml"
+		log="$WORK_DIR/$name.log"
+		if ss -ltn | grep -q ":$port "; then
+			echo "== $name: port $port is in use; skipping" >&2
+			summary+=("$name: SKIPPED (port $port in use)")
+			failed=1
+			port=$((port + 1))
+			continue
+		fi
+		mkdir -p "$root"
+		write_config "$config" "$root" "$branch" "$database" "$port"
+		echo "== $name: building the board on port $port (log: $log)"
+		if ! python3 "$SEED_FORUM/bin/seed-forum.py" --config "$config" --reset > "$log" 2>&1; then
+			echo "   board build FAILED; last lines of the log:"
+			tail -15 "$log" | sed 's/^/   /'
+			summary+=("$name: BUILD FAILED (see $log)")
+			failed=1
+		else
+			echo "== $name: running the checks"
+			if "$SCRIPT_DIR/feature_checks.py" "${EXTENSIONS[@]}" --seed-board "$root" --port "$port" \
+					--seed-forum "$SEED_FORUM" 2>&1 | tee -a "$log"; then
+				summary+=("$name: passed")
+			else
+				summary+=("$name: FAILED (see $log)")
+				failed=1
+			fi
+		fi
+		if [ "$KEEP" = 0 ]; then
+			# Unmount the extensions, so deleting the work dir can never reach a checkout
+			python3 "$SEED_FORUM/bin/seed-forum.py" --config "$config" --unmount >> "$log" 2>&1 \
+				|| echo "   Warning: could not unmount the extensions from $root; see $log" >&2
+			if [ "$database" != sqlite ]; then
+				python3 "$SEED_FORUM/bin/db-container.py" stop "$root" >> "$log" 2>&1 || true
+			fi
+		fi
+		port=$((port + 1))
+	done
+done
+
+echo "== Summary"
+printf '   %s\n' "${summary[@]}"
+exit "$failed"