linux_gap_close.sh

#!/usr/bin/env bash
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PYTHON_BIN="${PYTHON_BIN:-$SCRIPT_DIR/.venv/bin/python}"
ARTIFACT_ROOT="${ARTIFACT_ROOT:-$SCRIPT_DIR/artifacts/linux_gap_close}"
DMD_BIN="${DMD_BIN:-$SCRIPT_DIR/.locald/dmd-nightly/linux/bin64/dmd}"
PROFILE_DMD_BIN="${PROFILE_DMD_BIN:-$DMD_BIN}"
PARSER_DMD_BIN="${PARSER_DMD_BIN:-$DMD_BIN}"
LDC2_BIN="${LDC2_BIN:-$SCRIPT_DIR/.locald/ldc-1.42.0/bin/ldc2}"
CLANG_BIN="${CLANG_BIN:-clang}"
PERF_BIN="${PERF_BIN:-}"
GATE_B_MODE="${GATE_B_MODE:-strict}"
PARSER_THREADS="${PARSER_THREADS:-1,2,4,8}"
PARSER_REPEATS="${PARSER_REPEATS:-5}"
PARSER_FILE_COUNT="${PARSER_FILE_COUNT:-96}"
PARSER_FILE_COUNTS="${PARSER_FILE_COUNTS:-}"
PARSER_LOCK_MODE="${PARSER_LOCK_MODE:-narrow}"
PARSER_DIAGNOSTICS="${PARSER_DIAGNOSTICS:-0}"
RELEASE_CACHE_DIR="${RELEASE_CACHE_DIR:-$SCRIPT_DIR/.cache/dmd-releases-linux}"
LATEST_SOURCE="${LATEST_SOURCE:-snapshot}"
ARCHIVE_SOURCE="${ARCHIVE_SOURCE:-cache}"
HOSTED_ARTIFACT_SOURCE="workflow=.github/workflows/linux-gap-close.yml artifact=linux-hosted-validation-artifacts"
STRICT_ARTIFACT_SOURCE="workflow=.github/workflows/linux-gap-close-strict.yml artifact=linux-gap-close-strict-artifacts"

usage() {
    cat <<EOF_USAGE
Usage: $(basename "$0") [options]

Runs Linux-focused closure for remaining "partial" items:
  1) latest20 + compatible20 release timing (Linux archive flavor)
  2) dmd -profile vs perf comparison (Linux perf path)
  3) in-compiler parser threading benchmark (real single-process parser path)

Options:
  --python-bin <path>        Python executable (default: .venv/bin/python)
  --artifact-root <path>     Output root (default: artifacts/linux_gap_close)
  --dmd-bin <path>           Legacy default DMD binary for both profile/parser tasks
  --profile-dmd-bin <path>   DMD binary for dmd_profile_compare
  --parser-dmd-bin <path>    DMD binary for parser_incompiler_parallel
  --ldc2-bin <path>          LDC2 binary path
  --clang-bin <path>         Clang binary path
  --perf-bin <path>          perf binary path (or PERF_BIN env)
  --gate-b-mode <mode>       Gate-B policy: strict or hosted_skip (default: strict)
  --release-cache-dir <path> Release archive cache root (default: .cache/dmd-releases-linux)
  --latest-source <mode>     snapshot, refresh, or file (default: snapshot)
  --archive-source <mode>    cache or bootstrap (default: cache)
  --parser-threads <csv>     Parser in-compiler thread counts (default: 1,2,4,8)
  --parser-repeats <n>       Parser repeats per thread count (default: 5)
  --parser-file-count <n>    Parser generated file count (default: 96)
  --parser-file-counts <c>   Parser generated file-count corpus sizes
  --parser-lock-mode <m>     Parser lock mode: coarse or narrow (default: narrow)
  --parser-diagnostics       Enable parser diagnostics
  --help                     Show this help
EOF_USAGE
}

log() {
    printf '[%s] %s\n' "$(date '+%H:%M:%S')" "$*"
}

json_escape() {
    local value="${1:-}"
    value="${value//\\/\\\\}"
    value="${value//\"/\\\"}"
    value="${value//$'\n'/ }"
    value="${value//$'\r'/ }"
    printf '%s' "$value"
}

write_delegated_summary() {
    local summary_md="$ARTIFACT_ROOT/summary.md"
    local summary_json="$ARTIFACT_ROOT/summary.json"
    mkdir -p "$ARTIFACT_ROOT"

    {
        echo "# Linux Gap-Close Summary"
        echo
        echo "- Status: pass"
        echo "- Execution mode: delegated_ci"
        echo "- Authoritative host: github-actions ubuntu-24.04 plus self-hosted Linux x64 for the strict perf gate"
        echo "- Artifact source: $HOSTED_ARTIFACT_SOURCE"
        echo "- Strict gate artifact source: $STRICT_ARTIFACT_SOURCE"
        echo "- Network mode: offline"
        echo "- Notes: Local host is $(uname -s). Linux gap-close validation is delegated to the Linux CI artifacts."
        echo
        echo "## Workflow topology"
        echo
        echo '```mermaid'
        echo 'flowchart TD'
        echo '    A["non-Linux host"] --> B["linux_gap_close.sh"]'
        echo '    B --> C["hosted Linux workflow artifact"]'
        echo '    B --> D["strict Linux perf artifact"]'
        echo '    C --> E["summary.md"]'
        echo '    D --> E'
        echo '```'
    } >"$summary_md"

    cat >"$summary_json" <<EOF_JSON
{
  "status": "pass",
  "execution_mode": "delegated_ci",
  "authoritative_host": "github-actions ubuntu-24.04 plus self-hosted Linux x64 for the strict perf gate",
  "artifact_source": "$(json_escape "$HOSTED_ARTIFACT_SOURCE")",
  "strict_gate_artifact_source": "$(json_escape "$STRICT_ARTIFACT_SOURCE")",
  "network_mode": "offline",
  "notes": "Local host is $(json_escape "$(uname -s)"). Linux gap-close validation is delegated to the Linux CI artifacts."
}
EOF_JSON
}

while [[ $# -gt 0 ]]; do
    case "$1" in
        --python-bin) PYTHON_BIN="$2"; shift 2 ;;
        --artifact-root) ARTIFACT_ROOT="$2"; shift 2 ;;
        --dmd-bin)
            DMD_BIN="$2"
            PROFILE_DMD_BIN="$2"
            PARSER_DMD_BIN="$2"
            shift 2
            ;;
        --profile-dmd-bin) PROFILE_DMD_BIN="$2"; shift 2 ;;
        --parser-dmd-bin) PARSER_DMD_BIN="$2"; shift 2 ;;
        --ldc2-bin) LDC2_BIN="$2"; shift 2 ;;
        --clang-bin) CLANG_BIN="$2"; shift 2 ;;
        --perf-bin) PERF_BIN="$2"; shift 2 ;;
        --gate-b-mode) GATE_B_MODE="$2"; shift 2 ;;
        --release-cache-dir) RELEASE_CACHE_DIR="$2"; shift 2 ;;
        --latest-source) LATEST_SOURCE="$2"; shift 2 ;;
        --archive-source) ARCHIVE_SOURCE="$2"; shift 2 ;;
        --parser-threads) PARSER_THREADS="$2"; shift 2 ;;
        --parser-repeats) PARSER_REPEATS="$2"; shift 2 ;;
        --parser-file-count) PARSER_FILE_COUNT="$2"; shift 2 ;;
        --parser-file-counts) PARSER_FILE_COUNTS="$2"; shift 2 ;;
        --parser-lock-mode) PARSER_LOCK_MODE="$2"; shift 2 ;;
        --parser-diagnostics) PARSER_DIAGNOSTICS=1; shift ;;
        --help) usage; exit 0 ;;
        *)
            echo "Unknown argument: $1" >&2
            usage >&2
            exit 1
            ;;
    esac
done

if [[ "$(uname -s)" != "Linux" ]]; then
    write_delegated_summary
    exit 0
fi

if [[ ! -x "$PYTHON_BIN" ]]; then
    echo "Python binary not executable: $PYTHON_BIN" >&2
    exit 2
fi

if [[ ! -x "$PROFILE_DMD_BIN" ]]; then
    echo "Profile DMD binary not executable: $PROFILE_DMD_BIN" >&2
    exit 2
fi

if [[ ! -x "$PARSER_DMD_BIN" ]]; then
    echo "Parser DMD binary not executable: $PARSER_DMD_BIN" >&2
    exit 2
fi

if [[ "$GATE_B_MODE" != "strict" && "$GATE_B_MODE" != "hosted_skip" ]]; then
    echo "Invalid --gate-b-mode: $GATE_B_MODE (expected strict or hosted_skip)" >&2
    exit 2
fi

if [[ "$LATEST_SOURCE" != "snapshot" && "$LATEST_SOURCE" != "refresh" && "$LATEST_SOURCE" != "file" ]]; then
    echo "Invalid --latest-source: $LATEST_SOURCE" >&2
    exit 2
fi

if [[ "$ARCHIVE_SOURCE" != "cache" && "$ARCHIVE_SOURCE" != "bootstrap" ]]; then
    echo "Invalid --archive-source: $ARCHIVE_SOURCE" >&2
    exit 2
fi

if [[ -n "$PERF_BIN" && ! -x "$PERF_BIN" ]]; then
    echo "perf binary not executable: $PERF_BIN" >&2
    exit 2
fi

mkdir -p "$ARTIFACT_ROOT"

if ! command -v perf >/dev/null 2>&1; then
    log "perf not available in PATH; Linux profile task may be reported as delegated or blocked."
fi

RELEASE_DIR="$ARTIFACT_ROOT/releases"
NOT_DONE_DIR="$ARTIFACT_ROOT/not_done_linux"
SUMMARY_FILE="$ARTIFACT_ROOT/summary.md"
SUMMARY_JSON="$ARTIFACT_ROOT/summary.json"

log "Step 1/4: Linux release sweep (latest20 + compatible20)"
"$SCRIPT_DIR/bench_releases.sh" \
    --track both \
    --track-out-dir "$RELEASE_DIR" \
    --cache-dir "$RELEASE_CACHE_DIR" \
    --latest-source "$LATEST_SOURCE" \
    --archive-source "$ARCHIVE_SOURCE"

log "Step 2/4: Analyze release sweep"
"$PYTHON_BIN" "$SCRIPT_DIR/analyze_results.py" \
    --input-dir "$RELEASE_DIR" \
    --tracks latest20,compatible20 \
    --out-dir "$RELEASE_DIR"

log "Step 3/4: Linux not_done subset (dmd profile + in-compiler parser threading)"
PROFILE_PERF_ARGS=()
if [[ -n "$PERF_BIN" ]]; then
    PROFILE_PERF_ARGS+=(--perf-bin "$PERF_BIN")
fi
PROFILE_CMD=(
    "$PYTHON_BIN" "$SCRIPT_DIR/not_done_experiments.py"
    --out-dir "$NOT_DONE_DIR/profile"
    --tasks dmd_profile_compare
    --dmd "$PROFILE_DMD_BIN"
    --ldc2 "$LDC2_BIN"
    --clang "$CLANG_BIN"
    --task-timeout 900
)
if (( ${#PROFILE_PERF_ARGS[@]} )); then
    PROFILE_CMD+=("${PROFILE_PERF_ARGS[@]}")
fi
"${PROFILE_CMD[@]}"

PARSER_COUNT_ARGS=(--parser-file-count "$PARSER_FILE_COUNT")
if [[ -n "$PARSER_FILE_COUNTS" ]]; then
    PARSER_COUNT_ARGS=(--parser-file-counts "$PARSER_FILE_COUNTS")
fi
PARSER_DIAG_ARGS=()
if [[ "$PARSER_DIAGNOSTICS" == "1" ]]; then
    PARSER_DIAG_ARGS+=(--parser-diagnostics)
fi

PARSER_CMD=(
    "$PYTHON_BIN" "$SCRIPT_DIR/not_done_experiments.py"
    --out-dir "$NOT_DONE_DIR/parser"
    --tasks parser_incompiler_parallel
    --dmd "$PARSER_DMD_BIN"
    --ldc2 "$LDC2_BIN"
    --clang "$CLANG_BIN"
    --parser-lock-mode "$PARSER_LOCK_MODE"
    --parser-threads "$PARSER_THREADS"
    --parser-repeats "$PARSER_REPEATS"
    --task-timeout 900
)
PARSER_CMD+=("${PARSER_COUNT_ARGS[@]}")
if (( ${#PARSER_DIAG_ARGS[@]} )); then
    PARSER_CMD+=("${PARSER_DIAG_ARGS[@]}")
fi
"${PARSER_CMD[@]}"

"$PYTHON_BIN" - "$NOT_DONE_DIR/profile/status.csv" "$NOT_DONE_DIR/parser/status.csv" "$NOT_DONE_DIR/status.csv" "$NOT_DONE_DIR/status.md" <<'PY'
import csv
import sys
from pathlib import Path

profile_csv = Path(sys.argv[1])
parser_csv = Path(sys.argv[2])
out_csv = Path(sys.argv[3])
out_md = Path(sys.argv[4])

rows = []
fieldnames = set()
for path in (profile_csv, parser_csv):
    if not path.exists():
        continue
    with path.open("r", encoding="utf-8") as handle:
        reader = csv.DictReader(handle)
        for row in reader:
            rows.append(row)
            fieldnames.update(row.keys())

ordered = ["task", "status", "task_key"] + sorted(k for k in fieldnames if k not in {"task", "status", "task_key"})
with out_csv.open("w", newline="", encoding="utf-8") as handle:
    writer = csv.DictWriter(handle, fieldnames=ordered)
    writer.writeheader()
    for row in rows:
        writer.writerow({key: row.get(key, "") for key in ordered})

lines = [
    "# Linux not_done status",
    "",
    "## Status topology",
    "",
    "```mermaid",
    "flowchart TD",
    '    A["profile/status.csv"] --> C["merged status.csv"]',
    '    B["parser/status.csv"] --> C',
    '    C --> D["status.md"]',
    "```",
    "",
    "| Task | Status | Task Key |",
    "|---|---|---|",
]
for row in rows:
    lines.append(f"| {row.get('task', '')} | {row.get('status', '')} | {row.get('task_key', '')} |")
out_md.write_text("\n".join(lines) + "\n", encoding="utf-8")
PY

log "Step 4/4: Build summary"
"$PYTHON_BIN" - "$RELEASE_DIR" "$NOT_DONE_DIR" "$SUMMARY_FILE" "$SUMMARY_JSON" "$GATE_B_MODE" "$HOSTED_ARTIFACT_SOURCE" "$STRICT_ARTIFACT_SOURCE" <<'PY'
import csv
import json
import platform
import sys
from pathlib import Path

release_dir = Path(sys.argv[1])
not_done_dir = Path(sys.argv[2])
summary_file = Path(sys.argv[3])
summary_json = Path(sys.argv[4])
gate_b_mode = sys.argv[5]
hosted_artifact_source = sys.argv[6]
strict_artifact_source = sys.argv[7]

latest_csv = release_dir / "latest20" / "results_raw.csv"
compat_csv = release_dir / "compatible20" / "results_raw.csv"
status_csv = not_done_dir / "status.csv"
parser_speedup_csv = not_done_dir / "parser" / "parser_incompiler_parallel" / "speedup.csv"

def count_ok_fail(path: Path):
    ok = fail = 0
    if not path.exists():
        return ok, fail
    for row in csv.DictReader(path.open("r", encoding="utf-8")):
        if row.get("is_warmup") != "0":
            continue
        if row.get("ok") == "1":
            ok += 1
        else:
            fail += 1
    return ok, fail

latest_ok, latest_fail = count_ok_fail(latest_csv)
compat_ok, compat_fail = count_ok_fail(compat_csv)

profile_status = parser_incompiler_status = parser_performance_status = "missing"
profile_outcome = "missing"
profile_reason = ""
if status_csv.exists():
    for row in csv.DictReader(status_csv.open("r", encoding="utf-8")):
        key = row.get("task_key", "")
        if key == "dmd_profile_compare":
            profile_status = row.get("status", "unknown")
            profile_outcome = row.get("profiler_outcome", "") or "unknown"
            profile_reason = row.get("profiler_reason", "") or row.get("reason", "")
        if key == "parser_incompiler_parallel":
            parser_incompiler_status = row.get("status", "unknown")
            parser_performance_status = row.get("performance_status", "") or "unknown"

parser_threads_total = 0
parser_threads_with_success = 0
parser_threads_missing_success: list[str] = []
if parser_speedup_csv.exists():
    for row in csv.DictReader(parser_speedup_csv.open("r", encoding="utf-8")):
        parser_threads_total += 1
        thr = row.get("threads", "?")
        files = row.get("files", "?")
        try:
            successful = int((row.get("successful_runs", "") or "0").strip() or "0")
        except ValueError:
            successful = 0
        if successful > 0:
            parser_threads_with_success += 1
        else:
            parser_threads_missing_success.append(f"{files}x{thr}")

parser_thread_coverage_ok = parser_threads_total > 0 and parser_threads_with_success == parser_threads_total

if profile_status == "done":
    gate_b_result = "PASS"
elif gate_b_mode == "hosted_skip" and profile_outcome == "perf_unavailable":
    gate_b_result = "PASS"
else:
    gate_b_result = "FAIL"

if gate_b_mode == "hosted_skip" and profile_outcome == "perf_unavailable":
    gate_b_reason = "delegated to strict Linux artifact"
else:
    gate_b_reason = "-" if gate_b_result == "PASS" else (profile_reason or profile_outcome)

gate_a_pass = latest_ok > 0
gate_c_pass = parser_incompiler_status in {"done", "partial"}
gate_d_pass = parser_thread_coverage_ok
gate_e_pass = parser_performance_status == "done"
status = "pass" if (gate_a_pass and gate_b_result == "PASS" and gate_c_pass and gate_d_pass) else "fail"
notes = "Linux gap-close executed locally."
if gate_b_mode == "hosted_skip" and profile_outcome == "perf_unavailable":
    notes += f" Strict perf validation is delegated to {strict_artifact_source}."

lines = [
    "# Linux Gap-Close Summary",
    "",
    f"- Status: {status}",
    "- Execution mode: local",
    f"- Authoritative host: {platform.platform()}",
    f"- Artifact source: local:{summary_file.parent}",
    "- Network mode: offline",
    f"- Notes: {notes}",
    f"- Gate-B mode: {gate_b_mode}",
    f"- latest20 measured runs: ok={latest_ok} fail={latest_fail}",
    f"- compatible20 measured runs: ok={compat_ok} fail={compat_fail}",
    f"- dmd_profile_compare task status: {profile_status}",
    f"- dmd_profile_compare profiler outcome: {profile_outcome}",
    f"- parser_incompiler_parallel task status: {parser_incompiler_status}",
    f"- parser_incompiler_parallel performance status: {parser_performance_status}",
    "",
    "## Workflow topology",
    "",
    "```mermaid",
    "flowchart TD",
    '    A["bench_releases.sh\\nLinux latest20 + compatible20"] --> B["releases/report.md"]',
    '    C["not_done_experiments.py\\ndmd_profile_compare"] --> D["not_done_linux/profile/status.csv"]',
    '    E["not_done_experiments.py\\nparser_incompiler_parallel"] --> F["not_done_linux/parser/status.csv"]',
    '    D --> G["not_done_linux/status.md"]',
    '    F --> G',
    '    B --> H["summary.md\\nclosure gates"]',
    '    G --> H',
    "```",
    "",
    "## Closure gates",
    "",
    f"- Gate A (latest20 has successful Linux runs): {'PASS' if gate_a_pass else 'FAIL'}",
    f"- Gate B (dmd_profile_compare on Linux perf): {gate_b_result}",
    f"- Gate B reason: {gate_b_reason}",
    f"- Gate C (parser_incompiler_parallel executed cleanly): {'PASS' if gate_c_pass else 'FAIL'}",
    f"- Gate D (parser thread coverage: each configured thread has successful runs): {'PASS' if gate_d_pass else 'FAIL'}",
    f"- Gate E (parser speedup target met, advisory): {'PASS' if gate_e_pass else 'FAIL'}",
    f"- Parser threads missing success: {','.join(parser_threads_missing_success) if parser_threads_missing_success else '-'}",
    "",
    "## Key output paths",
    "",
    f"- Release analysis: `{release_dir / 'report.md'}`",
    f"- not_done status: `{not_done_dir / 'status.md'}`",
    f"- not_done raw status CSV: `{status_csv}`",
]
summary_file.write_text("\n".join(lines) + "\n", encoding="utf-8")
summary_json.write_text(
    json.dumps(
        {
            "status": status,
            "execution_mode": "local",
            "authoritative_host": platform.platform(),
            "artifact_source": f"local:{summary_file.parent}",
            "delegated_artifact_source": strict_artifact_source if gate_b_reason == "delegated to strict Linux artifact" else hosted_artifact_source,
            "network_mode": "offline",
            "notes": notes,
            "gate_b_mode": gate_b_mode,
            "latest20_ok_runs": latest_ok,
            "latest20_fail_runs": latest_fail,
            "compatible20_ok_runs": compat_ok,
            "compatible20_fail_runs": compat_fail,
            "profile_status": profile_status,
            "profile_outcome": profile_outcome,
            "profile_reason": profile_reason,
            "parser_incompiler_status": parser_incompiler_status,
            "parser_performance_status": parser_performance_status,
            "gate_a_result": "PASS" if gate_a_pass else "FAIL",
            "gate_b_result": gate_b_result,
            "gate_b_reason": gate_b_reason,
            "gate_c_result": "PASS" if gate_c_pass else "FAIL",
            "gate_d_result": "PASS" if gate_d_pass else "FAIL",
            "gate_e_result": "PASS" if gate_e_pass else "FAIL",
            "parser_threads_missing_success": parser_threads_missing_success,
        },
        indent=2,
        sort_keys=True,
    )
    + "\n",
    encoding="utf-8",
)
print(summary_file)

if status != "pass":
    raise SystemExit(3)
PY

log "Completed. Summary: $SUMMARY_FILE"