From 303493afc63b3d3c19a66fd11d30d23a129be758 Mon Sep 17 00:00:00 2001 From: Zakir Date: Sun, 22 Feb 2026 15:35:51 +0530 Subject: [PATCH 01/13] [CI][Compiler] Add PyInstaller binary build workflow for foryc (Phase 1 of #3292) --- .github/workflows/build-foryc-binaries.yml | 404 +++++++++++++++++++++ compiler/foryc.spec | 132 +++++++ compiler/requirements-dev.txt | 15 + 3 files changed, 551 insertions(+) create mode 100644 .github/workflows/build-foryc-binaries.yml create mode 100644 compiler/foryc.spec create mode 100644 compiler/requirements-dev.txt diff --git a/.github/workflows/build-foryc-binaries.yml b/.github/workflows/build-foryc-binaries.yml new file mode 100644 index 0000000000..bc9e967d3b --- /dev/null +++ b/.github/workflows/build-foryc-binaries.yml @@ -0,0 +1,404 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build foryc Binaries + +on: + workflow_dispatch: + + workflow_call: + outputs: + artifacts_ready: + description: "True if all 5 foryc binaries built and validated." + value: ${{ jobs.build-complete.outputs.ready }} + + push: + tags: ["v*"] + + pull_request: + paths: + - "compiler/**" + - ".github/workflows/build-foryc-binaries.yml" + +permissions: + contents: read + +env: + PYTHON_VERSION: "3.11" + +# ───────────────────────────────────────────────────────────────────────────── +# BUILD JOB +# Produces one standalone foryc binary per target platform. +# +# Runner notes (matched to repo convention in build-native-pr.yml): +# linux-x86_64 : ubuntu-22.04 +# linux-aarch64 : ubuntu-24.04-arm (native GitHub ARM64 runner) +# macos-x86_64 : macos-15-intel +# macos-aarch64 : macos-15 (native Apple Silicon) +# windows-x86_64: windows-latest +# +# UPX notes: +# All platforms get UPX --best --lzma compression. +# macOS aarch64 MUST be re-signed with codesign after UPX. +# Apple Silicon requires a valid code signature on all executables. +# Without re-signing, the binary runs fine in CI but silently fails +# for end users on macOS 12+ with SIP enabled. +# +# 10 MB constraint: +# Each binary must remain under 10 MB after UPX compression. +# This is a hard gate for crates.io embedding in Phase 2. +# Enforced with an explicit assertion step in each build job. +# ───────────────────────────────────────────────────────────────────────────── +jobs: + build: + name: build / ${{ matrix.target }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - target: linux-x86_64 + os: ubuntu-22.04 + artifact_name: foryc-linux-x86_64 + binary_path: compiler/dist/foryc + binary_name: foryc + use_upx: true + codesign: false + + - target: linux-aarch64 + os: ubuntu-24.04-arm + artifact_name: foryc-linux-aarch64 + binary_path: compiler/dist/foryc + binary_name: foryc + use_upx: true + codesign: false + + - target: macos-x86_64 + os: macos-15-intel + artifact_name: foryc-macos-x86_64 + binary_path: compiler/dist/foryc + binary_name: foryc + use_upx: true + codesign: false + + - target: macos-aarch64 + os: macos-15 + artifact_name: foryc-macos-aarch64 + binary_path: compiler/dist/foryc + binary_name: foryc + use_upx: true + codesign: true + + - target: windows-x86_64 + os: windows-latest + artifact_name: foryc-windows-x86_64 + binary_path: compiler/dist/foryc.exe + binary_name: foryc.exe + use_upx: true + codesign: false + + steps: + # Full checkout required: needs compiler/foryc.spec, + # compiler/requirements-dev.txt, and the full compiler/ package + # for pip install and pyinstaller to work. + - name: Checkout + uses: actions/checkout@v5 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + # ── Install UPX ───────────────────────────────────────────────────────── + - name: Install UPX (Linux) + if: runner.os == 'Linux' && matrix.use_upx + run: | + sudo apt-get update -qq + sudo apt-get install -y upx-ucl + upx --version + + - name: Install UPX (macOS) + if: runner.os == 'macOS' && matrix.use_upx + run: | + brew install upx + upx --version + + - name: Install UPX (Windows) + if: runner.os == 'Windows' && matrix.use_upx + shell: pwsh + run: | + choco install upx --yes --no-progress + upx --version + + # ── Install build dependencies ────────────────────────────────────────── + # pyinstaller version is pinned in requirements-dev.txt. + # fory_compiler is installed from source so PyInstaller's import + # tracer can walk the actual installed package tree. + - name: Install PyInstaller and fory_compiler + run: | + python -m pip install --upgrade pip + pip install -r compiler/requirements-dev.txt + pip install ./compiler + + # ── Build ─────────────────────────────────────────────────────────────── + # Must run from compiler/ so pathex=['.'] in foryc.spec resolves + # fory_compiler/__main__.py correctly. + - name: Build standalone binary with PyInstaller + working-directory: compiler + run: pyinstaller foryc.spec + + # ── Pre-compression smoke test ────────────────────────────────────────── + # Confirms the binary is functional before UPX touches it. + # A broken binary here gives a cleaner error than post-UPX. + - name: Smoke test (pre-UPX) + shell: bash + run: | + echo "=== Pre-UPX binary size ===" + python -c " + import os + p = '${{ matrix.binary_path }}' + s = os.path.getsize(p) + print(f'Size: {s:,} bytes ({s/1024/1024:.2f} MB)') + " + "${{ matrix.binary_path }}" --help + + # ── UPX compression ───────────────────────────────────────────────────── + # --best --lzma: maximum compression, ~10% better ratio than default. + # UPX on PyInstaller --onefile binaries is well-supported. + # The PyInstaller bootloader survives UPX compression intact. + - name: Compress with UPX + if: matrix.use_upx + shell: bash + run: | + upx --best --lzma "${{ matrix.binary_path }}" + echo "=== Post-UPX binary size ===" + python -c " + import os + p = '${{ matrix.binary_path }}' + s = os.path.getsize(p) + print(f'Size: {s:,} bytes ({s/1024/1024:.2f} MB)') + " + + # ── macOS aarch64: re-sign after UPX ──────────────────────────────────── + # UPX modifies the Mach-O binary, invalidating its code signature. + # Apple Silicon refuses to execute binaries with invalid signatures. + # --sign - creates an ad-hoc signature; no Apple Developer ID required. + # Ad-hoc signatures are sufficient for binaries distributed via crates.io + # since they are not quarantined (not downloaded from the internet at runtime). + - name: Re-sign binary after UPX (macOS aarch64 only) + if: matrix.codesign + run: | + codesign --force --deep --sign - "${{ matrix.binary_path }}" + codesign --verify --verbose "${{ matrix.binary_path }}" + + # ── Post-compression smoke test ───────────────────────────────────────── + # Critical test: the UPX-compressed (and re-signed) binary must execute. + # Failure here means UPX broke the binary on this platform. + - name: Smoke test (post-UPX) + shell: bash + run: | + "${{ matrix.binary_path }}" --help + + # ── crates.io 10 MB size gate ──────────────────────────────────────────── + # Each per-platform foryc-bin-* crate (Phase 2) embeds exactly one binary. + # crates.io hard limit is 10 MB per crate. + # If this assertion fails: add more entries to excludes[] in foryc.spec, + # investigate why binary grew, or reconsider distribution strategy. + - name: Assert binary is under 10 MB (crates.io hard limit) + shell: bash + run: | + python -c " + import os, sys + path = '${{ matrix.binary_path }}' + size = os.path.getsize(path) + limit = 10 * 1024 * 1024 + print(f'Final size: {size:,} bytes ({size/1024/1024:.2f} MB)') + print(f'Limit: {limit:,} bytes (10.00 MB)') + if size > limit: + print() + print('FAIL: Binary exceeds 10 MB crates.io per-crate limit.') + print('Add exclusions to compiler/foryc.spec or investigate UPX options.') + sys.exit(1) + print('PASS') + " + + # ── Upload artifact ───────────────────────────────────────────────────── + - name: Upload binary artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.artifact_name }} + path: ${{ matrix.binary_path }} + retention-days: 30 + if-no-files-found: error + + # ───────────────────────────────────────────────────────────────────────────── + # VALIDATE JOB + # Downloads each binary onto its native runner and runs: + # 1. --help sanity check + # Python is NOT set up in this job — binary must be fully self-contained. + # 2. End-to-end compile of compiler/examples/demo.fdl to Rust output. + # demo.fdl exercises: enum, optional, list, map, ref (cross-message + # reference using the 'ref' keyword), primitive arrays, type IDs, + # and name-based registration. It is the repo's own known-good fixture. + # ───────────────────────────────────────────────────────────────────────────── + validate: + name: validate / ${{ matrix.target }} + needs: build + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - target: linux-x86_64 + os: ubuntu-22.04 + artifact_name: foryc-linux-x86_64 + binary_name: foryc + + - target: linux-aarch64 + os: ubuntu-24.04-arm + artifact_name: foryc-linux-aarch64 + binary_name: foryc + + - target: macos-x86_64 + os: macos-15-intel + artifact_name: foryc-macos-x86_64 + binary_name: foryc + + - target: macos-aarch64 + os: macos-15 + artifact_name: foryc-macos-aarch64 + binary_name: foryc + + - target: windows-x86_64 + os: windows-latest + artifact_name: foryc-windows-x86_64 + binary_name: foryc.exe + + steps: + # Sparse checkout: only compiler/examples/ needed for demo.fdl. + # Python is deliberately NOT set up — the binary must run standalone. + - name: Checkout (sparse — compiler/examples only) + uses: actions/checkout@v5 + with: + sparse-checkout: compiler/examples + sparse-checkout-cone-mode: true + + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.artifact_name }} + path: ./artifact + + - name: Make executable (Unix) + if: runner.os != 'Windows' + run: chmod +x ./artifact/${{ matrix.binary_name }} + + # ── Test 1: --help ─────────────────────────────────────────────────────── + - name: Validate --help output + shell: bash + run: | + OUTPUT=$(./artifact/${{ matrix.binary_name }} --help 2>&1) + echo "$OUTPUT" + echo "$OUTPUT" | grep -iE "(fory|usage|compile|idl|lang)" || { + echo "ERROR: --help output missing expected keywords" + exit 1 + } + + # ── Test 2: End-to-end FDL → Rust ──────────────────────────────────────── + # Uses compiler/examples/demo.fdl from the sparse checkout. + # Correct invocation: --rust_out sets output dir; no --lang needed. + - name: End-to-end compile demo.fdl to Rust + shell: bash + run: | + OUT_DIR="${RUNNER_TEMP}/foryc-e2e-out" + mkdir -p "${OUT_DIR}" + + echo "=== Compiling compiler/examples/demo.fdl → Rust ===" + ./artifact/${{ matrix.binary_name }} \ + --rust_out "${OUT_DIR}" \ + compiler/examples/demo.fdl + + echo "=== Output files ===" + ls -la "${OUT_DIR}/" + + FILE_COUNT=$(ls "${OUT_DIR}" | wc -l) + if [ "${FILE_COUNT}" -eq 0 ]; then + echo "ERROR: foryc produced no output files" + exit 1 + fi + echo "PASS: ${FILE_COUNT} file(s) generated" + + # ───────────────────────────────────────────────────────────────────────────── + # SUMMARY JOB + # Single required status check for branch protection rules. + # The Phase 4 release pipeline reads this job's output via workflow_call. + # ───────────────────────────────────────────────────────────────────────────── + build-complete: + name: foryc / all binaries ready + needs: [build, validate] + runs-on: ubuntu-latest + if: always() + outputs: + ready: ${{ steps.check.outputs.ready }} + steps: + - name: Evaluate results + id: check + run: | + BUILD="${{ needs.build.result }}" + VALIDATE="${{ needs.validate.result }}" + echo "build: ${BUILD}" + echo "validate: ${VALIDATE}" + if [[ "${BUILD}" == "success" && "${VALIDATE}" == "success" ]]; then + echo "ready=true" >> "${GITHUB_OUTPUT}" + else + echo "ready=false" >> "${GITHUB_OUTPUT}" + exit 1 + fi + +# ───────────────────────────────────────────────────────────────────────────── +# FALLBACK: If ubuntu-24.04-arm runner is unavailable in your fork, +# replace the linux-aarch64 matrix entry with: +# +# - target: linux-aarch64 +# os: ubuntu-22.04 +# artifact_name: foryc-linux-aarch64 +# binary_path: compiler/dist/foryc +# binary_name: foryc +# use_upx: true +# codesign: false +# +# Then add these two steps before "Install PyInstaller": +# +# - name: Set up QEMU +# uses: docker/setup-qemu-action@v3 +# with: +# platforms: arm64 +# +# - name: Build via Docker (linux-aarch64) +# if: matrix.target == 'linux-aarch64' +# uses: addnab/docker-run-action@v3 +# with: +# image: python:3.11-slim-bookworm +# options: --platform linux/arm64 -v ${{ github.workspace }}:/ws +# run: | +# apt-get update -qq && apt-get install -y upx-ucl binutils +# pip install -r /ws/compiler/requirements-dev.txt +# pip install /ws/compiler +# cd /ws/compiler && pyinstaller foryc.spec +# +# QEMU builds are 10-20x slower than native. Expect 20-30 min per run. +# ───────────────────────────────────────────────────────────────────────────── diff --git a/compiler/foryc.spec b/compiler/foryc.spec new file mode 100644 index 0000000000..df4fa4fead --- /dev/null +++ b/compiler/foryc.spec @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# -*- mode: python ; coding: utf-8 -*- +# +# PyInstaller spec for building the standalone foryc binary. +# +# Run from the compiler/ directory: +# cd compiler && pyinstaller foryc.spec +# +# Output: compiler/dist/foryc (foryc.exe on Windows) +# +# UPX compression is intentionally NOT applied here. +# The CI workflow applies UPX manually per-platform for precise control +# (macOS aarch64 requires codesign after UPX; Windows needs a separate UPX path). + +a = Analysis( + ['fory_compiler/__main__.py'], + pathex=['.'], + binaries=[], + datas=[], + # fory_compiler has zero third-party pip dependencies. + # All frontends use hand-written lexers/parsers (pure stdlib). + # All generators are explicitly statically imported in generators/__init__.py. + # PyInstaller's import tracer catches everything automatically. + # This list is a complete belt-and-suspenders safety net — every module + # in the fory_compiler package is enumerated here to prevent any future + # refactoring from silently dropping a module from the binary. + hiddenimports=[ + # Entry point chain + 'fory_compiler', + 'fory_compiler.cli', + + # IR layer — all 5 modules + 'fory_compiler.ir', + 'fory_compiler.ir.ast', + 'fory_compiler.ir.emitter', + 'fory_compiler.ir.validator', + 'fory_compiler.ir.type_id', + 'fory_compiler.ir.types', + + # Frontend base utilities + 'fory_compiler.frontend', + 'fory_compiler.frontend.base', + 'fory_compiler.frontend.utils', + + # FDL frontend — hand-written lexer/parser + 'fory_compiler.frontend.fdl', + 'fory_compiler.frontend.fdl.lexer', + 'fory_compiler.frontend.fdl.parser', + + # Proto frontend — hand-written lexer/parser/translator + 'fory_compiler.frontend.proto', + 'fory_compiler.frontend.proto.ast', + 'fory_compiler.frontend.proto.lexer', + 'fory_compiler.frontend.proto.parser', + 'fory_compiler.frontend.proto.translator', + + # FBS (FlatBuffers schema) frontend — all 4 submodules + 'fory_compiler.frontend.fbs', + 'fory_compiler.frontend.fbs.ast', + 'fory_compiler.frontend.fbs.lexer', + 'fory_compiler.frontend.fbs.parser', + 'fory_compiler.frontend.fbs.translator', + + # Generators — all 5 language backends + 'fory_compiler.generators', + 'fory_compiler.generators.base', + 'fory_compiler.generators.java', + 'fory_compiler.generators.python', + 'fory_compiler.generators.cpp', + 'fory_compiler.generators.rust', + 'fory_compiler.generators.go', + ], + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + # foryc uses: argparse, copy, os, sys, pathlib, typing, dataclasses, + # enum, re, abc, io, collections, functools, itertools. + # Everything below is confirmed unused — aggressively excluded to + # reduce pre-UPX binary size. + excludes=[ + 'unittest', 'doctest', 'pdb', 'pydoc', 'py_compile', 'profile', + 'distutils', 'setuptools', 'pkg_resources', + 'email', 'html', 'http', 'urllib', 'xmlrpc', + 'xml.etree', 'xml.dom', 'xml.sax', + 'tkinter', '_tkinter', + 'curses', '_curses', 'readline', + 'dbm', 'sqlite3', '_sqlite3', + 'asyncio', 'concurrent', 'multiprocessing', + 'test', '_test', 'lib2to3', + ], + noarchive=False, +) + +pyz = PYZ(a.pure, a.zipped_data) + +exe = EXE( + pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + [], + name='foryc', + debug=False, + bootloader_ignore_signals=False, + strip=True, + upx=False, + upx_exclude=[], + runtime_tmpdir=None, + console=True, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, +) diff --git a/compiler/requirements-dev.txt b/compiler/requirements-dev.txt new file mode 100644 index 0000000000..648e25a071 --- /dev/null +++ b/compiler/requirements-dev.txt @@ -0,0 +1,15 @@ +# Development dependencies for building the standalone foryc binary. +# NOT part of the published fory-compiler package on PyPI. +# This file intentionally kept separate from pyproject.toml. +# +# Usage (from repo root): +# pip install -r compiler/requirements-dev.txt +# pip install ./compiler +# cd compiler && pyinstaller foryc.spec +# +# UPX must be installed separately as a system binary: +# Linux: sudo apt-get install upx-ucl +# macOS: brew install upx +# Windows: choco install upx + +pyinstaller>=6.0,<7.0 From cb22c41e3b73678b0f708629fd63d308c02992a1 Mon Sep 17 00:00:00 2001 From: Zakir Date: Sun, 22 Feb 2026 16:13:34 +0530 Subject: [PATCH 02/13] ci: fix urllib exclusion and pin windows runner to 2022 --- .github/workflows/build-foryc-binaries.yml | 4 ++-- compiler/foryc.spec | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-foryc-binaries.yml b/.github/workflows/build-foryc-binaries.yml index bc9e967d3b..b9fa5bf439 100644 --- a/.github/workflows/build-foryc-binaries.yml +++ b/.github/workflows/build-foryc-binaries.yml @@ -104,7 +104,7 @@ jobs: codesign: true - target: windows-x86_64 - os: windows-latest + os: windows-2022 artifact_name: foryc-windows-x86_64 binary_path: compiler/dist/foryc.exe binary_name: foryc.exe @@ -298,7 +298,7 @@ jobs: sparse-checkout-cone-mode: true - name: Download artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v5 with: name: ${{ matrix.artifact_name }} path: ./artifact diff --git a/compiler/foryc.spec b/compiler/foryc.spec index df4fa4fead..4ff6a5f749 100644 --- a/compiler/foryc.spec +++ b/compiler/foryc.spec @@ -96,7 +96,7 @@ a = Analysis( excludes=[ 'unittest', 'doctest', 'pdb', 'pydoc', 'py_compile', 'profile', 'distutils', 'setuptools', 'pkg_resources', - 'email', 'html', 'http', 'urllib', 'xmlrpc', + 'email', 'html', 'http', 'xmlrpc', 'xml.etree', 'xml.dom', 'xml.sax', 'tkinter', '_tkinter', 'curses', '_curses', 'readline', From e0acb4d5162a7f85a9096208724127bfdceb5d20 Mon Sep 17 00:00:00 2001 From: Zakir Date: Sun, 22 Feb 2026 16:29:16 +0530 Subject: [PATCH 03/13] ci: use Python 3.12 on Windows, skip broken UPX 5.x on macOS --- .github/workflows/build-foryc-binaries.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-foryc-binaries.yml b/.github/workflows/build-foryc-binaries.yml index b9fa5bf439..0baca9f9f3 100644 --- a/.github/workflows/build-foryc-binaries.yml +++ b/.github/workflows/build-foryc-binaries.yml @@ -38,7 +38,7 @@ permissions: contents: read env: - PYTHON_VERSION: "3.11" + PYTHON_VERSION: "3.12" # ───────────────────────────────────────────────────────────────────────────── # BUILD JOB @@ -92,7 +92,7 @@ jobs: artifact_name: foryc-macos-x86_64 binary_path: compiler/dist/foryc binary_name: foryc - use_upx: true + use_upx: false codesign: false - target: macos-aarch64 @@ -100,7 +100,7 @@ jobs: artifact_name: foryc-macos-aarch64 binary_path: compiler/dist/foryc binary_name: foryc - use_upx: true + use_upx: false codesign: true - target: windows-x86_64 From b6e9f6e59ee08d33a555fbd8aaaa09268d94fcc7 Mon Sep 17 00:00:00 2001 From: Zakir Date: Sun, 22 Feb 2026 16:40:49 +0530 Subject: [PATCH 04/13] ci: disable Windows Defender for PyInstaller, revert to Python 3.11, keep macOS UPX disabled --- .github/workflows/build-foryc-binaries.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-foryc-binaries.yml b/.github/workflows/build-foryc-binaries.yml index 0baca9f9f3..5f035624a4 100644 --- a/.github/workflows/build-foryc-binaries.yml +++ b/.github/workflows/build-foryc-binaries.yml @@ -154,6 +154,11 @@ jobs: pip install -r compiler/requirements-dev.txt pip install ./compiler + - name: Disable Windows Defender real-time monitoring (PyInstaller workaround) + if: runner.os == 'Windows' + shell: pwsh + run: Set-MpPreference -DisableRealtimeMonitoring $true + # ── Build ─────────────────────────────────────────────────────────────── # Must run from compiler/ so pathex=['.'] in foryc.spec resolves # fory_compiler/__main__.py correctly. From 1e6b16157deb6317fcc605a277e609cdc1bbf849 Mon Sep 17 00:00:00 2001 From: Zakir Date: Sun, 22 Feb 2026 16:49:29 +0530 Subject: [PATCH 05/13] ci: fix Windows DLL loading by setting runtime_tmpdir to cwd --- .github/workflows/build-foryc-binaries.yml | 2 +- compiler/foryc.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-foryc-binaries.yml b/.github/workflows/build-foryc-binaries.yml index 5f035624a4..3a71af1340 100644 --- a/.github/workflows/build-foryc-binaries.yml +++ b/.github/workflows/build-foryc-binaries.yml @@ -38,7 +38,7 @@ permissions: contents: read env: - PYTHON_VERSION: "3.12" + PYTHON_VERSION: "3.11" # ───────────────────────────────────────────────────────────────────────────── # BUILD JOB diff --git a/compiler/foryc.spec b/compiler/foryc.spec index 4ff6a5f749..0e49069486 100644 --- a/compiler/foryc.spec +++ b/compiler/foryc.spec @@ -122,7 +122,7 @@ exe = EXE( strip=True, upx=False, upx_exclude=[], - runtime_tmpdir=None, + runtime_tmpdir='.', console=True, disable_windowed_traceback=False, argv_emulation=False, From 668b6cd7854856f3d650b69e89276c46a384f465 Mon Sep 17 00:00:00 2001 From: Zakir Date: Sun, 22 Feb 2026 17:16:31 +0530 Subject: [PATCH 06/13] ci: try windows-2019 runner to avoid DLL security policy failure --- .github/workflows/build-foryc-binaries.yml | 2 +- compiler/foryc.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-foryc-binaries.yml b/.github/workflows/build-foryc-binaries.yml index 3a71af1340..de80a5a295 100644 --- a/.github/workflows/build-foryc-binaries.yml +++ b/.github/workflows/build-foryc-binaries.yml @@ -104,7 +104,7 @@ jobs: codesign: true - target: windows-x86_64 - os: windows-2022 + os: windows-2019 artifact_name: foryc-windows-x86_64 binary_path: compiler/dist/foryc.exe binary_name: foryc.exe diff --git a/compiler/foryc.spec b/compiler/foryc.spec index 0e49069486..4ff6a5f749 100644 --- a/compiler/foryc.spec +++ b/compiler/foryc.spec @@ -122,7 +122,7 @@ exe = EXE( strip=True, upx=False, upx_exclude=[], - runtime_tmpdir='.', + runtime_tmpdir=None, console=True, disable_windowed_traceback=False, argv_emulation=False, From c35576d3daa3dcf98b1e7dadc3e5e87c6a9fbad6 Mon Sep 17 00:00:00 2001 From: Zakir Date: Mon, 23 Feb 2026 11:58:57 +0530 Subject: [PATCH 07/13] =?UTF-8?q?ci:=20apply=20all=2010=20review=20fixes?= =?UTF-8?q?=20=E2=80=94=20quoting,=20validation,=20tag=20scope,=20pip=20ca?= =?UTF-8?q?che,=20defender=20in=20validate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build-foryc-binaries.yml | 134 +++++++++++++++++---- 1 file changed, 108 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build-foryc-binaries.yml b/.github/workflows/build-foryc-binaries.yml index de80a5a295..c7fa1de21f 100644 --- a/.github/workflows/build-foryc-binaries.yml +++ b/.github/workflows/build-foryc-binaries.yml @@ -27,7 +27,7 @@ on: value: ${{ jobs.build-complete.outputs.ready }} push: - tags: ["v*"] + tags: ["foryc-v*"] # FIX 7: Narrowed from "v*" to "foryc-v*" pull_request: paths: @@ -49,14 +49,12 @@ env: # linux-aarch64 : ubuntu-24.04-arm (native GitHub ARM64 runner) # macos-x86_64 : macos-15-intel # macos-aarch64 : macos-15 (native Apple Silicon) -# windows-x86_64: windows-latest +# windows-x86_64: windows-2019 # # UPX notes: -# All platforms get UPX --best --lzma compression. -# macOS aarch64 MUST be re-signed with codesign after UPX. -# Apple Silicon requires a valid code signature on all executables. -# Without re-signing, the binary runs fine in CI but silently fails -# for end users on macOS 12+ with SIP enabled. +# Linux and Windows targets use UPX --best --lzma compression. +# macOS targets skip UPX (UPX 4.x+ dropped macOS support entirely). +# macOS aarch64 requires ad-hoc codesigning for Apple Silicon. # # 10 MB constraint: # Each binary must remain under 10 MB after UPX compression. @@ -118,10 +116,15 @@ jobs: - name: Checkout uses: actions/checkout@v5 + # FIX 8: Added pip cache - name: Set up Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} + cache: "pip" + cache-dependency-path: | + compiler/requirements-dev.txt + compiler/pyproject.toml # ── Install UPX ───────────────────────────────────────────────────────── - name: Install UPX (Linux) @@ -154,10 +157,44 @@ jobs: pip install -r compiler/requirements-dev.txt pip install ./compiler + # FIX 9: Verify all hiddenimports are importable before PyInstaller runs + - name: Verify all hiddenimports are importable + working-directory: compiler + run: | + python -c " + mods = [ + 'fory_compiler', 'fory_compiler.cli', + 'fory_compiler.ir', 'fory_compiler.ir.ast', + 'fory_compiler.ir.emitter', 'fory_compiler.ir.validator', + 'fory_compiler.ir.type_id', 'fory_compiler.ir.types', + 'fory_compiler.frontend', 'fory_compiler.frontend.base', + 'fory_compiler.frontend.utils', + 'fory_compiler.frontend.fdl', 'fory_compiler.frontend.fdl.lexer', + 'fory_compiler.frontend.fdl.parser', + 'fory_compiler.frontend.proto', 'fory_compiler.frontend.proto.ast', + 'fory_compiler.frontend.proto.lexer', 'fory_compiler.frontend.proto.parser', + 'fory_compiler.frontend.proto.translator', + 'fory_compiler.frontend.fbs', 'fory_compiler.frontend.fbs.ast', + 'fory_compiler.frontend.fbs.lexer', 'fory_compiler.frontend.fbs.parser', + 'fory_compiler.frontend.fbs.translator', + 'fory_compiler.generators', 'fory_compiler.generators.base', + 'fory_compiler.generators.java', 'fory_compiler.generators.python', + 'fory_compiler.generators.cpp', 'fory_compiler.generators.rust', + 'fory_compiler.generators.go', + ] + for m in mods: + __import__(m) + print(f'OK: {m}') + print('All hiddenimports verified.') + " + - name: Disable Windows Defender real-time monitoring (PyInstaller workaround) if: runner.os == 'Windows' shell: pwsh - run: Set-MpPreference -DisableRealtimeMonitoring $true + run: | + Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE" + Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" + Add-MpPreference -ExclusionProcess "foryc.exe" # ── Build ─────────────────────────────────────────────────────────────── # Must run from compiler/ so pathex=['.'] in foryc.spec resolves @@ -169,42 +206,48 @@ jobs: # ── Pre-compression smoke test ────────────────────────────────────────── # Confirms the binary is functional before UPX touches it. # A broken binary here gives a cleaner error than post-UPX. + # FIX 3: Use env var for path safety - name: Smoke test (pre-UPX) shell: bash + env: + BINARY_PATH: ${{ matrix.binary_path }} run: | echo "=== Pre-UPX binary size ===" python -c " import os - p = '${{ matrix.binary_path }}' + p = os.environ['BINARY_PATH'] s = os.path.getsize(p) print(f'Size: {s:,} bytes ({s/1024/1024:.2f} MB)') " - "${{ matrix.binary_path }}" --help + "$BINARY_PATH" --help # ── UPX compression ───────────────────────────────────────────────────── # --best --lzma: maximum compression, ~10% better ratio than default. # UPX on PyInstaller --onefile binaries is well-supported. # The PyInstaller bootloader survives UPX compression intact. + # FIX 3: Use env var for path safety - name: Compress with UPX if: matrix.use_upx shell: bash + env: + BINARY_PATH: ${{ matrix.binary_path }} run: | - upx --best --lzma "${{ matrix.binary_path }}" + upx --best --lzma "$BINARY_PATH" echo "=== Post-UPX binary size ===" python -c " import os - p = '${{ matrix.binary_path }}' + p = os.environ['BINARY_PATH'] s = os.path.getsize(p) print(f'Size: {s:,} bytes ({s/1024/1024:.2f} MB)') " - # ── macOS aarch64: re-sign after UPX ──────────────────────────────────── - # UPX modifies the Mach-O binary, invalidating its code signature. - # Apple Silicon refuses to execute binaries with invalid signatures. + # FIX 2: Renamed step to reflect actual purpose + # ── macOS aarch64: ad-hoc codesign ─────────────────────────────────────── + # PyInstaller-built Apple Silicon binaries require valid code signatures. # --sign - creates an ad-hoc signature; no Apple Developer ID required. # Ad-hoc signatures are sufficient for binaries distributed via crates.io # since they are not quarantined (not downloaded from the internet at runtime). - - name: Re-sign binary after UPX (macOS aarch64 only) + - name: Ad-hoc codesign binary (macOS aarch64, required for Apple Silicon) if: matrix.codesign run: | codesign --force --deep --sign - "${{ matrix.binary_path }}" @@ -218,6 +261,7 @@ jobs: run: | "${{ matrix.binary_path }}" --help + # FIX 3: Use env var for path safety # ── crates.io 10 MB size gate ──────────────────────────────────────────── # Each per-platform foryc-bin-* crate (Phase 2) embeds exactly one binary. # crates.io hard limit is 10 MB per crate. @@ -225,10 +269,12 @@ jobs: # investigate why binary grew, or reconsider distribution strategy. - name: Assert binary is under 10 MB (crates.io hard limit) shell: bash + env: + BINARY_PATH: ${{ matrix.binary_path }} run: | python -c " import os, sys - path = '${{ matrix.binary_path }}' + path = os.environ['BINARY_PATH'] size = os.path.getsize(path) limit = 10 * 1024 * 1024 print(f'Final size: {size:,} bytes ({size/1024/1024:.2f} MB)') @@ -259,6 +305,8 @@ jobs: # demo.fdl exercises: enum, optional, list, map, ref (cross-message # reference using the 'ref' keyword), primitive arrays, type IDs, # and name-based registration. It is the repo's own known-good fixture. + # 3. End-to-end compile of compiler/examples/demo.fdl to Java output. + # Validates that all 5 language generators are correctly embedded. # ───────────────────────────────────────────────────────────────────────────── validate: name: validate / ${{ matrix.target }} @@ -289,7 +337,7 @@ jobs: binary_name: foryc - target: windows-x86_64 - os: windows-latest + os: windows-2022 artifact_name: foryc-windows-x86_64 binary_name: foryc.exe @@ -302,34 +350,45 @@ jobs: sparse-checkout: compiler/examples sparse-checkout-cone-mode: true + # FIX 1: Changed from v5 to v4 for compatibility with upload-artifact@v4 - name: Download artifact - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v4 with: name: ${{ matrix.artifact_name }} path: ./artifact + # NEW FIX 10: Disable Windows Defender in validate job (missing from original) + - name: Disable Windows Defender (required for PyInstaller onefile extraction) + if: runner.os == 'Windows' + shell: pwsh + run: | + Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" + Add-MpPreference -ExclusionProcess "foryc.exe" + - name: Make executable (Unix) if: runner.os != 'Windows' run: chmod +x ./artifact/${{ matrix.binary_name }} + # FIX 4: Tighter --help validation # ── Test 1: --help ─────────────────────────────────────────────────────── - name: Validate --help output shell: bash run: | OUTPUT=$(./artifact/${{ matrix.binary_name }} --help 2>&1) echo "$OUTPUT" - echo "$OUTPUT" | grep -iE "(fory|usage|compile|idl|lang)" || { - echo "ERROR: --help output missing expected keywords" + echo "$OUTPUT" | grep -qiE "^usage:.*foryc" || { + echo "ERROR: --help missing expected usage line" exit 1 } + # FIX 5: Check for .rs files, not just file count # ── Test 2: End-to-end FDL → Rust ──────────────────────────────────────── # Uses compiler/examples/demo.fdl from the sparse checkout. # Correct invocation: --rust_out sets output dir; no --lang needed. - name: End-to-end compile demo.fdl to Rust shell: bash run: | - OUT_DIR="${RUNNER_TEMP}/foryc-e2e-out" + OUT_DIR="${RUNNER_TEMP}/foryc-e2e-rust" mkdir -p "${OUT_DIR}" echo "=== Compiling compiler/examples/demo.fdl → Rust ===" @@ -340,12 +399,35 @@ jobs: echo "=== Output files ===" ls -la "${OUT_DIR}/" - FILE_COUNT=$(ls "${OUT_DIR}" | wc -l) - if [ "${FILE_COUNT}" -eq 0 ]; then - echo "ERROR: foryc produced no output files" + RS_COUNT=$(find "${OUT_DIR}" -name "*.rs" -size +0c | wc -l) + if [ "${RS_COUNT}" -eq 0 ]; then + echo "ERROR: no non-empty .rs files generated" + exit 1 + fi + echo "PASS: ${RS_COUNT} .rs file(s) generated" + + # FIX 6: Add Java generator validation + # ── Test 3: End-to-end FDL → Java ───────────────────────────────────────── + - name: End-to-end compile demo.fdl to Java + shell: bash + run: | + JAVA_OUT="${RUNNER_TEMP}/foryc-e2e-java" + mkdir -p "${JAVA_OUT}" + + echo "=== Compiling compiler/examples/demo.fdl → Java ===" + ./artifact/${{ matrix.binary_name }} \ + --java_out "${JAVA_OUT}" \ + compiler/examples/demo.fdl + + echo "=== Output files ===" + ls -la "${JAVA_OUT}/" + + JAVA_COUNT=$(find "${JAVA_OUT}" -name "*.java" -size +0c | wc -l) + if [ "${JAVA_COUNT}" -eq 0 ]; then + echo "ERROR: no non-empty .java files generated" exit 1 fi - echo "PASS: ${FILE_COUNT} file(s) generated" + echo "PASS: ${JAVA_COUNT} .java file(s) generated" # ───────────────────────────────────────────────────────────────────────────── # SUMMARY JOB From 624dcd396660693f9db7ee8e2de200004ebbd74b Mon Sep 17 00:00:00 2001 From: Zakir Date: Mon, 23 Feb 2026 12:43:45 +0530 Subject: [PATCH 08/13] ci: apply all 10 review fixes + windows-latest runner --- .github/workflows/build-foryc-binaries.yml | 91 ++++++++++++++-------- 1 file changed, 59 insertions(+), 32 deletions(-) diff --git a/.github/workflows/build-foryc-binaries.yml b/.github/workflows/build-foryc-binaries.yml index c7fa1de21f..069c0a2336 100644 --- a/.github/workflows/build-foryc-binaries.yml +++ b/.github/workflows/build-foryc-binaries.yml @@ -26,8 +26,9 @@ on: description: "True if all 5 foryc binaries built and validated." value: ${{ jobs.build-complete.outputs.ready }} + # FIX 7: Narrowed from "v*" to "foryc-v*" to avoid triggering on Java/Rust/Go release tags push: - tags: ["foryc-v*"] # FIX 7: Narrowed from "v*" to "foryc-v*" + tags: ["foryc-v*"] pull_request: paths: @@ -49,17 +50,19 @@ env: # linux-aarch64 : ubuntu-24.04-arm (native GitHub ARM64 runner) # macos-x86_64 : macos-15-intel # macos-aarch64 : macos-15 (native Apple Silicon) -# windows-x86_64: windows-2019 +# windows-x86_64: windows-latest (Server 2022) # # UPX notes: # Linux and Windows targets use UPX --best --lzma compression. -# macOS targets skip UPX (UPX 4.x+ dropped macOS support entirely). +# macOS targets skip UPX entirely — UPX 4.x+ dropped macOS Mach-O support. # macOS aarch64 requires ad-hoc codesigning for Apple Silicon. # # 10 MB constraint: # Each binary must remain under 10 MB after UPX compression. # This is a hard gate for crates.io embedding in Phase 2. # Enforced with an explicit assertion step in each build job. +# macOS binaries are uncompressed but remain under 10 MB because +# fory_compiler has zero third-party pip dependencies (pure stdlib only). # ───────────────────────────────────────────────────────────────────────────── jobs: build: @@ -101,8 +104,12 @@ jobs: use_upx: false codesign: true + # LATEST FIX: windows-2019 is deprecated and runners no longer provision. + # windows-latest (Server 2022) works correctly when combined with the + # Defender exclusion step below. The DLL issue was always caused by + # Defender scanning PyInstaller's extracted temp files, not the runner version. - target: windows-x86_64 - os: windows-2019 + os: windows-latest artifact_name: foryc-windows-x86_64 binary_path: compiler/dist/foryc.exe binary_name: foryc.exe @@ -116,7 +123,7 @@ jobs: - name: Checkout uses: actions/checkout@v5 - # FIX 8: Added pip cache + # FIX 8: Added pip cache to avoid cold installs on every run (~60-120s saved per target) - name: Set up Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@v5 with: @@ -157,7 +164,10 @@ jobs: pip install -r compiler/requirements-dev.txt pip install ./compiler - # FIX 9: Verify all hiddenimports are importable before PyInstaller runs + # FIX 9: Verify all hiddenimports are importable before PyInstaller runs. + # PyInstaller silently ignores nonexistent hiddenimports — this step + # catches renamed or removed modules before they silently disappear + # from the binary. - name: Verify all hiddenimports are importable working-directory: compiler run: | @@ -188,7 +198,11 @@ jobs: print('All hiddenimports verified.') " - - name: Disable Windows Defender real-time monitoring (PyInstaller workaround) + # LATEST FIX (build job): Defender exclusions instead of full disable. + # Targets GITHUB_WORKSPACE and RUNNER_TEMP — the two paths PyInstaller + # writes to during build and extraction. This is sufficient and more + # targeted than disabling real-time monitoring globally. + - name: Disable Windows Defender for PyInstaller paths if: runner.os == 'Windows' shell: pwsh run: | @@ -203,10 +217,11 @@ jobs: working-directory: compiler run: pyinstaller foryc.spec + # FIX 3 (occurrence 1/3): Use BINARY_PATH env var instead of inline matrix + # expression inside Python string — safe against paths with spaces or quotes. # ── Pre-compression smoke test ────────────────────────────────────────── # Confirms the binary is functional before UPX touches it. # A broken binary here gives a cleaner error than post-UPX. - # FIX 3: Use env var for path safety - name: Smoke test (pre-UPX) shell: bash env: @@ -221,11 +236,11 @@ jobs: " "$BINARY_PATH" --help + # FIX 3 (occurrence 2/3): Same env var fix. # ── UPX compression ───────────────────────────────────────────────────── # --best --lzma: maximum compression, ~10% better ratio than default. - # UPX on PyInstaller --onefile binaries is well-supported. - # The PyInstaller bootloader survives UPX compression intact. - # FIX 3: Use env var for path safety + # UPX on PyInstaller --onefile binaries is well-supported on Linux/Windows. + # macOS is excluded entirely (UPX 4.x+ dropped Mach-O support). - name: Compress with UPX if: matrix.use_upx shell: bash @@ -241,12 +256,10 @@ jobs: print(f'Size: {s:,} bytes ({s/1024/1024:.2f} MB)') " - # FIX 2: Renamed step to reflect actual purpose - # ── macOS aarch64: ad-hoc codesign ─────────────────────────────────────── - # PyInstaller-built Apple Silicon binaries require valid code signatures. - # --sign - creates an ad-hoc signature; no Apple Developer ID required. - # Ad-hoc signatures are sufficient for binaries distributed via crates.io - # since they are not quarantined (not downloaded from the internet at runtime). + # FIX 2: Renamed step — UPX is disabled for macOS targets so "after UPX" + # was misleading. PyInstaller-built Apple Silicon binaries require a valid + # code signature regardless of UPX. Ad-hoc signing requires no Developer ID + # and is sufficient for binaries distributed via crates.io (not quarantined). - name: Ad-hoc codesign binary (macOS aarch64, required for Apple Silicon) if: matrix.codesign run: | @@ -261,7 +274,7 @@ jobs: run: | "${{ matrix.binary_path }}" --help - # FIX 3: Use env var for path safety + # FIX 3 (occurrence 3/3): Same env var fix. # ── crates.io 10 MB size gate ──────────────────────────────────────────── # Each per-platform foryc-bin-* crate (Phase 2) embeds exactly one binary. # crates.io hard limit is 10 MB per crate. @@ -302,11 +315,8 @@ jobs: # 1. --help sanity check # Python is NOT set up in this job — binary must be fully self-contained. # 2. End-to-end compile of compiler/examples/demo.fdl to Rust output. - # demo.fdl exercises: enum, optional, list, map, ref (cross-message - # reference using the 'ref' keyword), primitive arrays, type IDs, - # and name-based registration. It is the repo's own known-good fixture. # 3. End-to-end compile of compiler/examples/demo.fdl to Java output. - # Validates that all 5 language generators are correctly embedded. + # Validates that all 5 language generators are correctly embedded in binary. # ───────────────────────────────────────────────────────────────────────────── validate: name: validate / ${{ matrix.target }} @@ -336,8 +346,14 @@ jobs: artifact_name: foryc-macos-aarch64 binary_name: foryc + # windows-latest (Server 2022) — intentionally different from + # build runner (windows-2019 was deprecated). Binary built on + # Server 2022 and validated on Server 2022 for consistency. + # Defender exclusions below are required for PyInstaller onefile + # extraction — without them, DLL load fails as Defender quarantines + # files extracted to RUNNER_TEMP mid-execution. - target: windows-x86_64 - os: windows-2022 + os: windows-latest artifact_name: foryc-windows-x86_64 binary_name: foryc.exe @@ -350,15 +366,23 @@ jobs: sparse-checkout: compiler/examples sparse-checkout-cone-mode: true - # FIX 1: Changed from v5 to v4 for compatibility with upload-artifact@v4 + # FIX 1: Changed from download-artifact@v5 to @v4. + # upload-artifact@v4 and download-artifact@v5 use incompatible artifact + # storage APIs — mismatched versions cause artifact-not-found failures. - name: Download artifact uses: actions/download-artifact@v4 with: name: ${{ matrix.artifact_name }} path: ./artifact - # NEW FIX 10: Disable Windows Defender in validate job (missing from original) - - name: Disable Windows Defender (required for PyInstaller onefile extraction) + # FIX 10 (NEW): Defender exclusions in validate job. + # The original PR had Defender disable only in the build job. + # PyInstaller onefile binaries extract their payload to RUNNER_TEMP on + # first run. Without Defender exclusions, real-time scanning quarantines + # the extracted DLLs mid-execution, causing DLL load failed errors. + # This was the actual root cause of the "windows DLL failure" — not the + # runner version. + - name: Disable Windows Defender for PyInstaller extraction paths if: runner.os == 'Windows' shell: pwsh run: | @@ -369,7 +393,8 @@ jobs: if: runner.os != 'Windows' run: chmod +x ./artifact/${{ matrix.binary_name }} - # FIX 4: Tighter --help validation + # FIX 4: Tightened from broad keyword grep to specific usage line check. + # Previous pattern matched "fory" anywhere including crash tracebacks. # ── Test 1: --help ─────────────────────────────────────────────────────── - name: Validate --help output shell: bash @@ -381,10 +406,9 @@ jobs: exit 1 } - # FIX 5: Check for .rs files, not just file count + # FIX 5: Changed from file count check to non-empty .rs file check. + # Previous check passed if foryc produced a 0-byte file or an error log. # ── Test 2: End-to-end FDL → Rust ──────────────────────────────────────── - # Uses compiler/examples/demo.fdl from the sparse checkout. - # Correct invocation: --rust_out sets output dir; no --lang needed. - name: End-to-end compile demo.fdl to Rust shell: bash run: | @@ -406,8 +430,11 @@ jobs: fi echo "PASS: ${RS_COUNT} .rs file(s) generated" - # FIX 6: Add Java generator validation - # ── Test 3: End-to-end FDL → Java ───────────────────────────────────────── + # FIX 6: Added Java generator validation. + # The foryc.spec embeds all 5 language backends (java, python, cpp, rust, go). + # Without this test, a silent module drop in any non-Rust generator would + # be invisible and the broken binary would be promoted to Phase 2 crates. + # ── Test 3: End-to-end FDL → Java ──────────────────────────────────────── - name: End-to-end compile demo.fdl to Java shell: bash run: | From 54fe2737c21f02be7a93a2690a593f78e0d60962 Mon Sep 17 00:00:00 2001 From: Zakir Date: Mon, 23 Feb 2026 13:13:37 +0530 Subject: [PATCH 09/13] fix ci --- .github/workflows/build-foryc-binaries.yml | 112 +++++++-------------- 1 file changed, 35 insertions(+), 77 deletions(-) diff --git a/.github/workflows/build-foryc-binaries.yml b/.github/workflows/build-foryc-binaries.yml index 069c0a2336..19521a620c 100644 --- a/.github/workflows/build-foryc-binaries.yml +++ b/.github/workflows/build-foryc-binaries.yml @@ -26,7 +26,6 @@ on: description: "True if all 5 foryc binaries built and validated." value: ${{ jobs.build-complete.outputs.ready }} - # FIX 7: Narrowed from "v*" to "foryc-v*" to avoid triggering on Java/Rust/Go release tags push: tags: ["foryc-v*"] @@ -60,9 +59,15 @@ env: # 10 MB constraint: # Each binary must remain under 10 MB after UPX compression. # This is a hard gate for crates.io embedding in Phase 2. -# Enforced with an explicit assertion step in each build job. # macOS binaries are uncompressed but remain under 10 MB because # fory_compiler has zero third-party pip dependencies (pure stdlib only). +# +# Windows Defender note: +# Set-MpPreference -DisableRealtimeMonitoring $true is required in BOTH +# the build and validate jobs. Add-MpPreference -ExclusionPath is NOT +# sufficient — it only excludes file system scans, not the memory-level +# LoadLibrary interception that causes PYI-3104 / ERROR_NOACCESS failures +# when PyInstaller extracts and maps python311.dll at runtime. # ───────────────────────────────────────────────────────────────────────────── jobs: build: @@ -104,10 +109,6 @@ jobs: use_upx: false codesign: true - # LATEST FIX: windows-2019 is deprecated and runners no longer provision. - # windows-latest (Server 2022) works correctly when combined with the - # Defender exclusion step below. The DLL issue was always caused by - # Defender scanning PyInstaller's extracted temp files, not the runner version. - target: windows-x86_64 os: windows-latest artifact_name: foryc-windows-x86_64 @@ -117,13 +118,9 @@ jobs: codesign: false steps: - # Full checkout required: needs compiler/foryc.spec, - # compiler/requirements-dev.txt, and the full compiler/ package - # for pip install and pyinstaller to work. - name: Checkout uses: actions/checkout@v5 - # FIX 8: Added pip cache to avoid cold installs on every run (~60-120s saved per target) - name: Set up Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@v5 with: @@ -155,19 +152,16 @@ jobs: upx --version # ── Install build dependencies ────────────────────────────────────────── - # pyinstaller version is pinned in requirements-dev.txt. - # fory_compiler is installed from source so PyInstaller's import - # tracer can walk the actual installed package tree. - name: Install PyInstaller and fory_compiler run: | python -m pip install --upgrade pip pip install -r compiler/requirements-dev.txt pip install ./compiler - # FIX 9: Verify all hiddenimports are importable before PyInstaller runs. - # PyInstaller silently ignores nonexistent hiddenimports — this step - # catches renamed or removed modules before they silently disappear - # from the binary. + # ── Verify all hiddenimports are importable ───────────────────────────── + # PyInstaller silently ignores nonexistent hiddenimports entries. + # This step catches renamed or removed modules before they silently + # disappear from the binary. - name: Verify all hiddenimports are importable working-directory: compiler run: | @@ -198,30 +192,24 @@ jobs: print('All hiddenimports verified.') " - # LATEST FIX (build job): Defender exclusions instead of full disable. - # Targets GITHUB_WORKSPACE and RUNNER_TEMP — the two paths PyInstaller - # writes to during build and extraction. This is sufficient and more - # targeted than disabling real-time monitoring globally. - - name: Disable Windows Defender for PyInstaller paths + # ── Windows Defender (build job) ──────────────────────────────────────── + # Full real-time monitoring disable is required — not just path exclusions. + # Add-MpPreference -ExclusionPath only covers filesystem scans. + # The PYI-3104 / LoadLibrary: Invalid access to memory location error + # is caused by Defender intercepting the memory-mapping of python311.dll + # during PyInstaller bootloader execution. Only DisableRealtimeMonitoring + # stops that interception. + - name: Disable Windows Defender real-time monitoring (PyInstaller workaround) if: runner.os == 'Windows' shell: pwsh - run: | - Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE" - Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" - Add-MpPreference -ExclusionProcess "foryc.exe" + run: Set-MpPreference -DisableRealtimeMonitoring $true # ── Build ─────────────────────────────────────────────────────────────── - # Must run from compiler/ so pathex=['.'] in foryc.spec resolves - # fory_compiler/__main__.py correctly. - name: Build standalone binary with PyInstaller working-directory: compiler run: pyinstaller foryc.spec - # FIX 3 (occurrence 1/3): Use BINARY_PATH env var instead of inline matrix - # expression inside Python string — safe against paths with spaces or quotes. # ── Pre-compression smoke test ────────────────────────────────────────── - # Confirms the binary is functional before UPX touches it. - # A broken binary here gives a cleaner error than post-UPX. - name: Smoke test (pre-UPX) shell: bash env: @@ -236,11 +224,7 @@ jobs: " "$BINARY_PATH" --help - # FIX 3 (occurrence 2/3): Same env var fix. # ── UPX compression ───────────────────────────────────────────────────── - # --best --lzma: maximum compression, ~10% better ratio than default. - # UPX on PyInstaller --onefile binaries is well-supported on Linux/Windows. - # macOS is excluded entirely (UPX 4.x+ dropped Mach-O support). - name: Compress with UPX if: matrix.use_upx shell: bash @@ -256,10 +240,10 @@ jobs: print(f'Size: {s:,} bytes ({s/1024/1024:.2f} MB)') " - # FIX 2: Renamed step — UPX is disabled for macOS targets so "after UPX" - # was misleading. PyInstaller-built Apple Silicon binaries require a valid - # code signature regardless of UPX. Ad-hoc signing requires no Developer ID - # and is sufficient for binaries distributed via crates.io (not quarantined). + # ── macOS aarch64: ad-hoc codesign ─────────────────────────────────────── + # PyInstaller-built Apple Silicon binaries require valid code signatures. + # UPX is disabled for macOS — this step runs regardless of UPX. + # --sign - creates an ad-hoc signature; no Apple Developer ID required. - name: Ad-hoc codesign binary (macOS aarch64, required for Apple Silicon) if: matrix.codesign run: | @@ -267,19 +251,12 @@ jobs: codesign --verify --verbose "${{ matrix.binary_path }}" # ── Post-compression smoke test ───────────────────────────────────────── - # Critical test: the UPX-compressed (and re-signed) binary must execute. - # Failure here means UPX broke the binary on this platform. - name: Smoke test (post-UPX) shell: bash run: | "${{ matrix.binary_path }}" --help - # FIX 3 (occurrence 3/3): Same env var fix. # ── crates.io 10 MB size gate ──────────────────────────────────────────── - # Each per-platform foryc-bin-* crate (Phase 2) embeds exactly one binary. - # crates.io hard limit is 10 MB per crate. - # If this assertion fails: add more entries to excludes[] in foryc.spec, - # investigate why binary grew, or reconsider distribution strategy. - name: Assert binary is under 10 MB (crates.io hard limit) shell: bash env: @@ -316,7 +293,7 @@ jobs: # Python is NOT set up in this job — binary must be fully self-contained. # 2. End-to-end compile of compiler/examples/demo.fdl to Rust output. # 3. End-to-end compile of compiler/examples/demo.fdl to Java output. - # Validates that all 5 language generators are correctly embedded in binary. + # Validates that all 5 language generators are correctly embedded. # ───────────────────────────────────────────────────────────────────────────── validate: name: validate / ${{ matrix.target }} @@ -346,12 +323,6 @@ jobs: artifact_name: foryc-macos-aarch64 binary_name: foryc - # windows-latest (Server 2022) — intentionally different from - # build runner (windows-2019 was deprecated). Binary built on - # Server 2022 and validated on Server 2022 for consistency. - # Defender exclusions below are required for PyInstaller onefile - # extraction — without them, DLL load fails as Defender quarantines - # files extracted to RUNNER_TEMP mid-execution. - target: windows-x86_64 os: windows-latest artifact_name: foryc-windows-x86_64 @@ -366,35 +337,28 @@ jobs: sparse-checkout: compiler/examples sparse-checkout-cone-mode: true - # FIX 1: Changed from download-artifact@v5 to @v4. - # upload-artifact@v4 and download-artifact@v5 use incompatible artifact - # storage APIs — mismatched versions cause artifact-not-found failures. - name: Download artifact uses: actions/download-artifact@v4 with: name: ${{ matrix.artifact_name }} path: ./artifact - # FIX 10 (NEW): Defender exclusions in validate job. - # The original PR had Defender disable only in the build job. - # PyInstaller onefile binaries extract their payload to RUNNER_TEMP on - # first run. Without Defender exclusions, real-time scanning quarantines - # the extracted DLLs mid-execution, causing DLL load failed errors. - # This was the actual root cause of the "windows DLL failure" — not the - # runner version. - - name: Disable Windows Defender for PyInstaller extraction paths + # ── Windows Defender (validate job) ──────────────────────────────────── + # MUST appear before the binary is executed — not just before the build. + # PyInstaller onefile extracts python311.dll to RUNNER_TEMP on first run. + # Defender intercepts the LoadLibrary call at the memory level, causing + # PYI-3104: Invalid access to memory location. + # Path exclusions (Add-MpPreference) do NOT fix this — full real-time + # monitoring disable is required in this job exactly as in the build job. + - name: Disable Windows Defender real-time monitoring (PyInstaller extraction workaround) if: runner.os == 'Windows' shell: pwsh - run: | - Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" - Add-MpPreference -ExclusionProcess "foryc.exe" + run: Set-MpPreference -DisableRealtimeMonitoring $true - name: Make executable (Unix) if: runner.os != 'Windows' run: chmod +x ./artifact/${{ matrix.binary_name }} - # FIX 4: Tightened from broad keyword grep to specific usage line check. - # Previous pattern matched "fory" anywhere including crash tracebacks. # ── Test 1: --help ─────────────────────────────────────────────────────── - name: Validate --help output shell: bash @@ -406,8 +370,6 @@ jobs: exit 1 } - # FIX 5: Changed from file count check to non-empty .rs file check. - # Previous check passed if foryc produced a 0-byte file or an error log. # ── Test 2: End-to-end FDL → Rust ──────────────────────────────────────── - name: End-to-end compile demo.fdl to Rust shell: bash @@ -430,11 +392,7 @@ jobs: fi echo "PASS: ${RS_COUNT} .rs file(s) generated" - # FIX 6: Added Java generator validation. - # The foryc.spec embeds all 5 language backends (java, python, cpp, rust, go). - # Without this test, a silent module drop in any non-Rust generator would - # be invisible and the broken binary would be promoted to Phase 2 crates. - # ── Test 3: End-to-end FDL → Java ──────────────────────────────────────── + # ── Test 3: End-to-end FDL → Java ───────────────────────────────────────── - name: End-to-end compile demo.fdl to Java shell: bash run: | From 28602fb57b1d7c997bbcba92da4d6722ae6f974a Mon Sep 17 00:00:00 2001 From: Zakir Date: Mon, 23 Feb 2026 21:50:22 +0530 Subject: [PATCH 10/13] fix ci --- .github/workflows/build-foryc-binaries.yml | 156 ++++++++++----------- compiler/foryc.spec | 40 ++++-- 2 files changed, 103 insertions(+), 93 deletions(-) diff --git a/.github/workflows/build-foryc-binaries.yml b/.github/workflows/build-foryc-binaries.yml index 19521a620c..98262c0ee6 100644 --- a/.github/workflows/build-foryc-binaries.yml +++ b/.github/workflows/build-foryc-binaries.yml @@ -42,32 +42,35 @@ env: # ───────────────────────────────────────────────────────────────────────────── # BUILD JOB -# Produces one standalone foryc binary per target platform. +# Produces one standalone foryc binary directory per target platform. # -# Runner notes (matched to repo convention in build-native-pr.yml): +# Runner notes: # linux-x86_64 : ubuntu-22.04 -# linux-aarch64 : ubuntu-24.04-arm (native GitHub ARM64 runner) +# linux-aarch64 : ubuntu-24.04-arm # macos-x86_64 : macos-15-intel -# macos-aarch64 : macos-15 (native Apple Silicon) -# windows-x86_64: windows-latest (Server 2022) +# macos-aarch64 : macos-15 +# windows-x86_64: windows-latest (Server 2022) # -# UPX notes: -# Linux and Windows targets use UPX --best --lzma compression. -# macOS targets skip UPX entirely — UPX 4.x+ dropped macOS Mach-O support. -# macOS aarch64 requires ad-hoc codesigning for Apple Silicon. +# --onedir mode: +# PyInstaller --onefile extracts DLLs to %TEMP% at runtime. Windows Defender +# on GitHub's hardened runners intercepts LoadLibrary at the memory-mapping +# level (PYI-xxxx ERROR_NOACCESS) even with DisableRealtimeMonitoring $true. +# --onedir pre-extracts everything at build time. No runtime extraction means +# no Defender interception. No Defender workaround needed anywhere. +# +# Output per target: compiler/dist/foryc/ directory +# Unix: compiler/dist/foryc/foryc +# Windows: compiler/dist/foryc/foryc.exe # -# 10 MB constraint: -# Each binary must remain under 10 MB after UPX compression. -# This is a hard gate for crates.io embedding in Phase 2. -# macOS binaries are uncompressed but remain under 10 MB because -# fory_compiler has zero third-party pip dependencies (pure stdlib only). +# UPX notes: +# UPX is applied to the executable inside dist/foryc/ only, not the dir. +# macOS skips UPX — UPX 4.x+ dropped Mach-O support entirely. # -# Windows Defender note: -# Set-MpPreference -DisableRealtimeMonitoring $true is required in BOTH -# the build and validate jobs. Add-MpPreference -ExclusionPath is NOT -# sufficient — it only excludes file system scans, not the memory-level -# LoadLibrary interception that causes PYI-3104 / ERROR_NOACCESS failures -# when PyInstaller extracts and maps python311.dll at runtime. +# Phase 2 distribution note: +# The onedir package is 20-40 MB total (python311.dll + stdlib .pyc files). +# This exceeds the 10 MB crates.io per-crate limit from --onefile planning. +# foryc-bin will distribute via zip archive or download-on-first-install, +# not direct binary embedding in a single crate. # ───────────────────────────────────────────────────────────────────────────── jobs: build: @@ -80,7 +83,7 @@ jobs: - target: linux-x86_64 os: ubuntu-22.04 artifact_name: foryc-linux-x86_64 - binary_path: compiler/dist/foryc + binary_path: compiler/dist/foryc/foryc binary_name: foryc use_upx: true codesign: false @@ -88,7 +91,7 @@ jobs: - target: linux-aarch64 os: ubuntu-24.04-arm artifact_name: foryc-linux-aarch64 - binary_path: compiler/dist/foryc + binary_path: compiler/dist/foryc/foryc binary_name: foryc use_upx: true codesign: false @@ -96,7 +99,7 @@ jobs: - target: macos-x86_64 os: macos-15-intel artifact_name: foryc-macos-x86_64 - binary_path: compiler/dist/foryc + binary_path: compiler/dist/foryc/foryc binary_name: foryc use_upx: false codesign: false @@ -104,7 +107,7 @@ jobs: - target: macos-aarch64 os: macos-15 artifact_name: foryc-macos-aarch64 - binary_path: compiler/dist/foryc + binary_path: compiler/dist/foryc/foryc binary_name: foryc use_upx: false codesign: true @@ -112,7 +115,7 @@ jobs: - target: windows-x86_64 os: windows-latest artifact_name: foryc-windows-x86_64 - binary_path: compiler/dist/foryc.exe + binary_path: compiler/dist/foryc/foryc.exe binary_name: foryc.exe use_upx: true codesign: false @@ -138,12 +141,6 @@ jobs: sudo apt-get install -y upx-ucl upx --version - - name: Install UPX (macOS) - if: runner.os == 'macOS' && matrix.use_upx - run: | - brew install upx - upx --version - - name: Install UPX (Windows) if: runner.os == 'Windows' && matrix.use_upx shell: pwsh @@ -152,6 +149,9 @@ jobs: upx --version # ── Install build dependencies ────────────────────────────────────────── + # pyinstaller version is pinned in requirements-dev.txt. + # fory_compiler installed from source so PyInstaller's import tracer + # can walk the actual installed package tree. - name: Install PyInstaller and fory_compiler run: | python -m pip install --upgrade pip @@ -160,8 +160,8 @@ jobs: # ── Verify all hiddenimports are importable ───────────────────────────── # PyInstaller silently ignores nonexistent hiddenimports entries. - # This step catches renamed or removed modules before they silently - # disappear from the binary. + # This step catches renamed or removed modules before PyInstaller runs, + # giving a clean ImportError instead of a silently broken binary. - name: Verify all hiddenimports are importable working-directory: compiler run: | @@ -192,24 +192,15 @@ jobs: print('All hiddenimports verified.') " - # ── Windows Defender (build job) ──────────────────────────────────────── - # Full real-time monitoring disable is required — not just path exclusions. - # Add-MpPreference -ExclusionPath only covers filesystem scans. - # The PYI-3104 / LoadLibrary: Invalid access to memory location error - # is caused by Defender intercepting the memory-mapping of python311.dll - # during PyInstaller bootloader execution. Only DisableRealtimeMonitoring - # stops that interception. - - name: Disable Windows Defender real-time monitoring (PyInstaller workaround) - if: runner.os == 'Windows' - shell: pwsh - run: Set-MpPreference -DisableRealtimeMonitoring $true - # ── Build ─────────────────────────────────────────────────────────────── + # Must run from compiler/ so pathex=['.'] in foryc.spec resolves + # fory_compiler/__main__.py correctly. - name: Build standalone binary with PyInstaller working-directory: compiler run: pyinstaller foryc.spec # ── Pre-compression smoke test ────────────────────────────────────────── + # Confirms the binary is functional before UPX touches it. - name: Smoke test (pre-UPX) shell: bash env: @@ -225,6 +216,8 @@ jobs: "$BINARY_PATH" --help # ── UPX compression ───────────────────────────────────────────────────── + # Applied to the executable stub only, not the entire dist/foryc/ dir. + # macOS skipped — UPX 4.x+ dropped Mach-O support. - name: Compress with UPX if: matrix.use_upx shell: bash @@ -241,13 +234,17 @@ jobs: " # ── macOS aarch64: ad-hoc codesign ─────────────────────────────────────── - # PyInstaller-built Apple Silicon binaries require valid code signatures. - # UPX is disabled for macOS — this step runs regardless of UPX. + # Apple Silicon requires valid code signatures on the exe AND all .dylib/.so + # files in the onedir package. --deep only traverses .app/.framework bundles, + # not flat directories, so we sign dylibs explicitly first. # --sign - creates an ad-hoc signature; no Apple Developer ID required. - name: Ad-hoc codesign binary (macOS aarch64, required for Apple Silicon) if: matrix.codesign run: | - codesign --force --deep --sign - "${{ matrix.binary_path }}" + find "$(dirname '${{ matrix.binary_path }}')" \ + \( -name "*.dylib" -o -name "*.so" \) \ + -exec codesign --force --sign - {} \; + codesign --force --sign - "${{ matrix.binary_path }}" codesign --verify --verbose "${{ matrix.binary_path }}" # ── Post-compression smoke test ───────────────────────────────────────── @@ -256,44 +253,50 @@ jobs: run: | "${{ matrix.binary_path }}" --help - # ── crates.io 10 MB size gate ──────────────────────────────────────────── - - name: Assert binary is under 10 MB (crates.io hard limit) + # ── Onedir package size report ─────────────────────────────────────────── + # In --onedir mode the executable is a ~1 MB bootloader stub. + # The actual payload (python311.dll + stdlib .pyc files) is 20-40 MB total. + # The 10 MB crates.io gate from --onefile does not apply here. + # Phase 2 foryc-bin will distribute via zip archive or download-on-install, + # NOT direct binary embedding in a single crate. + # This step reports both sizes and fails only on a 50 MB sanity limit. + - name: Report onedir package size shell: bash env: BINARY_PATH: ${{ matrix.binary_path }} run: | python -c " - import os, sys - path = os.environ['BINARY_PATH'] - size = os.path.getsize(path) - limit = 10 * 1024 * 1024 - print(f'Final size: {size:,} bytes ({size/1024/1024:.2f} MB)') - print(f'Limit: {limit:,} bytes (10.00 MB)') - if size > limit: - print() - print('FAIL: Binary exceeds 10 MB crates.io per-crate limit.') - print('Add exclusions to compiler/foryc.spec or investigate UPX options.') + import os, sys, pathlib + binary = pathlib.Path(os.environ['BINARY_PATH']) + dist_dir = binary.parent + total = sum(f.stat().st_size for f in dist_dir.rglob('*') if f.is_file()) + exe_size = binary.stat().st_size + print(f'Executable: {exe_size:,} bytes ({exe_size/1024/1024:.2f} MB)') + print(f'Total dir: {total:,} bytes ({total/1024/1024:.2f} MB)') + if total > 50 * 1024 * 1024: + print('FAIL: onedir package exceeds 50 MB sanity limit.') sys.exit(1) print('PASS') " - # ── Upload artifact ───────────────────────────────────────────────────── + # ── Upload artifact (entire onedir directory) ──────────────────────────── - name: Upload binary artifact uses: actions/upload-artifact@v4 with: name: ${{ matrix.artifact_name }} - path: ${{ matrix.binary_path }} + path: compiler/dist/foryc/ retention-days: 30 if-no-files-found: error # ───────────────────────────────────────────────────────────────────────────── # VALIDATE JOB - # Downloads each binary onto its native runner and runs: + # Downloads each binary directory onto its native runner and runs: # 1. --help sanity check - # Python is NOT set up in this job — binary must be fully self-contained. - # 2. End-to-end compile of compiler/examples/demo.fdl to Rust output. - # 3. End-to-end compile of compiler/examples/demo.fdl to Java output. - # Validates that all 5 language generators are correctly embedded. + # 2. End-to-end compile of compiler/examples/demo.fdl to Rust output + # 3. End-to-end compile of compiler/examples/demo.fdl to Java output + # + # No Defender workaround needed — onedir has no runtime extraction at all. + # Python is NOT set up — binary must be fully self-contained. # ───────────────────────────────────────────────────────────────────────────── validate: name: validate / ${{ matrix.target }} @@ -330,7 +333,7 @@ jobs: steps: # Sparse checkout: only compiler/examples/ needed for demo.fdl. - # Python is deliberately NOT set up — the binary must run standalone. + # Python deliberately NOT set up — binary must run standalone. - name: Checkout (sparse — compiler/examples only) uses: actions/checkout@v5 with: @@ -343,23 +346,12 @@ jobs: name: ${{ matrix.artifact_name }} path: ./artifact - # ── Windows Defender (validate job) ──────────────────────────────────── - # MUST appear before the binary is executed — not just before the build. - # PyInstaller onefile extracts python311.dll to RUNNER_TEMP on first run. - # Defender intercepts the LoadLibrary call at the memory level, causing - # PYI-3104: Invalid access to memory location. - # Path exclusions (Add-MpPreference) do NOT fix this — full real-time - # monitoring disable is required in this job exactly as in the build job. - - name: Disable Windows Defender real-time monitoring (PyInstaller extraction workaround) - if: runner.os == 'Windows' - shell: pwsh - run: Set-MpPreference -DisableRealtimeMonitoring $true - - name: Make executable (Unix) if: runner.os != 'Windows' run: chmod +x ./artifact/${{ matrix.binary_name }} # ── Test 1: --help ─────────────────────────────────────────────────────── + # Requires argparse usage line. Crash tracebacks won't match ^usage:.*foryc. - name: Validate --help output shell: bash run: | @@ -392,7 +384,7 @@ jobs: fi echo "PASS: ${RS_COUNT} .rs file(s) generated" - # ── Test 3: End-to-end FDL → Java ───────────────────────────────────────── + # ── Test 3: End-to-end FDL → Java ──────────────────────────────────────── - name: End-to-end compile demo.fdl to Java shell: bash run: | @@ -448,7 +440,7 @@ jobs: # - target: linux-aarch64 # os: ubuntu-22.04 # artifact_name: foryc-linux-aarch64 -# binary_path: compiler/dist/foryc +# binary_path: compiler/dist/foryc/foryc # binary_name: foryc # use_upx: true # codesign: false diff --git a/compiler/foryc.spec b/compiler/foryc.spec index 4ff6a5f749..747980198d 100644 --- a/compiler/foryc.spec +++ b/compiler/foryc.spec @@ -22,11 +22,18 @@ # Run from the compiler/ directory: # cd compiler && pyinstaller foryc.spec # -# Output: compiler/dist/foryc (foryc.exe on Windows) +# Output: compiler/dist/foryc/foryc (foryc.exe on Windows) +# +# --onedir mode is used instead of --onefile. +# --onefile extracts Python DLLs to %TEMP% at runtime, which Windows Defender +# intercepts at the memory-mapping level (PYI-xxxx: LoadLibrary: Invalid access +# to memory location / ERROR_NOACCESS). This cannot be suppressed on GitHub's +# hardened Windows Server 2022 runners even with DisableRealtimeMonitoring. +# --onedir pre-extracts everything during the build step; no runtime extraction +# occurs, so Defender is never triggered. # # UPX compression is intentionally NOT applied here. -# The CI workflow applies UPX manually per-platform for precise control -# (macOS aarch64 requires codesign after UPX; Windows needs a separate UPX path). +# The CI workflow applies UPX manually per-platform for precise control. a = Analysis( ['fory_compiler/__main__.py'], @@ -36,7 +43,6 @@ a = Analysis( # fory_compiler has zero third-party pip dependencies. # All frontends use hand-written lexers/parsers (pure stdlib). # All generators are explicitly statically imported in generators/__init__.py. - # PyInstaller's import tracer catches everything automatically. # This list is a complete belt-and-suspenders safety net — every module # in the fory_compiler package is enumerated here to prevent any future # refactoring from silently dropping a module from the binary. @@ -45,7 +51,7 @@ a = Analysis( 'fory_compiler', 'fory_compiler.cli', - # IR layer — all 5 modules + # IR layer — all 6 modules 'fory_compiler.ir', 'fory_compiler.ir.ast', 'fory_compiler.ir.emitter', @@ -92,7 +98,7 @@ a = Analysis( # foryc uses: argparse, copy, os, sys, pathlib, typing, dataclasses, # enum, re, abc, io, collections, functools, itertools. # Everything below is confirmed unused — aggressively excluded to - # reduce pre-UPX binary size. + # reduce binary size. excludes=[ 'unittest', 'doctest', 'pdb', 'pydoc', 'py_compile', 'profile', 'distutils', 'setuptools', 'pkg_resources', @@ -109,20 +115,18 @@ a = Analysis( pyz = PYZ(a.pure, a.zipped_data) +# --onedir mode: exclude_binaries=True keeps EXE as a stub only. +# COLLECT below pulls the exe + all DLLs + stdlib into dist/foryc/ directory. exe = EXE( pyz, a.scripts, - a.binaries, - a.zipfiles, - a.datas, [], + exclude_binaries=True, name='foryc', debug=False, bootloader_ignore_signals=False, strip=True, upx=False, - upx_exclude=[], - runtime_tmpdir=None, console=True, disable_windowed_traceback=False, argv_emulation=False, @@ -130,3 +134,17 @@ exe = EXE( codesign_identity=None, entitlements_file=None, ) + +# COLLECT bundles everything into dist/foryc/ +# strip=False here — stripping shared libraries/DLLs breaks them. +# Only the executable stub (above) is stripped. +coll = COLLECT( + exe, + a.binaries, + a.zipfiles, + a.datas, + strip=False, + upx=False, + upx_exclude=[], + name='foryc', +) From 30368fbcc61a5e1569bdaec93fa29bfdb984548f Mon Sep 17 00:00:00 2001 From: Zakir Date: Tue, 24 Feb 2026 02:01:54 +0530 Subject: [PATCH 11/13] fix ci --- .github/workflows/build-foryc-binaries.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-foryc-binaries.yml b/.github/workflows/build-foryc-binaries.yml index 98262c0ee6..b31b05cf21 100644 --- a/.github/workflows/build-foryc-binaries.yml +++ b/.github/workflows/build-foryc-binaries.yml @@ -65,6 +65,8 @@ env: # UPX notes: # UPX is applied to the executable inside dist/foryc/ only, not the dir. # macOS skips UPX — UPX 4.x+ dropped Mach-O support entirely. +# Windows skips UPX — the onedir bootloader stub's PE layout is +# incompatible with UPX --best --lzma (exit code 1, no output). # # Phase 2 distribution note: # The onedir package is 20-40 MB total (python311.dll + stdlib .pyc files). @@ -117,7 +119,7 @@ jobs: artifact_name: foryc-windows-x86_64 binary_path: compiler/dist/foryc/foryc.exe binary_name: foryc.exe - use_upx: true + use_upx: false codesign: false steps: From 41ff561668657ae502cd206102101245e4d877d5 Mon Sep 17 00:00:00 2001 From: Zakir Date: Tue, 24 Feb 2026 15:38:49 +0530 Subject: [PATCH 12/13] fix ci --- .github/workflows/build-foryc-binaries.yml | 562 ++++++++++++++++----- compiler/foryc.spec | 92 ++-- compiler/requirements-dev.txt | 17 + 3 files changed, 482 insertions(+), 189 deletions(-) diff --git a/.github/workflows/build-foryc-binaries.yml b/.github/workflows/build-foryc-binaries.yml index b31b05cf21..ab36df9832 100644 --- a/.github/workflows/build-foryc-binaries.yml +++ b/.github/workflows/build-foryc-binaries.yml @@ -34,6 +34,12 @@ on: - "compiler/**" - ".github/workflows/build-foryc-binaries.yml" +# Cancel in-progress runs on the same PR to avoid wasting CI minutes on +# force-pushes. Does NOT cancel tagged or dispatch runs. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + permissions: contents: read @@ -41,42 +47,158 @@ env: PYTHON_VERSION: "3.11" # ───────────────────────────────────────────────────────────────────────────── -# BUILD JOB -# Produces one standalone foryc binary directory per target platform. +# ACTION SHA PINS — verified live against GitHub commit graph 2026-02-24 # -# Runner notes: -# linux-x86_64 : ubuntu-22.04 -# linux-aarch64 : ubuntu-24.04-arm -# macos-x86_64 : macos-15-intel -# macos-aarch64 : macos-15 -# windows-x86_64: windows-latest (Server 2022) +# actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 +# actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 +# actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 +# actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 # -# --onedir mode: -# PyInstaller --onefile extracts DLLs to %TEMP% at runtime. Windows Defender -# on GitHub's hardened runners intercepts LoadLibrary at the memory-mapping -# level (PYI-xxxx ERROR_NOACCESS) even with DisableRealtimeMonitoring $true. -# --onedir pre-extracts everything at build time. No runtime extraction means -# no Defender interception. No Defender workaround needed anywhere. +# Node.js 24 runner requirement: +# setup-python v6, upload-artifact v6, and download-artifact v7 all run on +# Node.js 24 (runs.using: node24) and require Actions runner >= v2.327.1. +# GitHub-hosted runners (ubuntu-22.04, macos-15, windows-latest) are +# auto-updated and satisfy this requirement automatically. +# Self-hosted runners MUST be upgraded to >= v2.327.1 before using this +# workflow, otherwise setup-python, upload-artifact, and download-artifact +# will refuse to execute. # -# Output per target: compiler/dist/foryc/ directory -# Unix: compiler/dist/foryc/foryc -# Windows: compiler/dist/foryc/foryc.exe +# Re-verify SHAs at each Fory release: +# https://github.com/actions/checkout/releases +# https://github.com/actions/setup-python/releases +# https://github.com/actions/upload-artifact/releases +# https://github.com/actions/download-artifact/releases +# ───────────────────────────────────────────────────────────────────────────── + +# ───────────────────────────────────────────────────────────────────────────── +# SPEC-CHECK JOB (pull_request only — lightweight, no binary build) # -# UPX notes: -# UPX is applied to the executable inside dist/foryc/ only, not the dir. -# macOS skips UPX — UPX 4.x+ dropped Mach-O support entirely. -# Windows skips UPX — the onedir bootloader stub's PE layout is -# incompatible with UPX --best --lzma (exit code 1, no output). +# Full 5-platform binary builds on every compiler PR cost ~25 CI-minutes. +# On PRs we run a fast spec-check instead: +# 1. Verify all fory_compiler modules are importable via importlib.import_module +# (same pkgutil discovery as foryc.spec — single source of truth) +# 2. AST-parse foryc.spec to catch Python syntax errors without executing it +# (executing the spec requires PyInstaller-injected Analysis/PYZ/EXE/COLLECT +# globals absent in a plain interpreter; ast.parse validates syntax only) # -# Phase 2 distribution note: -# The onedir package is 20-40 MB total (python311.dll + stdlib .pyc files). -# This exceeds the 10 MB crates.io per-crate limit from --onefile planning. -# foryc-bin will distribute via zip archive or download-on-first-install, -# not direct binary embedding in a single crate. +# Full builds run on: workflow_dispatch, workflow_call, push tags foryc-v* # ───────────────────────────────────────────────────────────────────────────── jobs: + spec-check: + name: spec / import-check (PR) + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: "pip" + cache-dependency-path: | + compiler/requirements-dev.txt + compiler/pyproject.toml + + - name: Install fory_compiler (no pyinstaller needed for spec-check) + run: | + python -m pip install --upgrade pip + pip install ./compiler + + # importlib.import_module CONTRACT: + # Unlike __import__, importlib.import_module('fory_compiler.ir.ast') + # explicitly loads the named submodule, not just its parent chain. + # This catches broken submodules that a parent __init__.py would mask. + # + # onerror collects ALL failures before raising so every broken package + # is reported in one pass, not just the first one encountered. + - name: Verify all fory_compiler modules are importable + working-directory: compiler + run: | + python -c " + import importlib, pkgutil, fory_compiler + + walk_errors = [] + def onerror(name): + walk_errors.append(name) + + mods = ['fory_compiler'] + [ + m.name for m in pkgutil.walk_packages( + path=fory_compiler.__path__, + prefix=fory_compiler.__name__ + '.', + onerror=onerror, + ) + ] + if walk_errors: + raise ImportError( + 'walk_packages failed to recurse into: ' + str(walk_errors) + + '\nFix import errors in these packages before building.' + ) + import_errors = [] + for m in mods: + try: + importlib.import_module(m) + print(f'OK: {m}') + except Exception as e: + import_errors.append(f'{m}: {e}') + if import_errors: + raise ImportError( + 'Failed to import:\n' + '\n'.join(import_errors) + ) + print(f'All {len(mods)} module(s) verified.') + " + + # ast.parse validates Python syntax without executing the spec. + # Executing foryc.spec directly fails because Analysis/PYZ/EXE/COLLECT + # are PyInstaller-injected globals absent in a plain interpreter. + - name: Syntax-check foryc.spec (AST parse) + run: | + python -c " + import ast + with open('compiler/foryc.spec', encoding='utf-8') as f: + src = f.read() + tree = ast.parse(src, filename='foryc.spec') + print(f'foryc.spec: syntax OK ({len(list(ast.walk(tree)))} AST nodes)') + " + + # ───────────────────────────────────────────────────────────────────────────── + # BUILD JOB (tags / workflow_dispatch / workflow_call only — NOT pull_request) + # + # Runner notes: + # linux-x86_64 : ubuntu-22.04 + # linux-aarch64 : ubuntu-24.04-arm (native ARM64 runner — no QEMU) + # macos-x86_64 : macos-15-intel + # macos-aarch64 : macos-15 + # windows-x86_64: windows-latest (Server 2022) + # + # --onedir mode (not --onefile): + # --onefile extracts DLLs to %TEMP% at runtime. Windows Defender on GitHub's + # hardened runners intercepts LoadLibrary at the memory-mapping level + # (PYI-xxxx ERROR_NOACCESS) even with DisableRealtimeMonitoring $true. + # --onedir pre-extracts at build time; no runtime extraction, no interception. + # + # UPX notes: + # Applied to the exe stub only, not the entire dist/foryc/ directory. + # macOS: skipped — UPX 4.x+ dropped Mach-O support entirely. + # Windows: skipped — onedir bootloader stub PE layout incompatible with + # UPX --best --lzma (exits code 1, produces no output). + # + # Phase 2 distribution note: + # --onedir produces 20-40 MB total (python311.dll + stdlib .pyc files). + # This exceeds the 10 MB crates.io per-crate limit. + # foryc-bin will distribute via zip archive or download-on-first-install. + # foryc_path in foryc-build Config must resolve to a DIRECTORY, not a + # single file — Phase 2 implementors must account for this. + # + # windows-aarch64: + # Not included in this PR. Tracked in issue #3292. + # GitHub Actions windows-11-arm runners are now available but PyInstaller + # aarch64 Windows support requires validation. Targeted for Phase 2. + # ───────────────────────────────────────────────────────────────────────────── build: name: build / ${{ matrix.target }} + if: github.event_name != 'pull_request' runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -124,10 +246,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ env.PYTHON_VERSION }} cache: "pip" @@ -150,48 +272,69 @@ jobs: choco install upx --yes --no-progress upx --version + # ── macOS: pin deployment target ───────────────────────────────────────── + # Without MACOSX_DEPLOYMENT_TARGET, binaries built on macOS 15 embed + # LC_BUILD_VERSION minos=15.0 and silently fail on macOS ≤14 with: + # dyld: Symbol not found / incompatible library version + # 13.0 rationale: + # - Covers all active Apple Silicon + Intel users + # - Python 3.11 minimum is 10.9; 13.0 is safe + # - macOS 13 (Ventura) is the oldest release still receiving security + # patches as of 2026 + - name: Set macOS deployment target + if: runner.os == 'macOS' + run: echo "MACOSX_DEPLOYMENT_TARGET=13.0" >> "$GITHUB_ENV" + # ── Install build dependencies ────────────────────────────────────────── - # pyinstaller version is pinned in requirements-dev.txt. + # pyinstaller is pinned in requirements-dev.txt (>=6.0,<7.0). # fory_compiler installed from source so PyInstaller's import tracer - # can walk the actual installed package tree. + # walks the actual installed package tree, matching pkgutil.walk_packages + # in foryc.spec. - name: Install PyInstaller and fory_compiler run: | python -m pip install --upgrade pip pip install -r compiler/requirements-dev.txt pip install ./compiler - # ── Verify all hiddenimports are importable ───────────────────────────── - # PyInstaller silently ignores nonexistent hiddenimports entries. - # This step catches renamed or removed modules before PyInstaller runs, - # giving a clean ImportError instead of a silently broken binary. - - name: Verify all hiddenimports are importable + # ── Verify all fory_compiler modules are importable ───────────────────── + # Same pkgutil discovery as foryc.spec — single source of truth. + # importlib.import_module guarantees each named submodule is independently + # loaded, not just its parent chain. + # onerror collects ALL failing packages before raising. + - name: Verify all fory_compiler modules are importable working-directory: compiler run: | python -c " - mods = [ - 'fory_compiler', 'fory_compiler.cli', - 'fory_compiler.ir', 'fory_compiler.ir.ast', - 'fory_compiler.ir.emitter', 'fory_compiler.ir.validator', - 'fory_compiler.ir.type_id', 'fory_compiler.ir.types', - 'fory_compiler.frontend', 'fory_compiler.frontend.base', - 'fory_compiler.frontend.utils', - 'fory_compiler.frontend.fdl', 'fory_compiler.frontend.fdl.lexer', - 'fory_compiler.frontend.fdl.parser', - 'fory_compiler.frontend.proto', 'fory_compiler.frontend.proto.ast', - 'fory_compiler.frontend.proto.lexer', 'fory_compiler.frontend.proto.parser', - 'fory_compiler.frontend.proto.translator', - 'fory_compiler.frontend.fbs', 'fory_compiler.frontend.fbs.ast', - 'fory_compiler.frontend.fbs.lexer', 'fory_compiler.frontend.fbs.parser', - 'fory_compiler.frontend.fbs.translator', - 'fory_compiler.generators', 'fory_compiler.generators.base', - 'fory_compiler.generators.java', 'fory_compiler.generators.python', - 'fory_compiler.generators.cpp', 'fory_compiler.generators.rust', - 'fory_compiler.generators.go', + import importlib, pkgutil, fory_compiler + + walk_errors = [] + def onerror(name): + walk_errors.append(name) + + mods = ['fory_compiler'] + [ + m.name for m in pkgutil.walk_packages( + path=fory_compiler.__path__, + prefix=fory_compiler.__name__ + '.', + onerror=onerror, + ) ] + if walk_errors: + raise ImportError( + 'walk_packages failed to recurse into: ' + str(walk_errors) + + '\nFix import errors in these packages before building.' + ) + import_errors = [] for m in mods: - __import__(m) - print(f'OK: {m}') - print('All hiddenimports verified.') + try: + importlib.import_module(m) + print(f'OK: {m}') + except Exception as e: + import_errors.append(f'{m}: {e}') + if import_errors: + raise ImportError( + 'Failed to import:\n' + '\n'.join(import_errors) + ) + print(f'All {len(mods)} module(s) verified.') " # ── Build ─────────────────────────────────────────────────────────────── @@ -201,8 +344,46 @@ jobs: working-directory: compiler run: pyinstaller foryc.spec + # ── Verify macOS deployment target ─────────────────────────────────────── + # Asserts minos <= 13.x. Fails hard if minos cannot be parsed — a + # WARNING-and-pass would allow a minos=15.0 binary through silently. + - name: Verify macOS deployment target (minos ≤ 13.x) + if: runner.os == 'macOS' + env: + BINARY_PATH: ${{ matrix.binary_path }} + run: | + echo "=== otool LC_BUILD_VERSION ===" + otool -l "${BINARY_PATH}" | grep -A4 LC_BUILD_VERSION \ + || otool -l "${BINARY_PATH}" | grep -A3 LC_VERSION_MIN_MACOSX \ + || true + + MINOS=$(otool -l "${BINARY_PATH}" \ + | grep -A4 LC_BUILD_VERSION \ + | awk '/minos/{print $2}' \ + | head -1) + + if [ -z "${MINOS}" ]; then + MINOS=$(otool -l "${BINARY_PATH}" \ + | grep -A3 LC_VERSION_MIN_MACOSX \ + | awk '/version/{print $2}' \ + | head -1) + fi + + if [ -z "${MINOS}" ]; then + echo "ERROR: could not parse minos from otool output." + echo " Verify MACOSX_DEPLOYMENT_TARGET=13.0 was set during build." + exit 1 + fi + + MAJOR=$(echo "${MINOS}" | cut -d. -f1) + if [ "${MAJOR}" -gt 13 ]; then + echo "FAIL: minos=${MINOS} — binary requires macOS ${MAJOR}+ and will" + echo " not launch on macOS ≤13." + exit 1 + fi + echo "PASS: minos=${MINOS} (≤ 13.x — compatible with all active macOS releases)" + # ── Pre-compression smoke test ────────────────────────────────────────── - # Confirms the binary is functional before UPX touches it. - name: Smoke test (pre-UPX) shell: bash env: @@ -215,18 +396,16 @@ jobs: s = os.path.getsize(p) print(f'Size: {s:,} bytes ({s/1024/1024:.2f} MB)') " - "$BINARY_PATH" --help + "${BINARY_PATH}" --help # ── UPX compression ───────────────────────────────────────────────────── - # Applied to the executable stub only, not the entire dist/foryc/ dir. - # macOS skipped — UPX 4.x+ dropped Mach-O support. - name: Compress with UPX if: matrix.use_upx shell: bash env: BINARY_PATH: ${{ matrix.binary_path }} run: | - upx --best --lzma "$BINARY_PATH" + upx --best --lzma "${BINARY_PATH}" echo "=== Post-UPX binary size ===" python -c " import os @@ -236,32 +415,56 @@ jobs: " # ── macOS aarch64: ad-hoc codesign ─────────────────────────────────────── - # Apple Silicon requires valid code signatures on the exe AND all .dylib/.so - # files in the onedir package. --deep only traverses .app/.framework bundles, - # not flat directories, so we sign dylibs explicitly first. - # --sign - creates an ad-hoc signature; no Apple Developer ID required. - - name: Ad-hoc codesign binary (macOS aarch64, required for Apple Silicon) + # Apple Silicon requires valid signatures on the exe AND every .dylib/.so + # in the --onedir package. codesign --deep does NOT traverse flat + # directories (only .app/.framework bundles). + # BINARY_PATH via env prevents quoting failures on paths with spaces. + # Verify loop covers all dylibs — an unsigned dylib causes DYLD_LIBRARY_PATH + # resolution failure at runtime even if the main exe is correctly signed. + - name: Ad-hoc codesign (macOS aarch64 — Apple Silicon requirement) if: matrix.codesign + shell: bash + env: + BINARY_PATH: ${{ matrix.binary_path }} run: | - find "$(dirname '${{ matrix.binary_path }}')" \ - \( -name "*.dylib" -o -name "*.so" \) \ + DIST_DIR="$(dirname "${BINARY_PATH}")" + + echo "=== Signing all .dylib and .so files in ${DIST_DIR} ===" + find "${DIST_DIR}" \( -name "*.dylib" -o -name "*.so" \) \ -exec codesign --force --sign - {} \; - codesign --force --sign - "${{ matrix.binary_path }}" - codesign --verify --verbose "${{ matrix.binary_path }}" - # ── Post-compression smoke test ───────────────────────────────────────── - - name: Smoke test (post-UPX) + echo "=== Signing main executable ===" + codesign --force --sign - "${BINARY_PATH}" + + echo "=== Verifying all .dylib and .so signatures ===" + VERIFY_FAILURES=0 + while IFS= read -r -d '' lib; do + if ! codesign --verify --verbose "${lib}" 2>&1; then + echo "FAIL: signature verification failed for ${lib}" + VERIFY_FAILURES=$((VERIFY_FAILURES + 1)) + fi + done < <(find "${DIST_DIR}" \( -name "*.dylib" -o -name "*.so" \) -print0) + + echo "=== Verifying main executable signature ===" + codesign --verify --verbose "${BINARY_PATH}" + + if [ "${VERIFY_FAILURES}" -gt 0 ]; then + echo "FAIL: ${VERIFY_FAILURES} library signature verification(s) failed." + exit 1 + fi + echo "PASS: all signatures verified." + + # ── Final smoke test ───────────────────────────────────────────────────── + # Named "final" not "post-UPX" — UPX is skipped on 3 of 5 targets. + # Runs after all transformations (UPX, codesign) are complete. + - name: Smoke test (final) shell: bash + env: + BINARY_PATH: ${{ matrix.binary_path }} run: | - "${{ matrix.binary_path }}" --help + "${BINARY_PATH}" --help # ── Onedir package size report ─────────────────────────────────────────── - # In --onedir mode the executable is a ~1 MB bootloader stub. - # The actual payload (python311.dll + stdlib .pyc files) is 20-40 MB total. - # The 10 MB crates.io gate from --onefile does not apply here. - # Phase 2 foryc-bin will distribute via zip archive or download-on-install, - # NOT direct binary embedding in a single crate. - # This step reports both sizes and fails only on a 50 MB sanity limit. - name: Report onedir package size shell: bash env: @@ -281,27 +484,55 @@ jobs: print('PASS') " - # ── Upload artifact (entire onedir directory) ──────────────────────────── + # ── SHA-256 checksums ──────────────────────────────────────────────────── + # Generates SHA256SUMS.txt for Phase 2 foryc-build integrity verification. + # Python used for cross-platform consistency (sha256sum/shasum/certutil + # are all platform-specific; Python's hashlib is not). + # chr(92) == backslash — normalises Windows paths in the output file. + - name: Generate SHA-256 checksums + shell: bash + env: + BINARY_PATH: ${{ matrix.binary_path }} + run: | + python -c " + import hashlib, os, pathlib + binary = pathlib.Path(os.environ['BINARY_PATH']) + dist_dir = binary.parent + lines = [] + for f in sorted(dist_dir.rglob('*')): + if f.is_file() and f.name != 'SHA256SUMS.txt': + digest = hashlib.sha256(f.read_bytes()).hexdigest() + rel = str(f.relative_to(dist_dir)).replace(chr(92), '/') + lines.append(f'{digest} {rel}') + print(f'{digest} {rel}') + (dist_dir / 'SHA256SUMS.txt').write_text( + '\n'.join(lines) + '\n', encoding='utf-8' + ) + print(f'Wrote {len(lines)} checksums to SHA256SUMS.txt') + " + + # ── Upload artifact ────────────────────────────────────────────────────── + # retention-days: tagged releases → 90 days (Phase 2 runway) + # all other builds → 30 days - name: Upload binary artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: ${{ matrix.artifact_name }} path: compiler/dist/foryc/ - retention-days: 30 + retention-days: ${{ startsWith(github.ref, 'refs/tags/') && 90 || 30 }} if-no-files-found: error # ───────────────────────────────────────────────────────────────────────────── - # VALIDATE JOB - # Downloads each binary directory onto its native runner and runs: - # 1. --help sanity check - # 2. End-to-end compile of compiler/examples/demo.fdl to Rust output - # 3. End-to-end compile of compiler/examples/demo.fdl to Java output + # VALIDATE JOB (tags / workflow_dispatch / workflow_call only) # - # No Defender workaround needed — onedir has no runtime extraction at all. + # All 5 generator backends validated — a generator can be correctly included + # in hiddenimports at build time but silently fail at runtime (e.g. a + # conditional import inside the generator that PyInstaller missed). # Python is NOT set up — binary must be fully self-contained. # ───────────────────────────────────────────────────────────────────────────── validate: name: validate / ${{ matrix.target }} + if: github.event_name != 'pull_request' needs: build runs-on: ${{ matrix.os }} strategy: @@ -334,16 +565,14 @@ jobs: binary_name: foryc.exe steps: - # Sparse checkout: only compiler/examples/ needed for demo.fdl. - # Python deliberately NOT set up — binary must run standalone. - name: Checkout (sparse — compiler/examples only) - uses: actions/checkout@v5 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout: compiler/examples sparse-checkout-cone-mode: true - name: Download artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: ${{ matrix.artifact_name }} path: ./artifact @@ -352,70 +581,107 @@ jobs: if: runner.os != 'Windows' run: chmod +x ./artifact/${{ matrix.binary_name }} + # Pre-flight: clear error message instead of confusing foryc exit code. + - name: Assert compiler/examples/demo.fdl exists + shell: bash + run: | + if [ ! -f compiler/examples/demo.fdl ]; then + echo "ERROR: compiler/examples/demo.fdl not found." + echo " This file must exist in the repository for E2E validation." + exit 1 + fi + echo "PASS: compiler/examples/demo.fdl found." + # ── Test 1: --help ─────────────────────────────────────────────────────── - # Requires argparse usage line. Crash tracebacks won't match ^usage:.*foryc. - name: Validate --help output shell: bash run: | OUTPUT=$(./artifact/${{ matrix.binary_name }} --help 2>&1) echo "$OUTPUT" echo "$OUTPUT" | grep -qiE "^usage:.*foryc" || { - echo "ERROR: --help missing expected usage line" + echo "ERROR: --help output missing expected 'usage: ... foryc' line." exit 1 } - # ── Test 2: End-to-end FDL → Rust ──────────────────────────────────────── - - name: End-to-end compile demo.fdl to Rust + # ── Test 2: FDL → Rust ─────────────────────────────────────────────────── + - name: End-to-end compile demo.fdl → Rust shell: bash run: | OUT_DIR="${RUNNER_TEMP}/foryc-e2e-rust" mkdir -p "${OUT_DIR}" - - echo "=== Compiling compiler/examples/demo.fdl → Rust ===" - ./artifact/${{ matrix.binary_name }} \ - --rust_out "${OUT_DIR}" \ - compiler/examples/demo.fdl - - echo "=== Output files ===" + ./artifact/${{ matrix.binary_name }} --rust_out "${OUT_DIR}" compiler/examples/demo.fdl ls -la "${OUT_DIR}/" - RS_COUNT=$(find "${OUT_DIR}" -name "*.rs" -size +0c | wc -l) - if [ "${RS_COUNT}" -eq 0 ]; then - echo "ERROR: no non-empty .rs files generated" - exit 1 - fi + [ "${RS_COUNT}" -gt 0 ] || { echo "ERROR: no .rs files generated"; exit 1; } echo "PASS: ${RS_COUNT} .rs file(s) generated" - # ── Test 3: End-to-end FDL → Java ──────────────────────────────────────── - - name: End-to-end compile demo.fdl to Java + # ── Test 3: FDL → Java ─────────────────────────────────────────────────── + - name: End-to-end compile demo.fdl → Java shell: bash run: | - JAVA_OUT="${RUNNER_TEMP}/foryc-e2e-java" - mkdir -p "${JAVA_OUT}" - - echo "=== Compiling compiler/examples/demo.fdl → Java ===" - ./artifact/${{ matrix.binary_name }} \ - --java_out "${JAVA_OUT}" \ - compiler/examples/demo.fdl + OUT_DIR="${RUNNER_TEMP}/foryc-e2e-java" + mkdir -p "${OUT_DIR}" + ./artifact/${{ matrix.binary_name }} --java_out "${OUT_DIR}" compiler/examples/demo.fdl + ls -la "${OUT_DIR}/" + JAVA_COUNT=$(find "${OUT_DIR}" -name "*.java" -size +0c | wc -l) + [ "${JAVA_COUNT}" -gt 0 ] || { echo "ERROR: no .java files generated"; exit 1; } + echo "PASS: ${JAVA_COUNT} .java file(s) generated" - echo "=== Output files ===" - ls -la "${JAVA_OUT}/" + # ── Test 4: FDL → Python ───────────────────────────────────────────────── + - name: End-to-end compile demo.fdl → Python + shell: bash + run: | + OUT_DIR="${RUNNER_TEMP}/foryc-e2e-python" + mkdir -p "${OUT_DIR}" + ./artifact/${{ matrix.binary_name }} --python_out "${OUT_DIR}" compiler/examples/demo.fdl + ls -la "${OUT_DIR}/" + PY_COUNT=$(find "${OUT_DIR}" -name "*.py" -size +0c | wc -l) + [ "${PY_COUNT}" -gt 0 ] || { echo "ERROR: no .py files generated"; exit 1; } + echo "PASS: ${PY_COUNT} .py file(s) generated" - JAVA_COUNT=$(find "${JAVA_OUT}" -name "*.java" -size +0c | wc -l) - if [ "${JAVA_COUNT}" -eq 0 ]; then - echo "ERROR: no non-empty .java files generated" - exit 1 - fi - echo "PASS: ${JAVA_COUNT} .java file(s) generated" + # ── Test 5: FDL → C++ ──────────────────────────────────────────────────── + - name: End-to-end compile demo.fdl → C++ + shell: bash + run: | + OUT_DIR="${RUNNER_TEMP}/foryc-e2e-cpp" + mkdir -p "${OUT_DIR}" + ./artifact/${{ matrix.binary_name }} --cpp_out "${OUT_DIR}" compiler/examples/demo.fdl + ls -la "${OUT_DIR}/" + CPP_COUNT=$(find "${OUT_DIR}" \ + \( -name "*.h" -o -name "*.cc" -o -name "*.cpp" \) -size +0c | wc -l) + [ "${CPP_COUNT}" -gt 0 ] || { echo "ERROR: no C++ files generated"; exit 1; } + echo "PASS: ${CPP_COUNT} C++ file(s) generated" + + # ── Test 6: FDL → Go ───────────────────────────────────────────────────── + # Go output may be nested in subdirs based on go_package schema option. + - name: End-to-end compile demo.fdl → Go + shell: bash + run: | + OUT_DIR="${RUNNER_TEMP}/foryc-e2e-go" + mkdir -p "${OUT_DIR}" + ./artifact/${{ matrix.binary_name }} --go_out "${OUT_DIR}" compiler/examples/demo.fdl + find "${OUT_DIR}" -type f | sort + GO_COUNT=$(find "${OUT_DIR}" -name "*.go" -size +0c | wc -l) + [ "${GO_COUNT}" -gt 0 ] || { echo "ERROR: no .go files generated"; exit 1; } + echo "PASS: ${GO_COUNT} .go file(s) generated" # ───────────────────────────────────────────────────────────────────────────── # SUMMARY JOB # Single required status check for branch protection rules. - # The Phase 4 release pipeline reads this job's output via workflow_call. + # Phase 4 release pipeline reads artifacts_ready via workflow_call. + # + # GitHub Actions result values for a conditionally skipped job (if: false): + # needs..result == "skipped" — documented, always "skipped" not "". + # + # Three valid terminal states: + # 1. Tagged/dispatch: BUILD+VALIDATE success → ready=true, exit 0 + # SPEC_CHECK = "skipped" on these events — State 1 does not check it. + # 2. PR: SPEC_CHECK success, BUILD+VALIDATE skipped → ready=false, exit 0 + # 3. Any actual failure → ready=false, exit 1 # ───────────────────────────────────────────────────────────────────────────── build-complete: name: foryc / all binaries ready - needs: [build, validate] + needs: [spec-check, build, validate] runs-on: ubuntu-latest if: always() outputs: @@ -424,17 +690,33 @@ jobs: - name: Evaluate results id: check run: | + SPEC_CHECK="${{ needs.spec-check.result }}" BUILD="${{ needs.build.result }}" VALIDATE="${{ needs.validate.result }}" - echo "build: ${BUILD}" - echo "validate: ${VALIDATE}" + echo "spec-check: ${SPEC_CHECK}" + echo "build: ${BUILD}" + echo "validate: ${VALIDATE}" + + # State 1: Full build (tag / workflow_dispatch / workflow_call) if [[ "${BUILD}" == "success" && "${VALIDATE}" == "success" ]]; then echo "ready=true" >> "${GITHUB_OUTPUT}" - else + echo "PASS: full binary build and validation succeeded." + exit 0 + fi + + # State 2: PR — build+validate skipped by design, spec-check ran + if [[ "${BUILD}" == "skipped" && "${VALIDATE}" == "skipped" \ + && "${SPEC_CHECK}" == "success" ]]; then echo "ready=false" >> "${GITHUB_OUTPUT}" - exit 1 + echo "PASS: PR spec-check succeeded. Full build skipped by design." + exit 0 fi + # State 3: Failure + echo "ready=false" >> "${GITHUB_OUTPUT}" + echo "FAIL: spec-check=${SPEC_CHECK} build=${BUILD} validate=${VALIDATE}" + exit 1 + # ───────────────────────────────────────────────────────────────────────────── # FALLBACK: If ubuntu-24.04-arm runner is unavailable in your fork, # replace the linux-aarch64 matrix entry with: @@ -447,16 +729,24 @@ jobs: # use_upx: true # codesign: false # -# Then add these two steps before "Install PyInstaller": +# Add `if: matrix.target != 'linux-aarch64'` to these three steps: +# "Install PyInstaller and fory_compiler" +# "Verify all fory_compiler modules are importable" +# "Build standalone binary with PyInstaller" +# +# Then add these two steps BEFORE "Install PyInstaller and fory_compiler": # -# - name: Set up QEMU +# - name: Set up QEMU (linux-aarch64 fallback only) +# if: matrix.target == 'linux-aarch64' # uses: docker/setup-qemu-action@v3 +# # Pin to full SHA: https://github.com/docker/setup-qemu-action/releases # with: # platforms: arm64 # -# - name: Build via Docker (linux-aarch64) +# - name: Build via Docker (linux-aarch64 QEMU fallback) # if: matrix.target == 'linux-aarch64' # uses: addnab/docker-run-action@v3 +# # Pin to full SHA: https://github.com/addnab/docker-run-action/releases # with: # image: python:3.11-slim-bookworm # options: --platform linux/arm64 -v ${{ github.workspace }}:/ws diff --git a/compiler/foryc.spec b/compiler/foryc.spec index 747980198d..a7dd1ef5c0 100644 --- a/compiler/foryc.spec +++ b/compiler/foryc.spec @@ -35,63 +35,49 @@ # UPX compression is intentionally NOT applied here. # The CI workflow applies UPX manually per-platform for precise control. -a = Analysis( - ['fory_compiler/__main__.py'], - pathex=['.'], - binaries=[], - datas=[], - # fory_compiler has zero third-party pip dependencies. - # All frontends use hand-written lexers/parsers (pure stdlib). - # All generators are explicitly statically imported in generators/__init__.py. - # This list is a complete belt-and-suspenders safety net — every module - # in the fory_compiler package is enumerated here to prevent any future - # refactoring from silently dropping a module from the binary. - hiddenimports=[ - # Entry point chain - 'fory_compiler', - 'fory_compiler.cli', - - # IR layer — all 6 modules - 'fory_compiler.ir', - 'fory_compiler.ir.ast', - 'fory_compiler.ir.emitter', - 'fory_compiler.ir.validator', - 'fory_compiler.ir.type_id', - 'fory_compiler.ir.types', +# ── Auto-discover all fory_compiler submodules ────────────────────────────── +# pkgutil.walk_packages walks the installed package tree at spec-evaluation +# time, so this list stays correct automatically when new submodules are added. +# This eliminates the dual-maintenance problem between foryc.spec and the +# CI verify step — both derive from the same source of truth: the installed +# fory_compiler package tree. Adding a new generator (e.g. generators/kotlin.py) +# is automatically picked up by both the spec and the CI verify step. +# +# onerror CONTRACT (critical): +# walk_packages calls onerror(pkg_name) when it cannot import a package in +# order to recurse into it. Using `onerror=lambda name: None` silently drops +# the entire subtree — the binary builds successfully but crashes at runtime +# when any generator in that subtree is invoked. +# _walk_onerror raises immediately so the BUILD fails loudly, not silently. +import pkgutil +import fory_compiler as _fc - # Frontend base utilities - 'fory_compiler.frontend', - 'fory_compiler.frontend.base', - 'fory_compiler.frontend.utils', - # FDL frontend — hand-written lexer/parser - 'fory_compiler.frontend.fdl', - 'fory_compiler.frontend.fdl.lexer', - 'fory_compiler.frontend.fdl.parser', +def _walk_onerror(pkg_name: str) -> None: + raise ImportError( + f"pkgutil.walk_packages failed to recurse into {pkg_name!r}. " + f"This package cannot be imported at spec-evaluation time. " + f"Fix the import error in {pkg_name!r} before building the binary. " + f"Hint: run `python -c \"import {pkg_name}\"` in the compiler/ venv " + f"to reproduce the error." + ) - # Proto frontend — hand-written lexer/parser/translator - 'fory_compiler.frontend.proto', - 'fory_compiler.frontend.proto.ast', - 'fory_compiler.frontend.proto.lexer', - 'fory_compiler.frontend.proto.parser', - 'fory_compiler.frontend.proto.translator', - # FBS (FlatBuffers schema) frontend — all 4 submodules - 'fory_compiler.frontend.fbs', - 'fory_compiler.frontend.fbs.ast', - 'fory_compiler.frontend.fbs.lexer', - 'fory_compiler.frontend.fbs.parser', - 'fory_compiler.frontend.fbs.translator', +hiddenimports = ["fory_compiler"] + [ + m.name + for m in pkgutil.walk_packages( + path=_fc.__path__, + prefix=_fc.__name__ + ".", + onerror=_walk_onerror, # fail loudly — never silently drop a subtree + ) +] - # Generators — all 5 language backends - 'fory_compiler.generators', - 'fory_compiler.generators.base', - 'fory_compiler.generators.java', - 'fory_compiler.generators.python', - 'fory_compiler.generators.cpp', - 'fory_compiler.generators.rust', - 'fory_compiler.generators.go', - ], +a = Analysis( + ['fory_compiler/__main__.py'], + pathex=['.'], + binaries=[], + datas=[], + hiddenimports=hiddenimports, hookspath=[], hooksconfig={}, runtime_hooks=[], @@ -116,7 +102,7 @@ a = Analysis( pyz = PYZ(a.pure, a.zipped_data) # --onedir mode: exclude_binaries=True keeps EXE as a stub only. -# COLLECT below pulls the exe + all DLLs + stdlib into dist/foryc/ directory. +# COLLECT below pulls exe + all DLLs + stdlib into dist/foryc/ directory. exe = EXE( pyz, a.scripts, diff --git a/compiler/requirements-dev.txt b/compiler/requirements-dev.txt index 648e25a071..67f1f6e4c5 100644 --- a/compiler/requirements-dev.txt +++ b/compiler/requirements-dev.txt @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Development dependencies for building the standalone foryc binary. # NOT part of the published fory-compiler package on PyPI. # This file intentionally kept separate from pyproject.toml. From 2570a6a5dfc89c847eefaa1aeea79366f72e7c26 Mon Sep 17 00:00:00 2001 From: Zakir Date: Wed, 25 Feb 2026 17:18:07 +0530 Subject: [PATCH 13/13] fix ci --- .github/workflows/build-foryc-binaries.yml | 271 ++++++++------------- 1 file changed, 106 insertions(+), 165 deletions(-) diff --git a/.github/workflows/build-foryc-binaries.yml b/.github/workflows/build-foryc-binaries.yml index ab36df9832..ce20f71139 100644 --- a/.github/workflows/build-foryc-binaries.yml +++ b/.github/workflows/build-foryc-binaries.yml @@ -34,8 +34,6 @@ on: - "compiler/**" - ".github/workflows/build-foryc-binaries.yml" -# Cancel in-progress runs on the same PR to avoid wasting CI minutes on -# force-pushes. Does NOT cancel tagged or dispatch runs. concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} @@ -72,16 +70,6 @@ env: # ───────────────────────────────────────────────────────────────────────────── # SPEC-CHECK JOB (pull_request only — lightweight, no binary build) -# -# Full 5-platform binary builds on every compiler PR cost ~25 CI-minutes. -# On PRs we run a fast spec-check instead: -# 1. Verify all fory_compiler modules are importable via importlib.import_module -# (same pkgutil discovery as foryc.spec — single source of truth) -# 2. AST-parse foryc.spec to catch Python syntax errors without executing it -# (executing the spec requires PyInstaller-injected Analysis/PYZ/EXE/COLLECT -# globals absent in a plain interpreter; ast.parse validates syntax only) -# -# Full builds run on: workflow_dispatch, workflow_call, push tags foryc-v* # ───────────────────────────────────────────────────────────────────────────── jobs: spec-check: @@ -106,13 +94,6 @@ jobs: python -m pip install --upgrade pip pip install ./compiler - # importlib.import_module CONTRACT: - # Unlike __import__, importlib.import_module('fory_compiler.ir.ast') - # explicitly loads the named submodule, not just its parent chain. - # This catches broken submodules that a parent __init__.py would mask. - # - # onerror collects ALL failures before raising so every broken package - # is reported in one pass, not just the first one encountered. - name: Verify all fory_compiler modules are importable working-directory: compiler run: | @@ -143,15 +124,10 @@ jobs: except Exception as e: import_errors.append(f'{m}: {e}') if import_errors: - raise ImportError( - 'Failed to import:\n' + '\n'.join(import_errors) - ) + raise ImportError('Failed to import:\n' + '\n'.join(import_errors)) print(f'All {len(mods)} module(s) verified.') " - # ast.parse validates Python syntax without executing the spec. - # Executing foryc.spec directly fails because Analysis/PYZ/EXE/COLLECT - # are PyInstaller-injected globals absent in a plain interpreter. - name: Syntax-check foryc.spec (AST parse) run: | python -c " @@ -163,38 +139,14 @@ jobs: " # ───────────────────────────────────────────────────────────────────────────── - # BUILD JOB (tags / workflow_dispatch / workflow_call only — NOT pull_request) - # - # Runner notes: - # linux-x86_64 : ubuntu-22.04 - # linux-aarch64 : ubuntu-24.04-arm (native ARM64 runner — no QEMU) - # macos-x86_64 : macos-15-intel - # macos-aarch64 : macos-15 - # windows-x86_64: windows-latest (Server 2022) + # BUILD JOB # - # --onedir mode (not --onefile): - # --onefile extracts DLLs to %TEMP% at runtime. Windows Defender on GitHub's - # hardened runners intercepts LoadLibrary at the memory-mapping level - # (PYI-xxxx ERROR_NOACCESS) even with DisableRealtimeMonitoring $true. - # --onedir pre-extracts at build time; no runtime extraction, no interception. - # - # UPX notes: - # Applied to the exe stub only, not the entire dist/foryc/ directory. - # macOS: skipped — UPX 4.x+ dropped Mach-O support entirely. - # Windows: skipped — onedir bootloader stub PE layout incompatible with - # UPX --best --lzma (exits code 1, produces no output). - # - # Phase 2 distribution note: - # --onedir produces 20-40 MB total (python311.dll + stdlib .pyc files). - # This exceeds the 10 MB crates.io per-crate limit. - # foryc-bin will distribute via zip archive or download-on-first-install. - # foryc_path in foryc-build Config must resolve to a DIRECTORY, not a - # single file — Phase 2 implementors must account for this. - # - # windows-aarch64: - # Not included in this PR. Tracked in issue #3292. - # GitHub Actions windows-11-arm runners are now available but PyInstaller - # aarch64 Windows support requires validation. Targeted for Phase 2. + # Phase 2 --onedir contract: + # The artifact directory IS the foryc package — not a single executable. + # foryc_path in foryc-build Config MUST resolve to the extracted directory + # (e.g. OUT_DIR/foryc/{version}/{os}-{arch}/) not to the exe stub inside it. + # Phase 2 MUST use download-on-first-install, NOT include_bytes! — the + # directory is 20-40 MB which exceeds crates.io's 10 MB per-crate limit. # ───────────────────────────────────────────────────────────────────────────── build: name: build / ${{ matrix.target }} @@ -272,35 +224,39 @@ jobs: choco install upx --yes --no-progress upx --version + # ── UPX version guard ──────────────────────────────────────────────────── + # aarch64 ELF compression requires UPX >= 3.96. + # ubuntu-24.04-arm ships 4.2.2 so this guard should always pass — + # but it catches an unexpected downgrade before it silently corrupts output. + - name: Verify UPX version supports target architecture + if: matrix.use_upx + shell: bash + run: | + UPX_VER=$(upx --version | awk 'NR==1{print $2}') + MAJOR=$(echo "${UPX_VER}" | cut -d. -f1) + MINOR=$(echo "${UPX_VER}" | cut -d. -f2) + echo "UPX version: ${UPX_VER}" + if [ "${MAJOR}" -lt 3 ] || \ + ([ "${MAJOR}" -eq 3 ] && [ "${MINOR}" -lt 96 ]); then + echo "FAIL: UPX ${UPX_VER} does not support aarch64 ELF (requires >= 3.96)" + exit 1 + fi + echo "PASS: UPX ${UPX_VER} supports target architecture" + # ── macOS: pin deployment target ───────────────────────────────────────── # Without MACOSX_DEPLOYMENT_TARGET, binaries built on macOS 15 embed - # LC_BUILD_VERSION minos=15.0 and silently fail on macOS ≤14 with: - # dyld: Symbol not found / incompatible library version - # 13.0 rationale: - # - Covers all active Apple Silicon + Intel users - # - Python 3.11 minimum is 10.9; 13.0 is safe - # - macOS 13 (Ventura) is the oldest release still receiving security - # patches as of 2026 + # LC_BUILD_VERSION minos=15.0 and fail on macOS ≤14. + # 13.0 = oldest macOS release still receiving security patches as of 2026. - name: Set macOS deployment target if: runner.os == 'macOS' run: echo "MACOSX_DEPLOYMENT_TARGET=13.0" >> "$GITHUB_ENV" - # ── Install build dependencies ────────────────────────────────────────── - # pyinstaller is pinned in requirements-dev.txt (>=6.0,<7.0). - # fory_compiler installed from source so PyInstaller's import tracer - # walks the actual installed package tree, matching pkgutil.walk_packages - # in foryc.spec. - name: Install PyInstaller and fory_compiler run: | python -m pip install --upgrade pip pip install -r compiler/requirements-dev.txt pip install ./compiler - # ── Verify all fory_compiler modules are importable ───────────────────── - # Same pkgutil discovery as foryc.spec — single source of truth. - # importlib.import_module guarantees each named submodule is independently - # loaded, not just its parent chain. - # onerror collects ALL failing packages before raising. - name: Verify all fory_compiler modules are importable working-directory: compiler run: | @@ -331,57 +287,72 @@ jobs: except Exception as e: import_errors.append(f'{m}: {e}') if import_errors: - raise ImportError( - 'Failed to import:\n' + '\n'.join(import_errors) - ) + raise ImportError('Failed to import:\n' + '\n'.join(import_errors)) print(f'All {len(mods)} module(s) verified.') " # ── Build ─────────────────────────────────────────────────────────────── - # Must run from compiler/ so pathex=['.'] in foryc.spec resolves - # fory_compiler/__main__.py correctly. + # SOURCE_DATE_EPOCH pins the build timestamp to the last git commit. + # Without it, PyInstaller embeds the current wall-clock time in .pyc + # metadata, making SHA256SUMS.txt different on every run of the same + # commit — breaking Phase 2's binary integrity verification model. - name: Build standalone binary with PyInstaller working-directory: compiler - run: pyinstaller foryc.spec - - # ── Verify macOS deployment target ─────────────────────────────────────── - # Asserts minos <= 13.x. Fails hard if minos cannot be parsed — a - # WARNING-and-pass would allow a minos=15.0 binary through silently. - - name: Verify macOS deployment target (minos ≤ 13.x) + run: | + SOURCE_DATE_EPOCH=$(git log -1 --format=%ct) + export SOURCE_DATE_EPOCH + echo "SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}" + pyinstaller foryc.spec + + # ── Verify macOS deployment target — ALL binaries ───────────────────────── + # Checks exe stub AND every .dylib/.so in the --onedir package. + # A single dylib with minos=15.0 causes dyld: incompatible library version + # at runtime on macOS 13/14 even if the exe stub is correctly minos=13.0. + - name: Verify macOS deployment target (minos ≤ 13.x — all binaries) if: runner.os == 'macOS' env: BINARY_PATH: ${{ matrix.binary_path }} run: | - echo "=== otool LC_BUILD_VERSION ===" - otool -l "${BINARY_PATH}" | grep -A4 LC_BUILD_VERSION \ - || otool -l "${BINARY_PATH}" | grep -A3 LC_VERSION_MIN_MACOSX \ - || true - - MINOS=$(otool -l "${BINARY_PATH}" \ - | grep -A4 LC_BUILD_VERSION \ - | awk '/minos/{print $2}' \ - | head -1) - - if [ -z "${MINOS}" ]; then - MINOS=$(otool -l "${BINARY_PATH}" \ - | grep -A3 LC_VERSION_MIN_MACOSX \ - | awk '/version/{print $2}' \ + DIST_DIR="$(dirname "${BINARY_PATH}")" + FAILURES=0 + + check_minos() { + local target="$1" MINOS MAJOR + MINOS=$(otool -l "${target}" \ + | grep -A4 LC_BUILD_VERSION \ + | awk '/minos/{print $2}' \ | head -1) - fi + if [ -z "${MINOS}" ]; then + MINOS=$(otool -l "${target}" \ + | grep -A3 LC_VERSION_MIN_MACOSX \ + | awk '/version/{print $2}' \ + | head -1) + fi + if [ -z "${MINOS}" ]; then + # No deployment target load command — static lib or non-Mach-O + echo "SKIP (no minos): ${target}" + return 0 + fi + MAJOR=$(echo "${MINOS}" | cut -d. -f1) + if [ "${MAJOR}" -gt 13 ]; then + echo "FAIL minos=${MINOS}: ${target}" + return 1 + fi + echo "PASS minos=${MINOS}: ${target}" + } - if [ -z "${MINOS}" ]; then - echo "ERROR: could not parse minos from otool output." - echo " Verify MACOSX_DEPLOYMENT_TARGET=13.0 was set during build." - exit 1 - fi + check_minos "${BINARY_PATH}" || FAILURES=$((FAILURES + 1)) + + while IFS= read -r -d '' lib; do + check_minos "${lib}" || FAILURES=$((FAILURES + 1)) + done < <(find "${DIST_DIR}" \( -name "*.dylib" -o -name "*.so" \) -print0) - MAJOR=$(echo "${MINOS}" | cut -d. -f1) - if [ "${MAJOR}" -gt 13 ]; then - echo "FAIL: minos=${MINOS} — binary requires macOS ${MAJOR}+ and will" - echo " not launch on macOS ≤13." + if [ "${FAILURES}" -gt 0 ]; then + echo "FAIL: ${FAILURES} target(s) embed minos > 13.x." + echo " Check MACOSX_DEPLOYMENT_TARGET=13.0 was honoured by the Python framework." exit 1 fi - echo "PASS: minos=${MINOS} (≤ 13.x — compatible with all active macOS releases)" + echo "PASS: all binaries/dylibs have minos ≤ 13.x" # ── Pre-compression smoke test ────────────────────────────────────────── - name: Smoke test (pre-UPX) @@ -389,12 +360,11 @@ jobs: env: BINARY_PATH: ${{ matrix.binary_path }} run: | - echo "=== Pre-UPX binary size ===" python -c " import os p = os.environ['BINARY_PATH'] s = os.path.getsize(p) - print(f'Size: {s:,} bytes ({s/1024/1024:.2f} MB)') + print(f'Pre-UPX size: {s:,} bytes ({s/1024/1024:.2f} MB)') " "${BINARY_PATH}" --help @@ -406,21 +376,18 @@ jobs: BINARY_PATH: ${{ matrix.binary_path }} run: | upx --best --lzma "${BINARY_PATH}" - echo "=== Post-UPX binary size ===" python -c " import os p = os.environ['BINARY_PATH'] s = os.path.getsize(p) - print(f'Size: {s:,} bytes ({s/1024/1024:.2f} MB)') + print(f'Post-UPX size: {s:,} bytes ({s/1024/1024:.2f} MB)') " # ── macOS aarch64: ad-hoc codesign ─────────────────────────────────────── - # Apple Silicon requires valid signatures on the exe AND every .dylib/.so - # in the --onedir package. codesign --deep does NOT traverse flat - # directories (only .app/.framework bundles). - # BINARY_PATH via env prevents quoting failures on paths with spaces. - # Verify loop covers all dylibs — an unsigned dylib causes DYLD_LIBRARY_PATH - # resolution failure at runtime even if the main exe is correctly signed. + # codesign --deep does NOT traverse flat directories (only .app bundles). + # We sign every dylib/so explicitly, then verify each one. + # codesign --verify exit codes are POSIX-defined: 0=valid, non-zero=failure. + # Ad-hoc signatures exit 0 — the if! correctly catches real failures only. - name: Ad-hoc codesign (macOS aarch64 — Apple Silicon requirement) if: matrix.codesign shell: bash @@ -429,14 +396,10 @@ jobs: run: | DIST_DIR="$(dirname "${BINARY_PATH}")" - echo "=== Signing all .dylib and .so files in ${DIST_DIR} ===" find "${DIST_DIR}" \( -name "*.dylib" -o -name "*.so" \) \ -exec codesign --force --sign - {} \; - - echo "=== Signing main executable ===" codesign --force --sign - "${BINARY_PATH}" - echo "=== Verifying all .dylib and .so signatures ===" VERIFY_FAILURES=0 while IFS= read -r -d '' lib; do if ! codesign --verify --verbose "${lib}" 2>&1; then @@ -445,7 +408,6 @@ jobs: fi done < <(find "${DIST_DIR}" \( -name "*.dylib" -o -name "*.so" \) -print0) - echo "=== Verifying main executable signature ===" codesign --verify --verbose "${BINARY_PATH}" if [ "${VERIFY_FAILURES}" -gt 0 ]; then @@ -455,8 +417,6 @@ jobs: echo "PASS: all signatures verified." # ── Final smoke test ───────────────────────────────────────────────────── - # Named "final" not "post-UPX" — UPX is skipped on 3 of 5 targets. - # Runs after all transformations (UPX, codesign) are complete. - name: Smoke test (final) shell: bash env: @@ -484,11 +444,27 @@ jobs: print('PASS') " + # ── Write VERSION file ─────────────────────────────────────────────────── + # Embeds the git ref (tag name on releases, short SHA otherwise) so + # Phase 2 foryc-build can resolve the path: + # OUT_DIR/foryc/{version}/{os}-{arch}/ + # This must run BEFORE SHA256SUMS generation so VERSION is checksummed. + - name: Write VERSION file to artifact directory + shell: bash + env: + BINARY_PATH: ${{ matrix.binary_path }} + run: | + # Tagged release → GITHUB_REF_NAME = "foryc-v0.10.0" + # workflow_dispatch / workflow_call → short commit SHA + VERSION="${GITHUB_REF_NAME:-$(git rev-parse --short HEAD)}" + echo "${VERSION}" > "$(dirname "${BINARY_PATH}")/VERSION" + echo "Artifact version: ${VERSION}" + # ── SHA-256 checksums ──────────────────────────────────────────────────── - # Generates SHA256SUMS.txt for Phase 2 foryc-build integrity verification. + # Runs AFTER VERSION file is written — VERSION is included in checksums. # Python used for cross-platform consistency (sha256sum/shasum/certutil - # are all platform-specific; Python's hashlib is not). - # chr(92) == backslash — normalises Windows paths in the output file. + # are all platform-specific; hashlib is not). + # chr(92) == backslash — normalises Windows paths in the checksum file. - name: Generate SHA-256 checksums shell: bash env: @@ -511,9 +487,6 @@ jobs: print(f'Wrote {len(lines)} checksums to SHA256SUMS.txt') " - # ── Upload artifact ────────────────────────────────────────────────────── - # retention-days: tagged releases → 90 days (Phase 2 runway) - # all other builds → 30 days - name: Upload binary artifact uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: @@ -523,12 +496,7 @@ jobs: if-no-files-found: error # ───────────────────────────────────────────────────────────────────────────── - # VALIDATE JOB (tags / workflow_dispatch / workflow_call only) - # - # All 5 generator backends validated — a generator can be correctly included - # in hiddenimports at build time but silently fail at runtime (e.g. a - # conditional import inside the generator that PyInstaller missed). - # Python is NOT set up — binary must be fully self-contained. + # VALIDATE JOB # ───────────────────────────────────────────────────────────────────────────── validate: name: validate / ${{ matrix.target }} @@ -581,18 +549,15 @@ jobs: if: runner.os != 'Windows' run: chmod +x ./artifact/${{ matrix.binary_name }} - # Pre-flight: clear error message instead of confusing foryc exit code. - name: Assert compiler/examples/demo.fdl exists shell: bash run: | if [ ! -f compiler/examples/demo.fdl ]; then echo "ERROR: compiler/examples/demo.fdl not found." - echo " This file must exist in the repository for E2E validation." exit 1 fi echo "PASS: compiler/examples/demo.fdl found." - # ── Test 1: --help ─────────────────────────────────────────────────────── - name: Validate --help output shell: bash run: | @@ -603,57 +568,47 @@ jobs: exit 1 } - # ── Test 2: FDL → Rust ─────────────────────────────────────────────────── - name: End-to-end compile demo.fdl → Rust shell: bash run: | OUT_DIR="${RUNNER_TEMP}/foryc-e2e-rust" mkdir -p "${OUT_DIR}" ./artifact/${{ matrix.binary_name }} --rust_out "${OUT_DIR}" compiler/examples/demo.fdl - ls -la "${OUT_DIR}/" RS_COUNT=$(find "${OUT_DIR}" -name "*.rs" -size +0c | wc -l) [ "${RS_COUNT}" -gt 0 ] || { echo "ERROR: no .rs files generated"; exit 1; } echo "PASS: ${RS_COUNT} .rs file(s) generated" - # ── Test 3: FDL → Java ─────────────────────────────────────────────────── - name: End-to-end compile demo.fdl → Java shell: bash run: | OUT_DIR="${RUNNER_TEMP}/foryc-e2e-java" mkdir -p "${OUT_DIR}" ./artifact/${{ matrix.binary_name }} --java_out "${OUT_DIR}" compiler/examples/demo.fdl - ls -la "${OUT_DIR}/" JAVA_COUNT=$(find "${OUT_DIR}" -name "*.java" -size +0c | wc -l) [ "${JAVA_COUNT}" -gt 0 ] || { echo "ERROR: no .java files generated"; exit 1; } echo "PASS: ${JAVA_COUNT} .java file(s) generated" - # ── Test 4: FDL → Python ───────────────────────────────────────────────── - name: End-to-end compile demo.fdl → Python shell: bash run: | OUT_DIR="${RUNNER_TEMP}/foryc-e2e-python" mkdir -p "${OUT_DIR}" ./artifact/${{ matrix.binary_name }} --python_out "${OUT_DIR}" compiler/examples/demo.fdl - ls -la "${OUT_DIR}/" PY_COUNT=$(find "${OUT_DIR}" -name "*.py" -size +0c | wc -l) [ "${PY_COUNT}" -gt 0 ] || { echo "ERROR: no .py files generated"; exit 1; } echo "PASS: ${PY_COUNT} .py file(s) generated" - # ── Test 5: FDL → C++ ──────────────────────────────────────────────────── - name: End-to-end compile demo.fdl → C++ shell: bash run: | OUT_DIR="${RUNNER_TEMP}/foryc-e2e-cpp" mkdir -p "${OUT_DIR}" ./artifact/${{ matrix.binary_name }} --cpp_out "${OUT_DIR}" compiler/examples/demo.fdl - ls -la "${OUT_DIR}/" CPP_COUNT=$(find "${OUT_DIR}" \ \( -name "*.h" -o -name "*.cc" -o -name "*.cpp" \) -size +0c | wc -l) [ "${CPP_COUNT}" -gt 0 ] || { echo "ERROR: no C++ files generated"; exit 1; } echo "PASS: ${CPP_COUNT} C++ file(s) generated" - # ── Test 6: FDL → Go ───────────────────────────────────────────────────── - # Go output may be nested in subdirs based on go_package schema option. - name: End-to-end compile demo.fdl → Go shell: bash run: | @@ -667,17 +622,6 @@ jobs: # ───────────────────────────────────────────────────────────────────────────── # SUMMARY JOB - # Single required status check for branch protection rules. - # Phase 4 release pipeline reads artifacts_ready via workflow_call. - # - # GitHub Actions result values for a conditionally skipped job (if: false): - # needs..result == "skipped" — documented, always "skipped" not "". - # - # Three valid terminal states: - # 1. Tagged/dispatch: BUILD+VALIDATE success → ready=true, exit 0 - # SPEC_CHECK = "skipped" on these events — State 1 does not check it. - # 2. PR: SPEC_CHECK success, BUILD+VALIDATE skipped → ready=false, exit 0 - # 3. Any actual failure → ready=false, exit 1 # ───────────────────────────────────────────────────────────────────────────── build-complete: name: foryc / all binaries ready @@ -697,14 +641,12 @@ jobs: echo "build: ${BUILD}" echo "validate: ${VALIDATE}" - # State 1: Full build (tag / workflow_dispatch / workflow_call) if [[ "${BUILD}" == "success" && "${VALIDATE}" == "success" ]]; then echo "ready=true" >> "${GITHUB_OUTPUT}" echo "PASS: full binary build and validation succeeded." exit 0 fi - # State 2: PR — build+validate skipped by design, spec-check ran if [[ "${BUILD}" == "skipped" && "${VALIDATE}" == "skipped" \ && "${SPEC_CHECK}" == "success" ]]; then echo "ready=false" >> "${GITHUB_OUTPUT}" @@ -712,7 +654,6 @@ jobs: exit 0 fi - # State 3: Failure echo "ready=false" >> "${GITHUB_OUTPUT}" echo "FAIL: spec-check=${SPEC_CHECK} build=${BUILD} validate=${VALIDATE}" exit 1