diff --git a/.cargo/config.toml b/.cargo/config.toml index e15f5077..17f9915b 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,8 @@ # `cargo wasm` build edge-python release WebAssembly artifact. [alias] wasm = "build --release --target wasm32-unknown-unknown -p edge-python" + +# `--profile native` speed-first build of the native port binaries. +[profile.native] +inherits = "release" +opt-level = 3 diff --git a/.github/actions/cdn-deploy/action.yml b/.github/actions/cdn-deploy/action.yml index e5bb142b..a99310f4 100644 --- a/.github/actions/cdn-deploy/action.yml +++ b/.github/actions/cdn-deploy/action.yml @@ -32,6 +32,13 @@ runs: path: /tmp/cli/ merge-multiple: true + - name: Download native artifacts + uses: actions/download-artifact@v8 + with: + pattern: native-* + path: /tmp/native/ + merge-multiple: true + - name: Install brotli shell: bash run: command -v brotli >/dev/null || (sudo apt-get update && sudo apt-get install -y brotli) @@ -39,7 +46,7 @@ runs: - name: Stage site shell: bash run: | - mkdir -p _site/runtime _site/std _site/host _site/cli + mkdir -p _site/runtime _site/std _site/host _site/cli _site/native # compiler: served at /compiler.wasm as brotli-11; _headers tags the encoding. brotli -q 11 -c /tmp/compiler/compiler.wasm > _site/compiler.wasm @@ -68,6 +75,9 @@ runs: cp -r /tmp/cli/. _site/cli/ cp cli/setup/install.sh cli/setup/uninstall.sh _site/cli/ + # native: the runner tarball plus std plugin .so files, served under /native/. + cp -r /tmp/native/. _site/native/ + - name: Report served size shell: bash run: | diff --git a/.github/actions/native/action.yml b/.github/actions/native/action.yml new file mode 100644 index 00000000..4211249d --- /dev/null +++ b/.github/actions/native/action.yml @@ -0,0 +1,127 @@ +name: Native port +description: Host-less native builds. runner = the edge-native bin, std packages = plugin cdylibs. CDN staging + tag Release upload. + +inputs: + package: + description: "runner | json | re | math | struct" + required: true + target: + description: Rust target triple. + required: true + github-token: + description: Token for the tag Release upload (tags only). + required: false + default: "" + +runs: + using: composite + steps: + # Nightly rotates daily; its manifest hash keys an exact-version toolchain cache. + - name: Toolchain cache key + id: toolchain + shell: bash + run: echo "nightly=$(curl -fsSL https://static.rust-lang.org/dist/channel-rust-nightly.toml.sha256 | cut -c1-16)" >> "$GITHUB_OUTPUT" + + # Restoring update-hashes lets rustup see the toolchain as current and skip reinstalling. + - name: Cache toolchain + uses: actions/cache@v5 + with: + path: | + ~/.rustup/toolchains + ~/.rustup/update-hashes + key: rustup-${{ inputs.package == 'runner' && 'runner' || 'std' }}-${{ inputs.target }}-${{ steps.toolchain.outputs.nightly }} + + - name: Toolchain (stable) + if: inputs.package == 'runner' + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ inputs.target }} + components: clippy + + # build-std needs nightly plus rust-src. + - name: Toolchain (nightly) + uses: dtolnay/rust-toolchain@nightly + with: + targets: ${{ inputs.target }} + components: rust-src + + - name: Cache Rust (runner) + if: inputs.package == 'runner' + uses: Swatinem/rust-cache@v2 + with: + key: native-${{ inputs.target }} + + - name: Cache Rust (std) + if: inputs.package != 'runner' + uses: Swatinem/rust-cache@v2 + with: + workspaces: std/${{ inputs.package }} -> std/${{ inputs.package }}/target + key: native-${{ inputs.target }} + + # The bin sits behind `required-features`, so the workspace-wide lint job never covers it. + - name: Clippy (runner) + if: inputs.package == 'runner' + shell: bash + run: cargo +stable clippy --bin edge-native --no-default-features --features native --target "${{ inputs.target }}" -- -D warnings + + - name: Build runner + if: inputs.package == 'runner' + shell: bash + run: | + RUSTFLAGS="-Z location-detail=none -Z fmt-debug=none -Z unstable-options -C panic=immediate-abort" \ + cargo +nightly build \ + --profile native \ + --no-default-features --features native \ + --bin edge-native \ + --target "${{ inputs.target }}" \ + -Z build-std=std,panic_abort + + - name: Smoke test (runner) + if: inputs.package == 'runner' + shell: bash + run: | + printf 'print("native ok")\n' > /tmp/smoke.py + "target/${{ inputs.target }}/native/edge-native" /tmp/smoke.py | grep -qx "native ok" + + - name: Build std package + if: inputs.package != 'runner' + shell: bash + working-directory: std/${{ inputs.package }} + run: | + RUSTFLAGS="-Z location-detail=none -Z fmt-debug=none -Z unstable-options -C panic=immediate-abort" \ + cargo +nightly build \ + --profile native \ + --target "${{ inputs.target }}" \ + -Z build-std=std,panic_abort + + # Assets carry the short arch, the lib glob absorbs keyword-crate names like libedge_struct.so. + - name: Package + shell: bash + env: + TARGET: ${{ inputs.target }} + run: | + ARCH="${TARGET%%-*}" + mkdir -p _native + if [ "${{ inputs.package }}" = "runner" ]; then + cp "target/${TARGET}/native/edge-native" "_native/edge-${ARCH}" + tar -C _native -czf "_native/edge-${ARCH}.tar.gz" "edge-${ARCH}" + rm "_native/edge-${ARCH}" + else + cp "std/${{ inputs.package }}/target/${TARGET}/native/"lib*.so "_native/${{ inputs.package }}-${ARCH}.so" + fi + + # The `native-` prefix is the glob cdn-deploy stages under /native/. + - name: Upload artifact + uses: actions/upload-artifact@v6 + with: + name: native-${{ inputs.package }}-${{ inputs.target }} + path: _native/ + retention-days: 1 + + - name: Upload Release assets + if: startsWith(github.ref, 'refs/tags/') + uses: softprops/action-gh-release@v3 + with: + token: ${{ inputs.github-token }} + files: _native/* + fail_on_unmatched_files: true diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c1564de6..3e115e71 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -86,6 +86,29 @@ jobs: with: package: ${{ matrix.package }} + # Host-less native port, the runner bin plus std plugin cdylibs. Assets ship to the CDN and tag Releases. + native: + name: Native / ${{ matrix.package }} (${{ matrix.target }}) + needs: [compiler-check] + permissions: + contents: write + strategy: + fail-fast: false + matrix: + package: [runner, json, re, math, struct] + target: [aarch64-unknown-linux-gnu] + include: + - target: aarch64-unknown-linux-gnu + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/native + with: + package: ${{ matrix.package }} + target: ${{ matrix.target }} + github-token: ${{ github.token }} + # Clippy + check once (no matrix); gates the release build. cli-lint: name: CLI / Lint and Check @@ -132,7 +155,7 @@ jobs: cdn: name: Cloudflare Upload (CDN) - needs: [cli-test] + needs: [cli-test, native] if: github.event_name == 'push' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest steps: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a8f9dabe..947e1415 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,7 +21,7 @@ Run the following commands before sending a pull request to ensure code quality: The test suite (`tests/`, fixtures in `tests/cases/vm.json`) runs every case under `Limits::sandbox()`, not the default `none()`. The budget, heap, and call-depth guards short-circuit under `none` (`sandbox_off`), so only the bounded profile exercises them — that way a regression that lets a loop run unbounded, recurse without limit, or materialise an oversized collection becomes a failing `MemoryError` / `RecursionError` assertion instead of a hang. Every fixture must stay within the sandbox budget. -*Other packages have their own build and test setup — see the `README.md` in the relevant path. Code comments must be a single line of fewer than 30 words; if a change is too large, add a short section to the corresponding page under `./docs/content` instead.* +*Other packages have their own build and test setup — see the `README.md` in the relevant path. Code comments must be a single line of at most 17 words, never two comment lines in a row. If a change needs more, add a short section to the corresponding page under `./docs/content` instead.* `cli/` tests hit the CDN runtime by default; `EDGE_RUNTIME_DIR` and `EDGE_COMPILER_WASM` swap in local copies for end-to-end validation before a deploy: diff --git a/Cargo.toml b/Cargo.toml index bc00e702..0f16965f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,13 @@ path = "tests/main.rs" [features] default = ["prebuilt"] prebuilt = [] +# Gates the host-less native runner bin out of lib and wasm builds. +native = [] + +[[bin]] +name = "edge-native" +path = "src/bin/native.rs" +required-features = ["native"] [dependencies] wasm-abi = { workspace = true } diff --git a/README.md b/README.md index f898cc89..88c235c9 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ Cargo workspace rooted at the engine crate (`edge-python`); commands work from a ├── js ├── pdk ├── src +│ ├── bin │ ├── lexer │ ├── packages │ ├── parser @@ -51,6 +52,7 @@ Cargo workspace rooted at the engine crate (`edge-python`); commands work from a ```bash cargo wasm # release .wasm (the distributed artifact) cargo build --release # host .rlib + cdylib for Rust embedders +cargo build --profile native --no-default-features --features native # host-less native runner cargo test --release --no-default-features # run the compiler test suite ``` @@ -108,6 +110,24 @@ The runtime spawns a Web Worker that pre-fetches imports, dispatches native call Edge Python is a `cdylib`: a Rust host can instantiate `compiler.wasm` and call its exports directly, the same `.wasm` that ships to browsers; the host owns I/O. Declaring `edge-python` as a Cargo dependency fetches the matching release `.wasm` automatically (exposed as `DEP_COMPILER_WASM`), see [Consuming the release](https://edgepython.com/reference/wasm-abi#consuming-the-release-from-a-rust-crate). To add native modules from your own crate, implement the `Resolver` trait, see [Writing modules](https://edgepython.com/reference/writing-modules). +### Native + +Tagged releases attach the engine as a host-less native executable, plus each std package as a native plugin library; the CDN serves the same assets under `/native/` ([native port](https://edgepython.com/reference/native)). + +```python +# hello.py +async def greet(name): + await sleep(0.1) + print(f"hello {name}") + +await greet("edge") +``` + +```text +$ ./edge-aarch64 hello.py +hello edge +``` + ## What it is Edge Python targets sandboxed in-browser execution: a dynamic, multi-paradigm Python subset with classes, async/await, structural pattern matching, and compile-time module resolution. There is no bundled stdlib, modules are external artifacts. diff --git a/docs/content/reference/_meta.js b/docs/content/reference/_meta.js index d8cd555d..819ddcad 100644 --- a/docs/content/reference/_meta.js +++ b/docs/content/reference/_meta.js @@ -7,5 +7,6 @@ export default { 'writing-modules': 'Writing modules', cli: 'CLI', 'wasm-abi': 'WASM ABI', + native: 'Native port', 'limits-and-errors': 'Limits and errors', } diff --git a/docs/content/reference/native.md b/docs/content/reference/native.md new file mode 100644 index 00000000..e3654769 --- /dev/null +++ b/docs/content/reference/native.md @@ -0,0 +1,34 @@ +--- +title: "Native port" +description: "Host-less native builds of the engine and std packages, their release assets, runtime contract, and source builds." +--- + +Each tagged GitHub Release attaches native aarch64 builds alongside `compiler.wasm`, and the CDN serves the same assets under `https://cdn.edgepython.com/native/`, refreshed on every push to `main`. No host runtime ships with them yet. The executable runs with engine defaults plus a real wall clock, and the std libraries wait for an embedding host to load them. + +| Asset | Contents | +| --- | --- | +| `edge-aarch64.tar.gz` | `edge-aarch64`, the engine as a host-less script runner | +| `json-aarch64.so` `re-aarch64.so` `math-aarch64.so` `struct-aarch64.so` | std packages as native plugin libraries | + +## The runner + +`edge-aarch64 file.py` parses and runs one script. The exit-code contract matches [`edge run`](/reference/cli#edge-run-run-a-python-file). Output streams to stdout, an uncaught error prints its traceback to stderr and exits 1, and `raise SystemExit(code)` exits cleanly with that code. Everything executes under [sandbox limits](/reference/limits-and-errors#sandbox-limits). A wall clock drives the scheduler, so `sleep()` and timeouts wait in real time, matching the web runtime. + +The missing host shows up in three places. + +- Every `import` fails with `module not found (no resolver configured)`, so there is no module loading of any kind ([imports](/reference/imports)). +- [`input()`](/reference/builtins#input) drains piped stdin, one line per call, and an empty buffer raises `RuntimeError`. +- `frame()`, `receive()`, and native-module calls park the scheduler with no host to resume it, and the runner exits 1 with `script suspended awaiting ...`. + +## std packages as native libraries + +The `.so` assets are the same crates shipped as `.wasm` [plugin modules](/reference/writing-modules), compiled for the host platform instead. Each exports the [plugin ABI](/reference/wasm-abi) surface (`__edge_abi_version`, `__fn_`, the allocator pair) and leaves the six `env` imports (`edge_op`, `edge_encode`, ...) undefined for the embedding host to provide at load time. + +## Building from source + +```bash +cargo build --profile native --no-default-features --features native --bin edge-native +cd std/json && cargo build --profile native # any std package +``` + +The `native` profile inherits `release` and raises `opt-level` to 3. The release profile tunes for `.wasm` size at a real runtime cost, a trade that makes no sense off the wire. CI builds each target in [`actions/native`](https://github.com/dylan-sutton-chavez/edge-python/tree/main/.github/actions/native) and adds the same nightly size flags as the `.wasm` release, which cuts the runner about 22% and the std libraries about 3x versus the plain commands above. diff --git a/src/bin/native.rs b/src/bin/native.rs new file mode 100644 index 00000000..95a29627 --- /dev/null +++ b/src/bin/native.rs @@ -0,0 +1,94 @@ +use compiler::lexer::lex; +use compiler::parser::{Diagnostic, Parser}; +use compiler::vm::types::{SchedulerStatus, VmErr}; +use compiler::vm::{Limits, VM}; +use std::io::{IsTerminal, Read, Write}; +use std::process::ExitCode; + +// Streams one print payload to stdout, flushed so it interleaves with stderr. +fn stream_stdout(s: &str) { + let mut out = std::io::stdout().lock(); + let _ = out.write_all(s.as_bytes()); + let _ = out.flush(); +} + +// Wall-clock ns, the same base PendingTimer deadlines are minted against. +fn now_ns() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |d| d.as_nanos() as u64) +} + +fn pending_name(status: &SchedulerStatus) -> &'static str { + match status { + SchedulerStatus::PendingTimer(_) => "a timer", + SchedulerStatus::PendingFrame => "a render frame", + SchedulerStatus::PendingEvent => "an event", + SchedulerStatus::PendingHostCall => "a host call", + SchedulerStatus::Preempted => "a preempt resume", + SchedulerStatus::Done => "nothing", + } +} + +fn main() -> ExitCode { + let Some(path) = std::env::args().nth(1) else { + eprintln!("usage: edge-native "); + return ExitCode::from(2); + }; + let src = match std::fs::read_to_string(&path) { + Ok(s) => s, + Err(e) => { + eprintln!("error: cannot read '{path}': {e}"); + return ExitCode::from(2); + } + }; + let (tokens, lex_errs) = lex(&src); + let mut parser = Parser::new(&src, tokens.into_iter()); + for e in lex_errs { + parser.errors.push(Diagnostic { start: e.start, end: e.end, msg: e.msg.into() }); + } + let (mut chunk, errs) = parser.parse(); + if !errs.is_empty() { + for e in &errs { + eprintln!("{}", e.render(&src, Some(&path))); + } + return ExitCode::FAILURE; + } + compiler::vm::optimizer::constant_fold(&mut chunk); + let mut vm = VM::with_limits(&chunk, Limits::sandbox()); + vm.strict_input = true; + vm.print_hook = Some(stream_stdout); + vm.set_time_hook(now_ns); + let mut stdin = std::io::stdin(); + if !stdin.is_terminal() { + let mut buf = String::new(); + // Piped stdin mirrors the wasm host `set_input` line split. + if stdin.read_to_string(&mut buf).is_ok() && !buf.is_empty() { + vm.input_buffer = buf.split('\n').map(String::from).collect(); + } + } + // Serve timer parks like the web driver: sleep to the deadline, resume. + let result = loop { + match vm.run() { + Err(VmErr::HostYield(SchedulerStatus::PendingTimer(deadline))) => { + let now = now_ns(); + if deadline > now { std::thread::sleep(std::time::Duration::from_nanos(deadline - now)); } + } + other => break other, + } + }; + match result { + Ok(_) | Err(VmErr::HostYield(SchedulerStatus::Done)) => ExitCode::SUCCESS, + Err(VmErr::HostYield(status)) => { + eprintln!("error: script suspended awaiting {}, no host wired in this binary", pending_name(&status)); + ExitCode::FAILURE + } + Err(e) => { + if let Some(code) = vm.system_exit_code() { + return ExitCode::from((code & 0xFF) as u8); + } + let traceback = e.render_traceback(&src, vm.error_pos(), Some(&path), vm.call_stack_frames(), vm.function_names_ref()); + eprintln!("{traceback}"); + ExitCode::FAILURE + } + } +}