diff --git a/Cargo.lock b/Cargo.lock index 573a58e6..aaa40dad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -95,7 +95,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -106,7 +106,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1162,7 +1162,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1282,7 +1282,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1821,7 +1821,7 @@ dependencies = [ [[package]] name = "hyperlight-common" version = "0.16.0" -source = "git+https://github.com/hyperlight-dev/hyperlight?rev=cfa020441d880d599cc516a050e6cf10236ca5c4#cfa020441d880d599cc516a050e6cf10236ca5c4" +source = "git+https://github.com/hyperlight-dev/hyperlight?rev=90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd#90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd" dependencies = [ "anyhow", "bitflags 2.13.1", @@ -1840,7 +1840,7 @@ dependencies = [ [[package]] name = "hyperlight-component-macro" version = "0.16.0" -source = "git+https://github.com/hyperlight-dev/hyperlight?rev=cfa020441d880d599cc516a050e6cf10236ca5c4#cfa020441d880d599cc516a050e6cf10236ca5c4" +source = "git+https://github.com/hyperlight-dev/hyperlight?rev=90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd#90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd" dependencies = [ "env_logger", "hyperlight-component-util", @@ -1854,7 +1854,7 @@ dependencies = [ [[package]] name = "hyperlight-component-util" version = "0.16.0" -source = "git+https://github.com/hyperlight-dev/hyperlight?rev=cfa020441d880d599cc516a050e6cf10236ca5c4#cfa020441d880d599cc516a050e6cf10236ca5c4" +source = "git+https://github.com/hyperlight-dev/hyperlight?rev=90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd#90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd" dependencies = [ "itertools 0.15.0", "prettyplease 0.3.0", @@ -1862,16 +1862,16 @@ dependencies = [ "quote", "syn 3.0.3", "tracing", - "wasmparser 0.256.0", + "wasmparser 0.257.1", "wat", "wit-component", - "wit-parser 0.256.0", + "wit-parser 0.257.1", ] [[package]] name = "hyperlight-guest" version = "0.16.0" -source = "git+https://github.com/hyperlight-dev/hyperlight?rev=cfa020441d880d599cc516a050e6cf10236ca5c4#cfa020441d880d599cc516a050e6cf10236ca5c4" +source = "git+https://github.com/hyperlight-dev/hyperlight?rev=90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd#90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd" dependencies = [ "anyhow", "flatbuffers", @@ -1884,7 +1884,7 @@ dependencies = [ [[package]] name = "hyperlight-guest-bin" version = "0.16.0" -source = "git+https://github.com/hyperlight-dev/hyperlight?rev=cfa020441d880d599cc516a050e6cf10236ca5c4#cfa020441d880d599cc516a050e6cf10236ca5c4" +source = "git+https://github.com/hyperlight-dev/hyperlight?rev=90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd#90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd" dependencies = [ "buddy_system_allocator", "flatbuffers", @@ -1902,7 +1902,7 @@ dependencies = [ [[package]] name = "hyperlight-guest-macro" version = "0.16.0" -source = "git+https://github.com/hyperlight-dev/hyperlight?rev=cfa020441d880d599cc516a050e6cf10236ca5c4#cfa020441d880d599cc516a050e6cf10236ca5c4" +source = "git+https://github.com/hyperlight-dev/hyperlight?rev=90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd#90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1913,7 +1913,7 @@ dependencies = [ [[package]] name = "hyperlight-guest-tracing" version = "0.16.0" -source = "git+https://github.com/hyperlight-dev/hyperlight?rev=cfa020441d880d599cc516a050e6cf10236ca5c4#cfa020441d880d599cc516a050e6cf10236ca5c4" +source = "git+https://github.com/hyperlight-dev/hyperlight?rev=90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd#90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd" dependencies = [ "hyperlight-common", "spin 0.12.3", @@ -1924,7 +1924,7 @@ dependencies = [ [[package]] name = "hyperlight-host" version = "0.16.0" -source = "git+https://github.com/hyperlight-dev/hyperlight?rev=cfa020441d880d599cc516a050e6cf10236ca5c4#cfa020441d880d599cc516a050e6cf10236ca5c4" +source = "git+https://github.com/hyperlight-dev/hyperlight?rev=90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd#90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd" dependencies = [ "anyhow", "bindgen", @@ -1978,7 +1978,7 @@ dependencies = [ [[package]] name = "hyperlight-libc" version = "0.16.0" -source = "git+https://github.com/hyperlight-dev/hyperlight?rev=cfa020441d880d599cc516a050e6cf10236ca5c4#cfa020441d880d599cc516a050e6cf10236ca5c4" +source = "git+https://github.com/hyperlight-dev/hyperlight?rev=90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd#90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd" dependencies = [ "anyhow", "bindgen", @@ -2334,7 +2334,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "160f2eade097f30263b548aae5deb12ad349c909baa710fa24b92c9090b2e006" dependencies = [ "scopeguard", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -2698,7 +2698,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3513,7 +3513,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3816,7 +3816,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3936,10 +3936,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4525,16 +4525,6 @@ dependencies = [ "wasmparser 0.248.0", ] -[[package]] -name = "wasm-encoder" -version = "0.256.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec1492381bfd5ea51c2a99a919b676662559925cb8d7490547ec2e14c1ad3eb1" -dependencies = [ - "leb128fmt", - "wasmparser 0.256.0", -] - [[package]] name = "wasm-encoder" version = "0.257.1" @@ -4547,14 +4537,14 @@ dependencies = [ [[package]] name = "wasm-metadata" -version = "0.256.0" +version = "0.257.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6187f9fce741db43bea48f1ec42174897d763b8cdea7df726f2dd05561848d7" +checksum = "57460ad9bce753e8a80d47a445cb87c8724ea57f463d9c906a947c41032ce1ac" dependencies = [ "anyhow", "indexmap", - "wasm-encoder 0.256.0", - "wasmparser 0.256.0", + "wasm-encoder 0.257.1", + "wasmparser 0.257.1", ] [[package]] @@ -4583,19 +4573,6 @@ dependencies = [ "serde", ] -[[package]] -name = "wasmparser" -version = "0.256.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60bd825ffedc6cba8a642924ba7ae424afbc47811cffbcb7b92031ec24e59b4c" -dependencies = [ - "bitflags 2.13.1", - "hashbrown 0.17.1", - "indexmap", - "semver", - "serde", -] - [[package]] name = "wasmparser" version = "0.257.1" @@ -4603,8 +4580,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d92fc335fb6d48f46bda1d8b26b69e28320c15ac3272208333833d6e217e2b4a" dependencies = [ "bitflags 2.13.1", + "hashbrown 0.17.1", "indexmap", "semver", + "serde", ] [[package]] @@ -5161,7 +5140,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5501,9 +5480,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "wit-component" -version = "0.256.0" +version = "0.257.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d537ec182eed18f560184b870287cf443c4eccc315af178db3eae8811c3b6c" +checksum = "2e8b7d04a4c9491ffea354923ed70c8826d52c699fe20a52e8ceca95017dddb8" dependencies = [ "anyhow", "bitflags 2.13.1", @@ -5512,10 +5491,10 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "wasm-encoder 0.256.0", + "wasm-encoder 0.257.1", "wasm-metadata", - "wasmparser 0.256.0", - "wit-parser 0.256.0", + "wasmparser 0.257.1", + "wit-parser 0.257.1", ] [[package]] @@ -5557,9 +5536,9 @@ dependencies = [ [[package]] name = "wit-parser" -version = "0.256.0" +version = "0.257.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa073dadefba859cb03f689a0c2e3efc51f883bf86b907eaa9709bfd9e3a00a9" +checksum = "6f815340f0bb65d9775ae21e56b503b496683557b9b627b195d2766c25fb24ae" dependencies = [ "anyhow", "hashbrown 0.17.1", @@ -5571,7 +5550,7 @@ dependencies = [ "serde_derive", "serde_json", "unicode-ident", - "wasmparser 0.256.0", + "wasmparser 0.257.1", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f31f491a..f6df024c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,11 +13,11 @@ repository = "https://github.com/hyperlight-dev/hyperlight-wasm" readme = "README.md" [workspace.dependencies] -hyperlight-common = { git = "https://github.com/hyperlight-dev/hyperlight", rev = "cfa020441d880d599cc516a050e6cf10236ca5c4", default-features = false } -hyperlight-component-macro = { git = "https://github.com/hyperlight-dev/hyperlight", rev = "cfa020441d880d599cc516a050e6cf10236ca5c4" } -hyperlight-component-util = { git = "https://github.com/hyperlight-dev/hyperlight", rev = "cfa020441d880d599cc516a050e6cf10236ca5c4" } -hyperlight-guest = { git = "https://github.com/hyperlight-dev/hyperlight", rev = "cfa020441d880d599cc516a050e6cf10236ca5c4" } -hyperlight-guest-bin = { git = "https://github.com/hyperlight-dev/hyperlight", rev = "cfa020441d880d599cc516a050e6cf10236ca5c4" } -hyperlight-host = { git = "https://github.com/hyperlight-dev/hyperlight", rev = "cfa020441d880d599cc516a050e6cf10236ca5c4", default-features = false } +hyperlight-common = { git = "https://github.com/hyperlight-dev/hyperlight", rev = "90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd", default-features = false } +hyperlight-component-macro = { git = "https://github.com/hyperlight-dev/hyperlight", rev = "90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd" } +hyperlight-component-util = { git = "https://github.com/hyperlight-dev/hyperlight", rev = "90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd" } +hyperlight-guest = { git = "https://github.com/hyperlight-dev/hyperlight", rev = "90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd" } +hyperlight-guest-bin = { git = "https://github.com/hyperlight-dev/hyperlight", rev = "90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd" } +hyperlight-host = { git = "https://github.com/hyperlight-dev/hyperlight", rev = "90e7221d2c4ebe4c5bca2e036fea3fad96c6dfdd", default-features = false } hyperlight-wasm-macro = { version = "0.14.0", path = "src/hyperlight_wasm_macro" } hyperlight-wasm-runtime = { version = "0.14.0", path = "src/hyperlight_wasm_runtime" } diff --git a/rust-toolchain.toml b/rust-toolchain.toml index cb763d05..238f9bc4 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -2,4 +2,4 @@ channel = "1.94" # Target used for guest binaries. This is an additive list of targets in addition to host platform. # Will install the target if not already installed when building guest binaries. -targets = ["x86_64-unknown-none"] \ No newline at end of file +targets = ["x86_64-unknown-none", "aarch64-unknown-none"] \ No newline at end of file diff --git a/src/hyperlight_wasm/build.rs b/src/hyperlight_wasm/build.rs index d450e1e7..80f1fc82 100644 --- a/src/hyperlight_wasm/build.rs +++ b/src/hyperlight_wasm/build.rs @@ -20,7 +20,7 @@ limitations under the License. // This is done by reading the hyperlight-wasm-runtime binary into a static byte array named WASM_RUNTIME. // this build script writes the code to do that to a file named built.rs in the OUT_DIR. // this file is included in lib.rs. -// The hyperlight-wasm-runtime binary is expected to be in the x64/{config} directory. +// The hyperlight-wasm-runtime binary is expected in the cargo-hyperlight target directory. use std::fs::OpenOptions; use std::io::Write; @@ -167,8 +167,14 @@ fn build_wasm_runtime() -> PathBuf { cmd.status() .unwrap_or_else(|e| panic!("could not run cargo build hyperlight-wasm-runtime: {e:?}")); + let guest_target = match env::var("CARGO_CFG_TARGET_ARCH").as_deref() { + Ok("x86_64") => "x86_64-hyperlight-none", + Ok("aarch64") => "aarch64-hyperlight-none", + Ok(arch) => panic!("unsupported Hyperlight guest architecture: {arch}"), + Err(error) => panic!("CARGO_CFG_TARGET_ARCH is not set: {error}"), + }; let resource = target_dir - .join("x86_64-hyperlight-none") + .join(guest_target) .join(profile) .join("hyperlight-wasm-runtime"); diff --git a/src/hyperlight_wasm_aot/src/main.rs b/src/hyperlight_wasm_aot/src/main.rs index cb0ac8f9..c4df4e23 100644 --- a/src/hyperlight_wasm_aot/src/main.rs +++ b/src/hyperlight_wasm_aot/src/main.rs @@ -27,6 +27,7 @@ use wasmtime::{Config, Engine, Module, OptLevel, Precompiled}; #[derive(Debug)] enum SupportedTarget { X86_64UnknownNone, + Aarch64UnknownNone, WasmtimePulley64, } @@ -34,11 +35,20 @@ impl Display for SupportedTarget { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { SupportedTarget::X86_64UnknownNone => write!(f, "x86_64-unknown-none"), + SupportedTarget::Aarch64UnknownNone => write!(f, "aarch64-unknown-none"), SupportedTarget::WasmtimePulley64 => write!(f, "pulley64"), } } } +fn native_target() -> SupportedTarget { + if cfg!(target_arch = "aarch64") { + SupportedTarget::Aarch64UnknownNone + } else { + SupportedTarget::X86_64UnknownNone + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] enum WasmtimeVersion { Lts, @@ -105,15 +115,11 @@ fn precompile_bytes_lts( bytes: &[u8], debug: bool, minimal: bool, - pulley: bool, + target: &SupportedTarget, is_component: bool, ) -> Vec { let mut config = wasmtime_lts::Config::new(); - if pulley { - config.target("pulley64").unwrap(); - } else { - config.target("x86_64-unknown-none").unwrap(); - } + configure_lts(&mut config, target); if debug { config.debug_info(true); config.cranelift_opt_level(wasmtime_lts::OptLevel::None); @@ -133,7 +139,8 @@ fn precompile_bytes_lts( /// Detect and deserialize using the LTS wasmtime version fn detect_and_deserialize_lts(bytes: &[u8], debug: bool, file: &str) { let mut config = wasmtime_lts::Config::new(); - config.target("x86_64-unknown-none").unwrap(); + let target = get_aot_target(bytes).unwrap_or_else(|error| panic!("{error}")); + configure_lts(&mut config, &target); if debug { config.debug_info(true); config.cranelift_opt_level(wasmtime_lts::OptLevel::None); @@ -189,7 +196,7 @@ fn main() { let target = if pulley { SupportedTarget::WasmtimePulley64 } else { - SupportedTarget::X86_64UnknownNone + native_target() }; if debug { println!( @@ -213,25 +220,25 @@ fn main() { std::fs::write(outfile, serialized).unwrap(); } WasmtimeVersion::Lts => { - let target_name = if pulley { - "pulley64" + let target = if pulley { + SupportedTarget::WasmtimePulley64 } else { - "x86_64-unknown-none" + native_target() }; if debug { println!( "Aot Compiling {} to [{}]: {} with debug info and optimizations off (LTS wasmtime)", - input, target_name, outfile + input, target, outfile ); } else { println!( "Aot Compiling {} to [{}]: {} (LTS wasmtime)", - input, target_name, outfile + input, target, outfile ); } let bytes = std::fs::read(&input).unwrap(); let serialized = - precompile_bytes_lts(&bytes, debug, minimal, pulley, component); + precompile_bytes_lts(&bytes, debug, minimal, &target, component); std::fs::write(outfile, serialized).unwrap(); } } @@ -356,6 +363,9 @@ fn get_config(debug: bool, minimal: bool, target: &SupportedTarget) -> Config { // See https://github.com/bytecodealliance/wasmtime/pull/11553 unsafe { config.x86_float_abi_ok(true) }; } + SupportedTarget::Aarch64UnknownNone => { + config.target("aarch64-unknown-none").unwrap(); + } SupportedTarget::WasmtimePulley64 => { config.target("pulley64").unwrap(); } @@ -375,6 +385,10 @@ fn get_config(debug: bool, minimal: bool, target: &SupportedTarget) -> Config { config } +fn configure_lts(config: &mut wasmtime_lts::Config, target: &SupportedTarget) { + config.target(&target.to_string()).unwrap(); +} + /// Parses the AOT compiled file as an ELF file and extracts the target triple /// NOTE: These flag bits must match Wasmtime's EF_WASMTIME_PULLEY{64,32} values /// used when emitting RISC-V ELF object files. If Wasmtime changes these values, @@ -390,9 +404,7 @@ fn get_aot_target(bytes: &[u8]) -> Result { if let Ok(elf) = ElfFile64::::parse(bytes) { match elf.architecture() { Architecture::X86_64 => Ok(SupportedTarget::X86_64UnknownNone), - Architecture::Aarch64 => { - Err("Unsupported architecture Aarch64 in AOT compiled file".to_string()) - } + Architecture::Aarch64 => Ok(SupportedTarget::Aarch64UnknownNone), Architecture::S390x => { Err("Unsupported architecture S390x in AOT compiled file".to_string()) } diff --git a/src/hyperlight_wasm_runtime/build.rs b/src/hyperlight_wasm_runtime/build.rs index b8277603..c3862641 100644 --- a/src/hyperlight_wasm_runtime/build.rs +++ b/src/hyperlight_wasm_runtime/build.rs @@ -83,7 +83,11 @@ fn main() { cfg.include("src/include"); cfg.file("src/platform.c"); if cfg!(windows) { - env::set_var("AR_x86_64_unknown_none", "llvm-ar"); + match env::var("CARGO_CFG_TARGET_ARCH").as_deref() { + Ok("x86_64") => env::set_var("AR_x86_64_unknown_none", "llvm-ar"), + Ok("aarch64") => env::set_var("AR_aarch64_unknown_none", "llvm-ar"), + _ => {} + } } cfg.compile("wasmtime-hyperlight-platform"); diff --git a/src/hyperlight_wasm_runtime/src/component.rs b/src/hyperlight_wasm_runtime/src/component.rs index 986f8599..d3fcaf4d 100644 --- a/src/hyperlight_wasm_runtime/src/component.rs +++ b/src/hyperlight_wasm_runtime/src/component.rs @@ -118,7 +118,7 @@ pub extern "C" fn hyperlight_main() { // sets the Rust target to be compiled with the hard-float ABI manually via // `-Zbuild-std` and a custom target JSON configuration // See https://github.com/bytecodealliance/wasmtime/pull/11553 - #[cfg(all(not(feature = "wasmtime_lts"), not(pulley)))] + #[cfg(all(not(feature = "wasmtime_lts"), not(pulley), target_arch = "x86_64"))] unsafe { config.x86_float_abi_ok(true) }; diff --git a/src/hyperlight_wasm_runtime/src/module.rs b/src/hyperlight_wasm_runtime/src/module.rs index 1ff24b46..9f353be1 100644 --- a/src/hyperlight_wasm_runtime/src/module.rs +++ b/src/hyperlight_wasm_runtime/src/module.rs @@ -105,7 +105,7 @@ fn init_wasm_runtime(function_call: FunctionCall) -> Result> { // sets the Rust target to be compiled with the hard-float ABI manually via // `-Zbuild-std` and a custom target JSON configuration // See https://github.com/bytecodealliance/wasmtime/pull/11553 - #[cfg(all(not(feature = "wasmtime_lts"), not(pulley)))] + #[cfg(all(not(feature = "wasmtime_lts"), not(pulley), target_arch = "x86_64"))] unsafe { config.x86_float_abi_ok(true) }; diff --git a/src/hyperlight_wasm_runtime/src/platform.rs b/src/hyperlight_wasm_runtime/src/platform.rs index 364ddae6..c1988633 100644 --- a/src/hyperlight_wasm_runtime/src/platform.rs +++ b/src/hyperlight_wasm_runtime/src/platform.rs @@ -18,15 +18,25 @@ use core::ffi::c_void; use core::ptr::NonNull; use core::sync::atomic::{AtomicPtr, AtomicU64, Ordering}; +#[cfg(target_arch = "aarch64")] +use hyperlight_common::arch::exn::{DataFault, DataFaultKind, Exception}; use hyperlight_common::vmem; use hyperlight_guest_bin::exception::arch; use hyperlight_guest_bin::paging; +const FIRST_VADDR_BASE: u64 = 0x100_0000_0000; +const VADDR_SLOT_SIZE: u64 = 0x100_0000_0000; // Extremely stupid virtual address allocator // 0x1_0000_0000 is where the module is // we start at // 0x100_0000_0000 and go up from there -static FIRST_VADDR: AtomicU64 = AtomicU64::new(0x100_0000_0000u64); +static FIRST_VADDR: AtomicU64 = AtomicU64::new(FIRST_VADDR_BASE); + +fn is_wasmtime_address(address: u64) -> bool { + (FIRST_VADDR_BASE..FIRST_VADDR.load(Ordering::Acquire)).contains(&address) +} + +#[cfg(target_arch = "x86_64")] fn page_fault_handler( _exception_number: u64, info: *mut arch::ExceptionInfo, @@ -40,7 +50,7 @@ fn page_fault_handler( // TODO: replace this with some generic virtual memory area data // structure in hyperlight core - if (error_code & 0x1) == 0x0 && page_fault_address >= 0x100_0000_0000u64 { + if (error_code & 0x1) == 0x0 && is_wasmtime_address(page_fault_address) { unsafe { let phys_page = hyperlight_guest::prim_alloc::alloc_phys_pages(1); let virt_base = (page_fault_address & !0xFFF) as *mut u8; @@ -60,6 +70,78 @@ fn page_fault_handler( } false } + +#[cfg(target_arch = "aarch64")] +fn redirect_to_wasmtime_trap( + far: u64, + elr: &mut u64, + registers: &mut [u64; 31], + has_faulting_addr: bool, +) -> bool { + let requested_handler = WASMTIME_REQUESTED_TRAP_HANDLER.load(Ordering::Relaxed); + if requested_handler == 0 { + return false; + } + + registers[0] = *elr; + registers[1] = registers[29]; + registers[2] = has_faulting_addr as u64; + registers[3] = far; + *elr = requested_handler; + true +} + +#[cfg(target_arch = "aarch64")] +fn aarch64_exception_handler( + exception: Exception, + esr: u64, + far: u64, + elr: &mut u64, + registers: &mut [u64; 31], +) -> bool { + if matches!( + exception, + Exception::DataFault(DataFault { + from_lower_el: false, + kind: DataFaultKind::TranslationFault(_), + .. + }) + ) && is_wasmtime_address(far) + { + unsafe { + let phys_page = hyperlight_guest::prim_alloc::alloc_phys_pages(1); + let virt_base = (far & !((vmem::PAGE_SIZE - 1) as u64)) as *mut u8; + paging::map_region( + phys_page, + virt_base, + vmem::PAGE_SIZE as u64, + vmem::MappingKind::Basic(vmem::BasicMapping { + readable: true, + writable: true, + executable: true, + }), + ); + paging::barrier::first_valid_same_ctx(); + virt_base.write_bytes(0, vmem::PAGE_SIZE); + } + return true; + } + + match exception { + // UDF instructions report an unknown-reason exception class. + Exception::Other(_) if esr >> 26 == 0 => { + let instruction = unsafe { (*elr as *const u32).read_volatile() }; + if instruction == 0x0000_c11f { + redirect_to_wasmtime_trap(far, elr, registers, false) + } else { + false + } + } + _ => false, + } +} + +#[cfg(target_arch = "x86_64")] pub(crate) fn register_page_fault_handler() { // On amd64, vector 14 is #PF // See AMD64 Architecture Programmer's Manual, Volume 2 @@ -70,7 +152,10 @@ pub(crate) fn register_page_fault_handler() { Ordering::Release, ); } - +#[cfg(target_arch = "aarch64")] +pub(crate) fn register_page_fault_handler() { + arch::register_exception_handler(aarch64_exception_handler); +} // Wasmtime Embedding Interface /* We don't actually have any sensible virtual memory areas, so @@ -80,10 +165,10 @@ pub(crate) fn register_page_fault_handler() { * (see above) */ #[no_mangle] pub extern "C" fn wasmtime_mmap_new(_size: usize, _prot_flags: u32, ret: &mut *mut u8) -> i32 { - if _size > 0x100_0000_0000 { + if _size > VADDR_SLOT_SIZE as usize { panic!("wasmtime_mmap_{:x} {:x}", _size, _prot_flags); } - *ret = FIRST_VADDR.fetch_add(0x100_0000_0000, Ordering::Relaxed) as *mut u8; + *ret = FIRST_VADDR.fetch_add(VADDR_SLOT_SIZE, Ordering::Relaxed) as *mut u8; 0 } @@ -93,7 +178,7 @@ pub extern "C" fn wasmtime_mmap_new(_size: usize, _prot_flags: u32, ret: &mut *m * as we don't properly implement permissions at the moment. */ #[no_mangle] pub extern "C" fn wasmtime_mmap_remap(addr: *mut u8, size: usize, prot_flags: u32) -> i32 { - if size > 0x100_0000_0000 { + if size > VADDR_SLOT_SIZE as usize { panic!( "wasmtime_mmap_remap {:x} {:x} {:x}", addr as usize, size, prot_flags @@ -127,6 +212,7 @@ pub extern "C" fn wasmtime_page_size() -> usize { type wasmtime_trap_handler_t = extern "C" fn(ip: usize, fp: usize, has_faulting_addr: bool, faulting_addr: usize); static WASMTIME_REQUESTED_TRAP_HANDLER: AtomicU64 = AtomicU64::new(0); +#[cfg(target_arch = "x86_64")] fn wasmtime_trap_handler( exception_number: u64, info: *mut arch::ExceptionInfo, @@ -157,6 +243,7 @@ fn wasmtime_trap_handler( false } +#[cfg(target_arch = "x86_64")] #[no_mangle] pub extern "C" fn wasmtime_init_traps(handler: wasmtime_trap_handler_t) -> i32 { WASMTIME_REQUESTED_TRAP_HANDLER.store(handler as usize as u64, Ordering::Relaxed); @@ -177,19 +264,21 @@ pub extern "C" fn wasmtime_init_traps(handler: wasmtime_trap_handler_t) -> i32 { // takes over the exception. 0 } - +#[cfg(target_arch = "aarch64")] +#[no_mangle] +pub extern "C" fn wasmtime_init_traps(handler: wasmtime_trap_handler_t) -> i32 { + WASMTIME_REQUESTED_TRAP_HANDLER.store(handler as usize as u64, Ordering::Relaxed); + 0 +} // Copy a VA range to a new VA. Old and new VA, and len, must be // page-aligned. fn copy_va_mapping(base: *const u8, len: usize, to_va: *mut u8, remap_original: bool) { debug_assert!((base as usize).is_multiple_of(vmem::PAGE_SIZE)); debug_assert!(len.is_multiple_of(vmem::PAGE_SIZE)); - // TODO: all this barrier code is amd64 specific. It should be - // refactored to use some better architecture-independent APIs. - // - // On amd64, "upgrades" including the first time that a a valid - // translation exists for a VA, only need a light (serialising - // instruction) barrier. Since invlpg is also a barrier, we don't - // even need that, if we did do a downgrade remap just before. + // On amd64, first-valid mappings need only a serialising barrier. + // On Aarch64, both possible destination replacements and source + // downgrades use TLBI. + #[cfg(target_arch = "x86_64")] let mut needs_first_valid_exposure_barrier = false; // TODO: make this more efficient by directly exposing the ability @@ -249,21 +338,51 @@ fn copy_va_mapping(base: *const u8, len: usize, to_va: *mut u8, remap_original: new_kind, ); } + #[cfg(target_arch = "aarch64")] + invalidate_page(to_va.wrapping_add((mapping.virt_base - base_u) as usize) as u64); if do_downgrade { // Since we have downgraded a page from writable to CoW we // need to do an invlpg on it. Because invlpg is a // serialising instruction, we don't need the other // barrier for the new mapping. + #[cfg(target_arch = "x86_64")] unsafe { core::arch::asm!("invlpg [{}]", in(reg) mapping.virt_base, options(readonly, nostack, preserves_flags)); } - needs_first_valid_exposure_barrier = false; + #[cfg(target_arch = "aarch64")] + invalidate_page(mapping.virt_base); + #[cfg(target_arch = "x86_64")] + { + needs_first_valid_exposure_barrier = false; + } } else { - needs_first_valid_exposure_barrier = true; + #[cfg(target_arch = "x86_64")] + { + needs_first_valid_exposure_barrier = true; + } } } - if needs_first_valid_exposure_barrier { - paging::barrier::first_valid_same_ctx(); + #[cfg(target_arch = "x86_64")] + { + if needs_first_valid_exposure_barrier { + paging::barrier::first_valid_same_ctx(); + } + } +} + +#[cfg(target_arch = "aarch64")] +fn invalidate_page(virt: u64) { + unsafe { + core::arch::asm!( + " + dsb ish + tlbi vae1is, {page} + dsb ish + isb + ", + page = in(reg) (virt >> 12), + options(readonly, nostack, preserves_flags) + ); } } @@ -280,7 +399,7 @@ pub extern "C" fn wasmtime_memory_image_new( // identifier. We will construct the image by mapping a copy of // the original VA range here, making the original copy CoW as we // go. - let new_virt = FIRST_VADDR.fetch_add(0x100_0000_0000, Ordering::Relaxed) as *mut u8; + let new_virt = FIRST_VADDR.fetch_add(VADDR_SLOT_SIZE, Ordering::Relaxed) as *mut u8; copy_va_mapping(ptr, len, new_virt, true); *ret = new_virt as *mut c_void; 0 @@ -326,6 +445,8 @@ impl wasmtime::CustomCodeMemory for WasmtimeCodeMemory { _ptr: *const u8, _len: usize, ) -> core::result::Result<(), wasmtime::Error> { + #[cfg(target_arch = "aarch64")] + sync_instruction_cache(_ptr, _len); Ok(()) } fn unpublish_executable( @@ -337,6 +458,58 @@ impl wasmtime::CustomCodeMemory for WasmtimeCodeMemory { } } +#[cfg(target_arch = "aarch64")] +fn sync_instruction_cache(base: *const u8, len: usize) { + unsafe { + let ctr_el0: u64; + core::arch::asm!("mrs {}, ctr_el0", out(reg) ctr_el0); + let iminline = 4 * (1 << (ctr_el0 & 0xf)); + let dminline = 4 * (1 << ((ctr_el0 >> 16) & 0xf)); + // The loads and branches let Hyperlight's KVM path emulate trapped + // cache-maintenance instructions and resume at the correct loop point. + core::arch::asm!( + " + ldr xzr, [{addr}] + msr nzcv, xzr + b 2f + + 0: ldr xzr, [{tmp}] + msr nzcv, xzr + b 3f + 1: ldr xzr, [{tmp}] + msr nzcv, xzr + b 4f + + 2: mov {tmp}, {addr} + + 3: dc cvau, {tmp} + b.eq 0b + add {tmp}, {tmp}, {dminline:x} + cmp {tmp}, {max} + b.lt 3b + + dsb ish + + mov {tmp}, {addr} + + 4: ic ivau, {tmp} + b.eq 1b + add {tmp}, {tmp}, {iminline:x} + cmp {tmp}, {max} + b.lt 4b + + dsb ish + isb + ", + iminline = in(reg) iminline, + dminline = in(reg) dminline, + addr = in(reg) base as usize, + max = in(reg) base as usize + len, + tmp = out(reg) _ + ); + } +} + pub(crate) unsafe fn map_buffer(phys: u64, len: u64) -> NonNull<[u8]> { // TODO: Use a VA allocator let virt = phys as *mut u8;