diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8de7f49..37d1c60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: run: bun test - name: Check remaining shell syntax - run: bash -n completions/rootcell.bash src/bin/rootcell-vmnet-helper.sh + run: bash -n completions/rootcell.bash images/scripts/build-release.sh images/scripts/validate-dist.sh - name: Install ShellCheck run: | @@ -54,7 +54,7 @@ jobs: sudo apt-get install -y shellcheck - name: Run ShellCheck - run: shellcheck --severity=warning completions/rootcell.bash src/bin/rootcell-vmnet-helper.sh + run: shellcheck --severity=warning completions/rootcell.bash images/scripts/build-release.sh images/scripts/validate-dist.sh - name: Compile Python modules run: python3 -m compileall proxy diff --git a/.github/workflows/macos-host-build.yml b/.github/workflows/macos-host-build.yml index f192e4f..4fbdbb6 100644 --- a/.github/workflows/macos-host-build.yml +++ b/.github/workflows/macos-host-build.yml @@ -56,5 +56,5 @@ jobs: - name: Build Darwin-only host packages run: | nix build --print-build-logs \ - .#packages.aarch64-darwin.lima \ - .#packages.aarch64-darwin.socket_vmnet + .#packages.aarch64-darwin.vfkit \ + .#packages.aarch64-darwin.zstd diff --git a/BRAND.md b/BRAND.md index c0ee216..069c2ac 100644 --- a/BRAND.md +++ b/BRAND.md @@ -53,9 +53,9 @@ Prefer: Avoid leading with: - TLS MITM -- socket_vmnet +- VM runtime internals - NixOS module internals -- Lima named-network details +- provider-specific network details - provider-specific setup Those details matter, but they belong after the reader understands what the diff --git a/README.md b/README.md index abfd53f..cc2b7b7 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ rootcell gives you a local workspace where an agent can exercise root inside the VM without receiving broad access to your Mac: - A fresh NixOS VM for the agent's shell and tools. -- No default host-home mount from Lima. +- No host-home mount in the agent VM. - A separate firewall VM with the only public internet route. - DNS, HTTPS, and SSH allowlists you can review and hot-reload. - A per-VM SSH key for Git pushes. @@ -62,7 +62,7 @@ The two VMs have different jobs: Rootcell supports named instances. Plain `./rootcell` uses the `default` instance and creates VMs named `agent` and `firewall`. `./rootcell --instance dev` creates `agent-dev` and `firewall-dev`, with separate CA material, -allowlists, Keychain mappings, and a separate vmnet network. +allowlists, Keychain mappings, and a separate private VM link. HTTPS egress is transparent from inside the agent VM. A normal command like `curl https://github.com` either works because the host is allowlisted, or fails @@ -112,24 +112,14 @@ First run downloads compatible rootcell VM images from the configured release manifest, creates instance-local vfkit disks, and provisions the VMs. Later runs normally take seconds. -### VM Provider Selection +### Host Runtime -vfkit is the default VM provider: +vfkit is the supported VM runtime: ```bash ./rootcell ``` -The Lima provider remains as a rollback path while vfkit support settles: - -```bash -ROOTCELL_VM_PROVIDER=lima ./rootcell -``` - -The legacy Lima path still requires the one-time `socket_vmnet` and -`rootcell-vmnet` sudo setup printed by `./rootcell` when that provider is -selected. - Image resolution is controlled by: ```bash @@ -282,8 +272,6 @@ common.nix shared NixOS config for both VMs agent-vm.nix agent VM network and trust-store config firewall-vm.nix firewall VM services and nftables rules home.nix pi, Git, SSH, and developer tools for the agent VM -nixos.yaml Lima config for the agent VM -firewall.yaml Lima config for the firewall VM network.nix default inter-VM network settings .env.defaults seed values for per-instance `.env` secrets.env.defaults seed Keychain secret mappings for per-instance `secrets.env` @@ -293,7 +281,6 @@ proxy/ allowlists and mitmproxy/dnsmasq firewall code agent_spy_tui.py Textual browser for `./rootcell spy --tui` pi/agent/ global pi instructions, skills, and extensions completions/ bash and zsh completion for `rootcell` -pkgs/socket_vmnet.nix local package for Lima's vmnet helper ``` ## VM Lifecycle @@ -303,36 +290,6 @@ control key and generated SSH config live under `.rootcell/instances//ssh/ The agent VM is reached through SSH ProxyJump via the firewall VM; no VSOCK device is attached on the vfkit path. -### Lima Rollback - -The commands below apply to the legacy Lima provider when run with -`ROOTCELL_VM_PROVIDER=lima`. - -Stop the VMs but keep their disks and Nix store caches: - -```bash -limactl stop agent -limactl stop firewall -./rootcell - -limactl stop agent-dev firewall-dev -./rootcell --instance dev -``` - -Delete the VMs for a clean slate: - -```bash -limactl delete agent firewall --force -./rootcell - -limactl delete agent-dev firewall-dev --force -./rootcell --instance dev -``` - -If you edit `nixos.yaml` or `firewall.yaml`, Lima will not apply those changes -to existing VMs automatically. Either run `limactl edit ` or delete and -recreate the VM. - ## Configuration ### Environment @@ -347,10 +304,10 @@ ROOTCELL_SUBNET_POOL_END=192.168.254.0 ``` The first run also writes `.rootcell/instances//state.json` with the -instance's vmnet UUID and allocated `/24`. By default, rootcell chooses the first -free subnet from `192.168.100.0/24` through `192.168.254.0/24`, uses `.2` for -the firewall, and uses `.3` for the agent. Existing state is not recalculated if -you later edit the pool values. +instance's allocated `/24`. By default, rootcell chooses the first free subnet +from `192.168.100.0/24` through `192.168.254.0/24`, uses `.2` for the firewall, +and uses `.3` for the agent. Existing state is not recalculated if you later +edit the pool values. To pin a new instance to a specific subnet before first run, set both IPs in that instance's `.env`: @@ -421,21 +378,13 @@ Named instances are isolated from each other: ``` Each instance gets its own VMs, state directory, CA, allowlists, Keychain mapping -file, control SSH key, private-link sockets, and `/24`. +file, control SSH key, private-link state, and `/24`. The `default` instance migrates from legacy repo-local files on first run: if `.env`, `secrets.env`, `proxy/allowed-*.txt`, or `pki/` already exist, rootcell copies them into `.rootcell/instances/default/`. Named instances seed from the checked-in defaults. -Existing VMs created by the legacy Lima provider are not migrated in place. Use -the Lima rollback provider to delete them if needed: - -```bash -limactl delete agent firewall --force -./rootcell -``` - ## Troubleshooting See what the firewall is denying: diff --git a/agent-vm.nix b/agent-vm.nix index 69da1e3..bdeec61 100644 --- a/agent-vm.nix +++ b/agent-vm.nix @@ -17,17 +17,16 @@ in # anyway. All meaningful filtering happens in the firewall VM. networking.firewall.enable = false; - # Networking: only the per-instance socket_vmnet interface is configured. - # The repo's patched Lima launcher skips the default usernet NIC for this - # VM, so the private socket_vmnet link is enp0s1 and there is no direct - # host usernet path a root-capable agent could reconfigure into egress. + # Networking: only the per-instance private vfkit link is configured, so + # there is no direct host control path a root-capable agent could reconfigure + # into egress. networking.useDHCP = false; networking.useNetworkd = true; systemd.network.enable = true; systemd.network.wait-online.enable = false; - # The private link from nixos.yaml/vfkit is enp0s1. Cloud-init performs a - # MAC-matched bootstrap before provisioning, then this NixOS config owns the - # steady-state interface. + # The private vfkit link is enp0s1. Cloud-init performs a MAC-matched + # bootstrap before provisioning, then this NixOS config owns the steady-state + # interface. systemd.network.networks."10-enp0s1" = { matchConfig.Name = "enp0s1"; networkConfig = { diff --git a/common.nix b/common.nix index 533d0e0..e82c8e9 100644 --- a/common.nix +++ b/common.nix @@ -1,4 +1,4 @@ -{ config, modulesPath, pkgs, lib, username, nixos-lima, ... }: +{ modulesPath, pkgs, lib, username, ... }: # Shared NixOS bits used by both the agent VM and the firewall VM. Things # that are genuinely VM-specific (hostname, networking, firewall policy, @@ -6,24 +6,12 @@ { imports = [ - # Required for the guest to boot under qemu/vz. + # Required for the guest to boot under virtio VM runtimes. (modulesPath + "/profiles/qemu-guest.nix") - # Provides `services.lima.*` options. Sets up lima-init at boot and - # runs the lima-guestagent daemon as a systemd service. - nixos-lima.nixosModules.lima ]; - options.rootcell.limaGuestSupport = lib.mkOption { - type = lib.types.bool; - default = true; - description = "Enable Lima guest initialization and guest agent support."; - }; - config = { - # Activate the nixos-lima module only for the Lima rollback provider. - services.lima.enable = lib.mkDefault config.rootcell.limaGuestSupport; - - # Rootcell's default vfkit path manages guests over SSH through the firewall. + # Rootcell manages guests over SSH through the firewall. services.openssh.enable = true; users.users.${username} = { @@ -70,7 +58,6 @@ environment.enableAllTerminfo = true; boot.kernelPackages = pkgs.linuxPackages_latest; - # Pin to the NixOS release nixos-lima is built against. Don't bump casually. system.stateVersion = "25.11"; }; } diff --git a/firewall-vm.nix b/firewall-vm.nix index 234d82f..f7478ba 100644 --- a/firewall-vm.nix +++ b/firewall-vm.nix @@ -10,9 +10,9 @@ in # Firewall VM: a tiny appliance VM that brokers all egress for the agent VM. # # Two NICs (kernel names from systemd predictable naming): -# enp0s1 vzNAT — internet egress (default route) -# enp0s2 socket_vmnet — private per-instance link to the agent VM -# (IPs come from network.nix/network-local.nix) +# enp0s1 vfkit NAT — internet egress (default route) +# enp0s2 private vfkit — per-instance link to the agent VM +# (IPs come from network.nix/network-local.nix) # # Hybrid filtering — HTTPS is intercepted, SSH is explicit, HTTP is denied: # @@ -82,10 +82,9 @@ in networkConfig.DHCP = "ipv4"; }; - # enp0s2 = socket_vmnet, our private link to the agent VM. (The kernel - # names this NIC enp0s2 via systemd predictable naming because it's - # the second virtio-net device — Lima's `interface:` field can't - # actually rename the kernel device, so we just use enp0s2 directly.) + # enp0s2 = private vfkit link to the agent VM. The kernel names this NIC + # enp0s2 via systemd predictable naming because it is the second virtio-net + # device. # Static address; DHCP would conflict with the agent's static .2. systemd.network.networks."20-enp0s2" = { matchConfig.Name = "enp0s2"; @@ -107,8 +106,8 @@ in # ── Firewall ────────────────────────────────────────────────────────── # NixOS firewall manages the filter table. We add a separate nat table - # below for the REDIRECT rules. Inbound on enp0s2 (the socket_vmnet link - # to the agent VM) is allowed only on the explicit-mitmproxy port + # below for the REDIRECT rules. Inbound on enp0s2 (the private link to the + # agent VM) is allowed only on the explicit-mitmproxy port # (8080), the transparent-mitmproxy port (8081, which is the # post-REDIRECT destination), and dnsmasq (53). networking.nftables.enable = true; @@ -148,9 +147,9 @@ in }; # ── Mutable allowlist directory ─────────────────────────────────────── - # `./rootcell allow` writes here via `limactl cp`, which connects as the - # unprivileged Lima guest user — so the dir is owned by ${username}, - # not root. The dnsmasq-allowlist.conf seed is empty: dnsmasq's + # `./rootcell allow` writes here over SSH as ${username}, so the dir is owned + # by ${username}, not root. The dnsmasq-allowlist.conf seed is empty: + # dnsmasq's # pre-start check refuses to launch without the conf-file existing, # but with `no-resolv` and no `server=` directives, an empty # conf-file means every query returns REFUSED — fail-closed by @@ -160,13 +159,12 @@ in # can overwrite root-owned files in a user-owned directory. # # The CA pem (key + cert) for TLS MITM is staged here too, but - # written by `./rootcell` via `limactl cp /tmp + sudo install -m 0600 - # -o root -g root` — never touchable by the lima user (who has - # passwordless sudo, but the explicit ownership chmod makes the - # blast radius "must already be root" rather than "any read of - # /etc/agent-vm leaks the key"). Loaded into the mitmproxy services - # via systemd LoadCredential, which surfaces it as a tmpfs file - # readable only by the service uid. + # written by `./rootcell` via SCP to /tmp plus `sudo install -m 0600 -o root + # -g root` — never touchable by the normal guest user. That user has + # passwordless sudo, but the explicit ownership chmod makes the blast radius + # "must already be root" rather than "any read of /etc/agent-vm leaks the + # key". Loaded into the mitmproxy services via systemd LoadCredential, which + # surfaces it as a tmpfs file readable only by the service uid. systemd.tmpfiles.rules = [ "d /etc/agent-vm 0755 ${username} users -" "f /etc/agent-vm/dnsmasq-allowlist.conf 0644 root root -" @@ -215,9 +213,9 @@ in # # ConditionPathExists guards the bootstrap window: on the very first # nixos-rebuild the CA is not yet copied in (./rootcell does that AFTER - # rebuild — we can't `limactl cp` to /etc/agent-vm/ before tmpfiles - # creates the dir), so the services skip cleanly. ./rootcell then pushes - # the CA and `systemctl restart`s, which re-evaluates the condition. + # rebuild — we can't copy into /etc/agent-vm/ before tmpfiles creates the + # dir), so the services skip cleanly. ./rootcell then pushes the CA and + # `systemctl restart`s, which re-evaluates the condition. systemd.services.mitmproxy-explicit = { description = "mitmproxy (explicit CONNECT — for SSH ProxyCommand)"; after = [ "network-online.target" ]; diff --git a/firewall.yaml b/firewall.yaml deleted file mode 100644 index 0b15f98..0000000 --- a/firewall.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# Lima config for the firewall VM. Tiny appliance: mitmproxy + dnsmasq, -# brokers all egress for the agent VM. See firewall-vm.nix for the NixOS -# side and proxy/README.md for ops notes. - -images: - - location: "https://github.com/nixos-lima/nixos-lima/releases/download/v0.0.5/nixos-lima-v0.0.5-aarch64.qcow2" - arch: "aarch64" - digest: "sha512:e1daeb0dcec65c624253603ab5ec06f0831b0940cd95a88903f9bfd0ee4009b2c45806b868674c7e8cb12941e50799e85d710fc0e9ad659059108cebbc4d19c1" - -mounts: [] - -portForwards: - # Same port-68 ignore as the agent VM: nixos-lima/issues/50. - - proto: udp - guestPort: 68 - guestIP: 0.0.0.0 - ignore: true - -ssh: - # Keep the Lima control plane independent from guest IP configuration. - overVsock: true - -user: - home: "/home/{{.User}}" - -# Memory: 1 GiB suffices for the steady-state services (mitmproxy + -# dnsmasq), but the initial `nixos-rebuild switch` peaks around 1-2 GiB -# during nixpkgs eval and parallel substituter decompression. 4 GiB -# avoids OOM-kill mid-build. vz balloons unused pages back to the host, -# so this isn't 4 GiB resident on the host all the time. -# Disk: 16 GiB lets the Nix store hold the firewall's closure plus a -# few generations before nix-gc kicks in. After provisioning settles, -# both can be cut back via `limactl edit firewall`. -memory: "4GiB" -cpus: 2 -disk: "16GiB" - -vmType: vz -mountType: virtiofs - -containerd: - system: false - user: false - -# Two NICs (kernel names assigned by systemd predictable naming): -# enp0s1 — Apple Virtio NAT (always present with vmType: vz; gives the -# firewall its internet egress; default route lives here) -# enp0s2 — per-instance socket_vmnet segment shared with the agent VM -# (the /24 and socket path are allocated by ./rootcell) -# -# We don't redeclare the Virtio NAT here. `vzNAT: true` would be -# redundant with vmType: vz's default. The Lima `interface:` field used -# to live here as a label, but Lima doesn't actually rename the kernel -# device — the kernel uses systemd predictable names regardless — so it -# was misleading and is dropped. -networks: - - socket: /private/var/run/rootcell/placeholder/default.sock diff --git a/flake.lock b/flake.lock index 6603310..168c261 100644 --- a/flake.lock +++ b/flake.lock @@ -1,23 +1,5 @@ { "nodes": { - "flake-utils": { - "inputs": { - "systems": "systems" - }, - "locked": { - "lastModified": 1731533236, - "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, "home-manager": { "inputs": { "nixpkgs": [ @@ -39,67 +21,6 @@ "type": "github" } }, - "nixlib": { - "locked": { - "lastModified": 1736643958, - "narHash": "sha256-tmpqTSWVRJVhpvfSN9KXBvKEXplrwKnSZNAoNPf/S/s=", - "owner": "nix-community", - "repo": "nixpkgs.lib", - "rev": "1418bc28a52126761c02dd3d89b2d8ca0f521181", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "nixpkgs.lib", - "type": "github" - } - }, - "nixos-generators": { - "inputs": { - "nixlib": "nixlib", - "nixpkgs": [ - "nixos-lima", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1769813415, - "narHash": "sha256-nnVmNNKBi1YiBNPhKclNYDORoHkuKipoz7EtVnXO50A=", - "owner": "nix-community", - "repo": "nixos-generators", - "rev": "8946737ff703382fda7623b9fab071d037e897d5", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "nixos-generators", - "type": "github" - } - }, - "nixos-lima": { - "inputs": { - "flake-utils": "flake-utils", - "nixos-generators": "nixos-generators", - "nixpkgs": [ - "nixpkgs" - ], - "nixpkgs-unstable": "nixpkgs-unstable" - }, - "locked": { - "lastModified": 1778181039, - "narHash": "sha256-kTJADfxmQvnuVt2grYhIng8HGNrlC6Ldvc3oa7buHjY=", - "owner": "nixos-lima", - "repo": "nixos-lima", - "rev": "777ecbc389e7ee720410d516f2bf1e3a03b3417b", - "type": "github" - }, - "original": { - "owner": "nixos-lima", - "ref": "master", - "repo": "nixos-lima", - "type": "github" - } - }, "nixpkgs": { "locked": { "lastModified": 1778003029, @@ -116,43 +37,11 @@ "type": "github" } }, - "nixpkgs-unstable": { - "locked": { - "lastModified": 1778124196, - "narHash": "sha256-pYEytCNic/czazbV9r3tbQ6BZzqRBg/41x2dIC5ymOo=", - "owner": "nixos", - "repo": "nixpkgs", - "rev": "68a8af93ff4297686cb68880845e61e5e2e41d92", - "type": "github" - }, - "original": { - "owner": "nixos", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, "root": { "inputs": { "home-manager": "home-manager", - "nixos-lima": "nixos-lima", "nixpkgs": "nixpkgs" } - }, - "systems": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 1192c66..06c5f32 100644 --- a/flake.nix +++ b/flake.nix @@ -2,21 +2,15 @@ description = "rootcell: root-capable coding-agent workspaces with allowlisted egress"; inputs = { - # Must match what nixos-lima is built against. v0.0.5 = nixos-25.11. nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11"; - nixos-lima = { - url = "github:nixos-lima/nixos-lima/master"; - inputs.nixpkgs.follows = "nixpkgs"; - }; - home-manager = { url = "github:nix-community/home-manager/release-25.11"; inputs.nixpkgs.follows = "nixpkgs"; }; }; - outputs = { self, nixpkgs, nixos-lima, home-manager, ... }: + outputs = { self, nixpkgs, home-manager, ... }: let # Apple Silicon hosts use aarch64-linux guests. # Switch to "x86_64-linux" if you're on an Intel Mac or x86 Linux host. @@ -24,36 +18,25 @@ # Username inside the guest. MUST agree with: # - GUEST_USER in ./rootcell - # - --set '.user.name = ""' passed to limactl start username = "luser"; pkgs = nixpkgs.legacyPackages.${system}; mkVM = module: nixpkgs.lib.nixosSystem { inherit system; - # nixos-lima is referenced from common.nix; username from both. - specialArgs = { inherit username nixos-lima; }; + specialArgs = { inherit username; }; modules = [ module ]; }; - mkVfkitVM = module: nixpkgs.lib.nixosSystem { - inherit system; - specialArgs = { inherit username nixos-lima; }; - modules = [ - module - ({ ... }: { rootcell.limaGuestSupport = false; }) - ]; - }; - - mkVfkitImage = module: nixpkgs.lib.nixosSystem { + mkImage = module: nixpkgs.lib.nixosSystem { inherit system; - specialArgs = { inherit username nixos-lima; }; + specialArgs = { inherit username; }; modules = [ module ./images/vfkit-image.nix ]; }; - agentImage = (mkVfkitImage ./agent-vm.nix).config.system.build.image; - firewallImage = (mkVfkitImage ./firewall-vm.nix).config.system.build.image; - builderImage = (mkVfkitImage ./images/builder-vm.nix).config.system.build.image; + agentImage = (mkImage ./agent-vm.nix).config.system.build.image; + firewallImage = (mkImage ./firewall-vm.nix).config.system.build.image; + builderImage = (mkImage ./images/builder-vm.nix).config.system.build.image; rootcellSourceRevision = self.rev or self.dirtyRev or "unknown"; nixpkgsRevision = nixpkgs.rev or "unknown"; @@ -87,30 +70,13 @@ JSON ''; - # Host-side packages. vfkit is the default macOS VM runtime; the - # patched Lima and socket_vmnet outputs remain as rollback support. + # Host-side packages. vfkit is the macOS VM runtime. forEachDarwin = nixpkgs.lib.genAttrs [ "aarch64-darwin" "x86_64-darwin" ]; darwinPkgs = forEachDarwin (sys: let p = nixpkgs.legacyPackages.${sys}; in { - lima = p.lima.overrideAttrs (old: rec { - version = "2.1.1"; - src = p.fetchFromGitHub { - owner = "lima-vm"; - repo = "lima"; - rev = "v${version}"; - hash = "sha256-U054xA3utBcSfpyvsZi4MvgJGNa7QyAYJf9usNXpgXg="; - }; - vendorHash = "sha256-C4YCuFVXkL5vS6lWZCGkEeZQgAkP55buPDGZ/wvMnAA="; - patches = (old.patches or []) ++ [ - ./patches/lima-vz-vsock-no-default-usernet.patch - ]; - meta = old.meta // { - knownVulnerabilities = []; - }; - }); vfkit = p.vfkit; - socket_vmnet = p.callPackage ./pkgs/socket_vmnet.nix { }; + zstd = p.zstd; }); in { @@ -119,11 +85,9 @@ JSON nixosConfigurations = { agent-vm = mkVM ./agent-vm.nix; firewall-vm = mkVM ./firewall-vm.nix; - agent-vm-vfkit = mkVfkitVM ./agent-vm.nix; - firewall-vm-vfkit = mkVfkitVM ./firewall-vm.nix; - agent-vm-vfkit-image = mkVfkitImage ./agent-vm.nix; - firewall-vm-vfkit-image = mkVfkitImage ./firewall-vm.nix; - builder-vm-vfkit-image = mkVfkitImage ./images/builder-vm.nix; + agent-vm-vfkit-image = mkImage ./agent-vm.nix; + firewall-vm-vfkit-image = mkImage ./firewall-vm.nix; + builder-vm-vfkit-image = mkImage ./images/builder-vm.nix; }; # Home Manager only attaches to the agent VM. The firewall VM is an @@ -137,10 +101,9 @@ JSON inherit rootcellSourceRevision nixpkgsRevision; packages = forEachDarwin (sys: { - lima = darwinPkgs.${sys}.lima; - vfkit = darwinPkgs.${sys}.vfkit; - socket_vmnet = darwinPkgs.${sys}.socket_vmnet; - default = darwinPkgs.${sys}.vfkit; + vfkit = darwinPkgs.${sys}.vfkit; + zstd = darwinPkgs.${sys}.zstd; + default = darwinPkgs.${sys}.vfkit; }) // { aarch64-linux = { inherit agentImage firewallImage builderImage rootcellImages; diff --git a/home.nix b/home.nix index 5337eb8..b45f0a4 100644 --- a/home.nix +++ b/home.nix @@ -6,7 +6,7 @@ # Pi reads provider keys from the env. DON'T put them in this file — the # Nix store is world-readable. Configure secret entries in secrets.env; `rootcell` # reads those macOS Keychain secrets on the host and exports them on -# `limactl shell`. +# guest sessions. let net = import ./network.nix; diff --git a/images/vfkit-image.nix b/images/vfkit-image.nix index e36e934..ceda644 100644 --- a/images/vfkit-image.nix +++ b/images/vfkit-image.nix @@ -5,8 +5,6 @@ (modulesPath + "/virtualisation/disk-image.nix") ]; - rootcell.limaGuestSupport = false; - image = { format = "raw"; efiSupport = true; diff --git a/network.nix b/network.nix index 997f18e..60c5855 100644 --- a/network.nix +++ b/network.nix @@ -13,16 +13,14 @@ let defaults = { - # IP of the firewall VM on the inter-VM socket_vmnet network. The - # agent VM uses this as its default route, DNS server, and SSH proxy. + # IP of the firewall VM on the private inter-VM network. The agent VM uses + # this as its default route, DNS server, and SSH proxy. # - # NOTE: do not put either VM at the .1 of the subnet. Apple's - # Keep .1 free. vmnet.framework may reserve that address for the host - # side of host-mode networks, and using it in the firewall VM creates - # confusing ARP and connection behavior. + # Keep .1 free for host-side or control-plane addresses if the private link + # implementation changes later. firewallIp = "192.168.100.2"; - # IP of the agent VM on the same network. + # IP of the agent VM on the same private network. agentIp = "192.168.100.3"; # Subnet prefix length for the inter-VM network. diff --git a/nixos.yaml b/nixos.yaml deleted file mode 100644 index 5574778..0000000 --- a/nixos.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# Lima config for the agent VM. The agent has root inside this VM, so its -# only network interface is the per-instance socket_vmnet segment shared -# with the firewall VM (allocated by ./rootcell --instance NAME). -# No internet egress except through the firewall. The repo's patched Lima -# launcher skips the default usernet NIC for this VM and brings up the -# localhost SSH forwarder by polling VSOCK directly. - -images: - - location: "https://github.com/nixos-lima/nixos-lima/releases/download/v0.0.5/nixos-lima-v0.0.5-aarch64.qcow2" - arch: "aarch64" - digest: "sha512:e1daeb0dcec65c624253603ab5ec06f0831b0940cd95a88903f9bfd0ee4009b2c45806b868674c7e8cb12941e50799e85d710fc0e9ad659059108cebbc4d19c1" - -mounts: [] - -portForwards: - # Tell Lima's port-forwarding to ignore port 68 to prevent interception of host DHCP packets - # Apparently this is an issue with NixOS that does not occur on other Linux distros - # See: https://github.com/nixos-lima/nixos-lima/issues/50 - - proto: udp - guestPort: 68 - guestIP: 0.0.0.0 - ignore: true - -ssh: - # The agent VM has no direct usernet NIC in steady state, so Lima - # control-plane SSH must use VSOCK. - overVsock: true - -user: - home: "/home/{{.User}}" - -memory: "16GiB" -cpus: 8 -disk: "60GiB" - -vmType: vz -mountType: virtiofs - -containerd: - system: false - user: false - -# Only one explicit Lima network: an unmanaged socket_vmnet socket created -# for this rootcell instance. The patched launcher suppresses Lima's -# implicit default usernet NIC, so this is the only agent NIC and appears -# as enp0s1 in the guest. The socket path is overridden at create time by -# `./rootcell --instance NAME`. -networks: - - socket: /private/var/run/rootcell/placeholder/default.sock diff --git a/patches/lima-vz-vsock-no-default-usernet.patch b/patches/lima-vz-vsock-no-default-usernet.patch deleted file mode 100644 index fab6acd..0000000 --- a/patches/lima-vz-vsock-no-default-usernet.patch +++ /dev/null @@ -1,95 +0,0 @@ ---- a/pkg/driver/vz/vm_darwin.go 2026-05-10 15:30:50 -+++ b/pkg/driver/vz/vm_darwin.go 2026-05-10 15:30:50 -@@ -48,0 +49,20 @@ -+const disableDefaultUsernetForVsockEnv = "LIMA_DISABLE_DEFAULT_USERNET_FOR_VSOCK" -+ -+func useSSHOverVsock(inst *limatype.Instance) bool { -+ useSSHOverVsock := *inst.Config.OS == limatype.LINUX -+ if inst.Config.SSH.OverVsock != nil { -+ useSSHOverVsock = *inst.Config.SSH.OverVsock -+ } -+ return useSSHOverVsock -+} -+ -+func disableDefaultUsernetForVsock(inst *limatype.Instance) bool { -+ if !useSSHOverVsock(inst) { -+ return false -+ } -+ if limayaml.FirstUsernetIndex(inst.Config) != -1 { -+ return false -+ } -+ return os.Getenv(disableDefaultUsernetForVsockEnv) == inst.Name -+} -+ -@@ -110,4 +130 @@ -- useSSHOverVsock := *inst.Config.OS == limatype.LINUX -- if inst.Config.SSH.OverVsock != nil { -- useSSHOverVsock = *inst.Config.SSH.OverVsock -- } -+ useSSHOverVsock := useSSHOverVsock(inst) -@@ -121,0 +139,17 @@ -+ } else if usernetClient == nil { -+ hostAddress := net.JoinHostPort(inst.SSHAddress, strconv.Itoa(usernetSSHLocalPort)) -+ if err := wrapper.waitAndStartVsockForwarder(ctx, 22, hostAddress); err == nil { -+ if onVsockEvent != nil { -+ onVsockEvent(&events.VsockEvent{ -+ Type: events.VsockEventStarted, -+ HostAddr: hostAddress, -+ VsockPort: 22, -+ }) -+ } -+ } else { -+ logrus.WithError(err).WithField("hostAddress", hostAddress).Warn("Failed to start vsock SSH forwarder") -+ if onVsockEvent != nil { -+ onVsockEvent(&events.VsockEvent{Type: events.VsockEventFailed, Reason: err.Error()}) -+ } -+ sendErrCh <- err -+ } -@@ -154,2 +188,4 @@ -- err := usernetClient.ConfigureDriver(ctx, inst, usernetSSHLocalPort) -- if err != nil { -+ if usernetClient == nil { -+ return -+ } -+ if err := usernetClient.ConfigureDriver(ctx, inst, usernetSSHLocalPort); err != nil { -@@ -164 +200,3 @@ -- _ = usernetClient.UnExposeSSH(inst.SSHLocalPort) -+ if usernetClient != nil { -+ _ = usernetClient.UnExposeSSH(inst.SSHLocalPort) -+ } -@@ -182,0 +221,3 @@ -+ if disableDefaultUsernetForVsock(inst) { -+ return nil, nil, nil -+ } -@@ -419 +460 @@ -- if firstUsernetIndex == -1 { -+ if firstUsernetIndex == -1 && !disableDefaultUsernetForVsock(inst) { -@@ -434 +475 @@ -- } else { -+ } else if firstUsernetIndex != -1 { ---- a/pkg/driver/vz/vsock_forwarder.go 2026-05-10 15:30:50 -+++ b/pkg/driver/vz/vsock_forwarder.go 2026-05-10 15:30:50 -@@ -10,0 +11 @@ -+ "fmt" -@@ -11,0 +13 @@ -+ "time" -@@ -16,0 +19,19 @@ -+func (m *virtualMachineWrapper) waitAndStartVsockForwarder(ctx context.Context, vsockPort uint32, hostAddress string) error { -+ ticker := time.NewTicker(500 * time.Millisecond) -+ defer ticker.Stop() -+ -+ var lastErr error -+ for { -+ if err := m.startVsockForwarder(ctx, vsockPort, hostAddress); err == nil { -+ return nil -+ } else { -+ lastErr = err -+ } -+ select { -+ case <-ctx.Done(): -+ return fmt.Errorf("timed out waiting for vsock:%d on VM: %w", vsockPort, lastErr) -+ case <-ticker.C: -+ } -+ } -+} -+ diff --git a/pkgs/socket_vmnet.nix b/pkgs/socket_vmnet.nix deleted file mode 100644 index 6a0ad49..0000000 --- a/pkgs/socket_vmnet.nix +++ /dev/null @@ -1,49 +0,0 @@ -# socket_vmnet — small daemon that exposes macOS's vmnet.framework over a -# Unix socket so unprivileged tools (Lima, in our case) can use it. -# -# Not in nixpkgs as of writing. We package it ourselves so the binary is a -# Nix-store artifact (root-owned, immutable, byte-for-byte reproducible); -# the only piece outside Nix is the one-time `sudo install` of this binary -# into /opt/socket_vmnet/bin so rootcell's vmnet helper has a stable target. -# The `rootcell` script's preflight builds via this derivation, compares to -# /opt/socket_vmnet, and prints the install command if it's missing or -# stale. See README → "One-time host setup". -# -# Upstream: https://github.com/lima-vm/socket_vmnet - -{ stdenv, fetchFromGitHub, lib }: - -stdenv.mkDerivation rec { - pname = "socket_vmnet"; - version = "1.2.2"; - - src = fetchFromGitHub { - owner = "lima-vm"; - repo = "socket_vmnet"; - rev = "v${version}"; - hash = "sha256-D5Z4aml82h397ho48HFeXwR6y2XkopFIKjO09jUgFdo="; - }; - - # The Makefile shells out to `git` to embed the version, which isn't in - # the sandbox — pass VERSION explicitly to bypass. - makeFlags = [ "VERSION=${version}" ]; - - # The upstream `install.bin` target uses `logger` (BSD syslog) for - # status output, which also isn't in the sandbox. Side-step it and - # copy the built binaries directly. We don't ship the launchd plists - # under share/doc — rootcell invokes socket_vmnet through its own helper - # at runtime, not as a system launchd daemon. - installPhase = '' - runHook preInstall - install -Dm 0755 -t $out/bin socket_vmnet socket_vmnet_client - runHook postInstall - ''; - - meta = with lib; { - description = "Bind macOS vmnet.framework to a Unix socket"; - homepage = "https://github.com/lima-vm/socket_vmnet"; - license = licenses.asl20; - platforms = platforms.darwin; - mainProgram = "socket_vmnet"; - }; -} diff --git a/proxy/README.md b/proxy/README.md index 234258d..fc06146 100644 --- a/proxy/README.md +++ b/proxy/README.md @@ -125,24 +125,27 @@ Plain hostnames (no globs). dnsmasq matches as a suffix, so listing ```bash # What's the firewall VM logging? -limactl shell firewall -- journalctl -u mitmproxy-explicit -u mitmproxy-transparent -u dnsmasq -f +ssh -F .rootcell/instances/default/ssh/config rootcell-firewall -- \ + journalctl -u mitmproxy-explicit -u mitmproxy-transparent -u dnsmasq -f # What is the agent sending to Bedrock? ./rootcell spy ./rootcell spy --tui # Is mitmproxy listening on both ports? -limactl shell firewall -- ss -tln '( sport = :8080 or sport = :8081 )' +ssh -F .rootcell/instances/default/ssh/config rootcell-firewall -- \ + "ss -tln '( sport = :8080 or sport = :8081 )'" # Is the NAT REDIRECT rule loaded? -limactl shell firewall -- sudo nft list table ip agent-vm-nat +ssh -F .rootcell/instances/default/ssh/config rootcell-firewall -- \ + sudo nft list table ip agent-vm-nat # What's the agent VM seeing? ./rootcell -- curl -v https://example.com 2>&1 | head -20 # Allowlist content currently inside the VM: -limactl shell firewall -- cat /etc/agent-vm/allowed-https.txt -limactl shell firewall -- cat /etc/agent-vm/dnsmasq-allowlist.conf +ssh -F .rootcell/instances/default/ssh/config rootcell-firewall -- \ + "cat /etc/agent-vm/allowed-https.txt && cat /etc/agent-vm/dnsmasq-allowlist.conf" ``` ## Files in this directory diff --git a/src/bin/rootcell-vmnet-helper.sh b/src/bin/rootcell-vmnet-helper.sh deleted file mode 100644 index 3a41606..0000000 --- a/src/bin/rootcell-vmnet-helper.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SOCKET_VMNET="/opt/socket_vmnet/bin/socket_vmnet" -ROOT="/private/var/run/rootcell" -runtime_dir="" -socket_path="" -pid_path="" -log_path="" - -usage() { - echo "usage: rootcell-vmnet {start INSTANCE UUID|status INSTANCE|stop INSTANCE}" >&2 -} - -die() { - echo "rootcell-vmnet: $*" >&2 - exit 2 -} - -validate_instance() { - local name="$1" - [[ "$name" =~ ^[a-z]([a-z0-9-]{0,30}[a-z0-9])?$ ]] || die "invalid instance name: $name" -} - -validate_uuid() { - local uuid="$1" - [[ "$uuid" =~ ^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[1-5][0-9A-Fa-f]{3}-[89ABab][0-9A-Fa-f]{3}-[0-9A-Fa-f]{12}$ ]] || die "invalid UUID: $uuid" -} - -sudo_uid() { - [[ "${SUDO_UID:-}" =~ ^[0-9]+$ ]] || die "must be run through sudo with SUDO_UID set" - echo "$SUDO_UID" -} - -paths_for() { - local instance="$1" - local uid - uid="$(sudo_uid)" - runtime_dir="$ROOT/$uid" - socket_path="$runtime_dir/$instance.sock" - pid_path="$runtime_dir/$instance.pid" - log_path="$runtime_dir/$instance.log" -} - -pid_matches() { - local pid="$1" - local socket="$2" - local command - command="$(ps -ww -p "$pid" -o command= 2>/dev/null || true)" - [[ "$command" == *"$SOCKET_VMNET"* && "$command" == *"$socket"* ]] -} - -running_pid() { - local pid="" - if [[ -f "$pid_path" ]]; then - pid="$(tr -d '[:space:]' < "$pid_path")" - fi - if [[ "$pid" =~ ^[0-9]+$ ]] && kill -0 "$pid" 2>/dev/null && pid_matches "$pid" "$socket_path"; then - echo "$pid" - return 0 - fi - return 1 -} - -cleanup_stale() { - local pid="" - if [[ -f "$pid_path" ]]; then - pid="$(tr -d '[:space:]' < "$pid_path")" - if [[ "$pid" =~ ^[0-9]+$ ]] && kill -0 "$pid" 2>/dev/null; then - if pid_matches "$pid" "$socket_path"; then - die "socket_vmnet is running but the socket is not ready: $pid" - fi - die "pidfile points at an unexpected process: $pid" - fi - fi - rm -f "$pid_path" "$socket_path" -} - -start_instance() { - local instance="$1" - local uuid="$2" - validate_instance "$instance" - validate_uuid "$uuid" - paths_for "$instance" - install -d -m 0755 "$ROOT" "$runtime_dir" - if running_pid >/dev/null; then - if [[ -S "$socket_path" ]]; then - exit 0 - fi - die "socket_vmnet is running but the socket is not ready" - fi - cleanup_stale - nohup "$SOCKET_VMNET" \ - --vmnet-mode=host \ - --vmnet-network-identifier="$uuid" \ - --pidfile="$pid_path" \ - "$socket_path" \ - >"$log_path" 2>&1 & - - for _ in $(seq 1 50); do - if running_pid >/dev/null && [[ -S "$socket_path" ]]; then - exit 0 - fi - sleep 0.1 - done - - echo "rootcell-vmnet: socket_vmnet did not become ready" >&2 - if [[ -f "$log_path" ]]; then - tail -n 40 "$log_path" >&2 || true - fi - exit 1 -} - -status_instance() { - local instance="$1" - validate_instance "$instance" - paths_for "$instance" - if running_pid >/dev/null && [[ -S "$socket_path" ]]; then - echo "running" - exit 0 - fi - echo "stopped" - exit 1 -} - -stop_instance() { - local instance="$1" - local pid - validate_instance "$instance" - paths_for "$instance" - if pid="$(running_pid)"; then - kill "$pid" - for _ in $(seq 1 50); do - if ! kill -0 "$pid" 2>/dev/null; then - rm -f "$pid_path" "$socket_path" - exit 0 - fi - sleep 0.1 - done - die "socket_vmnet did not stop: $pid" - fi - cleanup_stale -} - -command="${1:-}" -case "$command" in - start) - [[ "$#" -eq 3 ]] || { usage; exit 2; } - start_instance "$2" "$3" - ;; - status) - [[ "$#" -eq 2 ]] || { usage; exit 2; } - status_instance "$2" - ;; - stop) - [[ "$#" -eq 2 ]] || { usage; exit 2; } - stop_instance "$2" - ;; - *) - usage - exit 2 - ;; -esac diff --git a/src/bin/test.ts b/src/bin/test.ts index be89f7e..e844d5d 100755 --- a/src/bin/test.ts +++ b/src/bin/test.ts @@ -1,7 +1,7 @@ #!/usr/bin/env bun import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync } from "node:fs"; import { join, resolve, dirname } from "node:path"; -import { runCapture, runInherited } from "../rootcell/process.ts"; +import { runCapture } from "../rootcell/process.ts"; const REPO_DIR = findRepoDir(import.meta.path); const TEST_INSTANCE = "test"; @@ -9,7 +9,6 @@ const AGENT_VM_NAME = "agent-test"; const FIREWALL_VM_NAME = "firewall-test"; const FIREWALL_IP = "192.168.109.2"; const AGENT_IP = "192.168.109.3"; -const VM_PROVIDER = process.env.ROOTCELL_VM_PROVIDER ?? "vfkit"; interface TestCase { readonly name: string; @@ -78,41 +77,22 @@ function shellQuote(value: string): string { return `'${value.replaceAll("'", "'\\''")}'`; } -function limactl(args: readonly string[]): string { - return commandOk("limactl", args); -} - function agentSh(script: string): string { - if (VM_PROVIDER === "vfkit") { - return sshGuest("rootcell-agent", script); - } - return limactl(["shell", AGENT_VM_NAME, "--", "bash", "-lc", script]); + return sshGuest("rootcell-agent", script); } function agentShCapture(script: string): ReturnType { - if (VM_PROVIDER === "vfkit") { - return runCapture("ssh", ["-F", sshConfigPath(), "rootcell-agent", `bash -lc ${shellQuote(script)}`], { - allowFailure: true, - }); - } - return runCapture("limactl", ["shell", AGENT_VM_NAME, "--", "bash", "-lc", script], { + return runCapture("ssh", ["-F", sshConfigPath(), "rootcell-agent", `bash -lc ${shellQuote(script)}`], { allowFailure: true, }); } function firewallSh(script: string): string { - if (VM_PROVIDER === "vfkit") { - return sshGuest("rootcell-firewall", script); - } - return limactl(["shell", FIREWALL_VM_NAME, "--", "bash", "-lc", script]); + return sshGuest("rootcell-firewall", script); } function agentShFails(script: string): void { - if (VM_PROVIDER === "vfkit") { - commandFails("ssh", ["-F", sshConfigPath(), "rootcell-agent", `bash -lc ${shellQuote(script)}`]); - return; - } - commandFails("limactl", ["shell", AGENT_VM_NAME, "--", "bash", "-lc", script]); + commandFails("ssh", ["-F", sshConfigPath(), "rootcell-agent", `bash -lc ${shellQuote(script)}`]); } function sshGuest(alias: "rootcell-agent" | "rootcell-firewall", script: string): string { @@ -192,66 +172,18 @@ function vfkitVmIsRunning(name: string): void { } } -function yamlHasOverVsock(path: string): boolean { - const lines = readFileSync(path, "utf8").split(/\r?\n/); - let inSsh = false; - for (const line of lines) { - if (line.startsWith("ssh:")) { - inSsh = true; - continue; - } - if (inSsh && /^[^ \t#]/.test(line)) { - inSsh = false; - } - if (inSsh && /^[ \t]+overVsock:[ \t]+true[ \t]*$/.test(line)) { - return true; - } - } - return false; -} - -function limaInstanceHasOverVsock(name: string): void { - const out = limactl(["list", name, "--json"]); - if (!out.includes('"overVsock":true')) { - throw new TestFailure("overVsock was not true"); - } -} - -function checkedInYamlsHaveOverVsock(): void { - if (!yamlHasOverVsock(join(REPO_DIR, "nixos.yaml")) || !yamlHasOverVsock(join(REPO_DIR, "firewall.yaml"))) { - throw new TestFailure("checked-in Lima YAML is missing ssh.overVsock: true"); - } -} - function syncDefaultAllowlists(): void { log("syncing .defaults allowlists into test firewall..."); - if (VM_PROVIDER === "vfkit") { - const proxyDir = join(REPO_DIR, ".rootcell", "instances", TEST_INSTANCE, "proxy"); - mkdirSync(proxyDir, { recursive: true, mode: 0o700 }); - for (const file of ["allowed-https.txt", "allowed-ssh.txt", "allowed-dns.txt"]) { - copyFileSync(join(REPO_DIR, "proxy", `${file}.defaults`), join(proxyDir, file)); - } - commandOk(join(REPO_DIR, "rootcell"), ["--instance", TEST_INSTANCE, "allow"]); - return; - } + const proxyDir = join(REPO_DIR, ".rootcell", "instances", TEST_INSTANCE, "proxy"); + mkdirSync(proxyDir, { recursive: true, mode: 0o700 }); for (const file of ["allowed-https.txt", "allowed-ssh.txt", "allowed-dns.txt"]) { - limactl([ - "cp", - join(REPO_DIR, "proxy", `${file}.defaults`), - `${FIREWALL_VM_NAME}:/etc/agent-vm/${file}`, - ]); + copyFileSync(join(REPO_DIR, "proxy", `${file}.defaults`), join(proxyDir, file)); } - runInherited("limactl", ["shell", FIREWALL_VM_NAME, "--", "sudo", "/etc/agent-vm/reload.sh"], { - ignoredOutput: true, - }); + commandOk(join(REPO_DIR, "rootcell"), ["--instance", TEST_INSTANCE, "allow"]); } function agentRestartsViaWrapper(): void { - if (VM_PROVIDER === "vfkit") { - stopVfkitVm(AGENT_VM_NAME); - } else { - limactl(["stop", AGENT_VM_NAME]); - } + stopVfkitVm(AGENT_VM_NAME); commandOk(join(REPO_DIR, "rootcell"), ["--instance", TEST_INSTANCE, "provision"]); syncDefaultAllowlists(); agentSh("true"); @@ -330,17 +262,6 @@ function vfkitCases(): TestCase[] { ]; } -function limaCases(): TestCase[] { - return [ - { name: "checked-in Lima YAMLs force SSH over VSOCK", run: checkedInYamlsHaveOverVsock }, - { name: "agent VM is Running", run: () => commandOk("bash", ["-c", `[ "$(limactl list --format '{{.Status}}' ${AGENT_VM_NAME})" = Running ]`]) }, - { name: "firewall VM is Running", run: () => commandOk("bash", ["-c", `[ "$(limactl list --format '{{.Status}}' ${FIREWALL_VM_NAME})" = Running ]`]) }, - { name: "agent Lima config has ssh.overVsock enabled", run: () => { limaInstanceHasOverVsock(AGENT_VM_NAME); } }, - { name: "firewall Lima config has ssh.overVsock enabled", run: () => { limaInstanceHasOverVsock(FIREWALL_VM_NAME); } }, - { name: "agent Lima user has lingering enabled", run: () => agentSh('loginctl show-user "$USER" -p Linger | grep -q Linger=yes') }, - ]; -} - function sharedCases(): TestCase[] { return [ { name: "agent restarts via ./rootcell after locked-down networking", run: agentRestartsViaWrapper }, @@ -349,11 +270,11 @@ function sharedCases(): TestCase[] { { name: "agent spy tui flags parse with help", run: () => commandOk("bash", ["-c", `'${join(REPO_DIR, "rootcell")}' --instance ${TEST_INSTANCE} spy --tui --raw --no-dedupe --help | grep -q -- '--tui'`]) }, { name: "firewall spy formatter installed", run: () => firewallSh('test -x /etc/agent-vm/agent_spy.py && test -x /etc/agent-vm/agent_spy_tui.py && command -v python3 >/dev/null && python3 -c "import textual" && test -d /run/agent-vm-spy') }, { name: "agent VM has test IP on enp0s1", run: () => agentSh(`ip -4 -o addr show enp0s1 | grep -q '${AGENT_IP}/'`) }, - { name: "agent VM has no default Lima usernet NIC", run: () => agentSh("! ip link show enp0s2 >/dev/null 2>&1") }, + { name: "agent VM has no second virtio-net NIC", run: () => agentSh("! ip link show enp0s2 >/dev/null 2>&1") }, { name: "agent VM routes default traffic through firewall", run: () => agentSh(`ip route show default | grep -q '^default via ${FIREWALL_IP} dev enp0s1'`) }, { name: "agent VM has no direct usernet address", run: () => agentSh("! ip -4 -o addr show | grep -q '192\\.168\\.5\\.'") }, { name: "firewall VM has test IP on enp0s2", run: () => firewallSh(`ip -4 -o addr show enp0s2 | grep -q '${FIREWALL_IP}/'`) }, - { name: "agent reaches firewall dnsmasq on private VMNET link", run: () => agentSh(`dig @${FIREWALL_IP} +short +time=5 +tries=1 github.com | grep -qE '^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$'`) }, + { name: "agent reaches firewall dnsmasq on private link", run: () => agentSh(`dig @${FIREWALL_IP} +short +time=5 +tries=1 github.com | grep -qE '^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$'`) }, { name: "home-manager dev CLIs on PATH", run: () => agentSh("command -v pi && command -v rg && command -v gh && command -v jq >/dev/null") }, { name: "pi --help runs", run: () => agentSh('out=$(pi --help) && [ -n "$out" ]') }, { name: "HTTPS allowed: github.com", run: () => agentSh('code=$(curl -sS --max-time 10 -o /dev/null -w "%{http_code}" https://github.com) && [[ "$code" =~ ^[23] ]]') }, @@ -373,13 +294,7 @@ function sharedCases(): TestCase[] { } function buildCases(): TestCase[] { - if (VM_PROVIDER === "vfkit") { - return [...vfkitCases(), ...sharedCases()]; - } - if (VM_PROVIDER === "lima") { - return [...limaCases(), ...sharedCases()]; - } - throw new TestFailure(`unsupported ROOTCELL_VM_PROVIDER '${VM_PROVIDER}'`); + return [...vfkitCases(), ...sharedCases()]; } function runCase(testCase: TestCase): boolean { @@ -398,9 +313,7 @@ function runCase(testCase: TestCase): boolean { } function removeTestInstanceState(): void { - if (VM_PROVIDER === "vfkit") { - stopVfkitInstance(); - } + stopVfkitInstance(); rmSync(join(REPO_DIR, ".rootcell", "instances", TEST_INSTANCE), { recursive: true, force: true, @@ -415,31 +328,17 @@ function main(args: readonly string[]): number { if (parsed.teardown) { log("deleting test VMs..."); - if (VM_PROVIDER === "vfkit") { - stopVfkitInstance(); - } else { - runInherited("limactl", ["delete", "-f", AGENT_VM_NAME, FIREWALL_VM_NAME], { - allowFailure: true, - ignoredOutput: true, - }); - } + stopVfkitInstance(); removeTestInstanceState(); return 0; } if (parsed.clean) { log("deleting test VMs for a fresh provision..."); - if (VM_PROVIDER === "vfkit") { - stopVfkitInstance(); - } else { - runInherited("limactl", ["delete", "-f", AGENT_VM_NAME, FIREWALL_VM_NAME], { - allowFailure: true, - ignoredOutput: true, - }); - } + stopVfkitInstance(); removeTestInstanceState(); } - log(`provisioning ${VM_PROVIDER} test VMs (first run takes ~15 min)...`); + log("provisioning vfkit test VMs (first run takes ~15 min)..."); commandOk(join(REPO_DIR, "rootcell"), ["--instance", TEST_INSTANCE, "provision"]); syncDefaultAllowlists(); log("running tests..."); diff --git a/src/rootcell/images.ts b/src/rootcell/images.ts index 6820a49..880141b 100644 --- a/src/rootcell/images.ts +++ b/src/rootcell/images.ts @@ -9,7 +9,7 @@ import { renameSync, } from "node:fs"; import { basename, join } from "node:path"; -import { runCapture, runInherited, runStdoutToFile } from "./process.ts"; +import { commandExists, runCapture, runInherited, runStdoutToFile } from "./process.ts"; import type { RootcellConfig } from "./types.ts"; export const ROOTCELL_IMAGE_SCHEMA_VERSION = 1; @@ -44,6 +44,8 @@ export interface RootcellImageEntry { } export class ImageStore { + private zstdBin = process.env.ROOTCELL_ZSTD ?? ""; + constructor( private readonly config: RootcellConfig, private readonly log: (message: string) => void, @@ -97,12 +99,33 @@ export class ImageStore { private expandImage(entry: RootcellImageEntry, compressedPath: string, rawPath: string): void { const tmp = `${rawPath}.tmp`; if (entry.compression === "zstd") { - runStdoutToFile("zstd", ["-d", "-c", compressedPath], tmp); + runStdoutToFile(this.ensureZstd(), ["-d", "-c", compressedPath], tmp); } else { runInherited("cp", [compressedPath, tmp]); } renameSync(tmp, rawPath); } + + private ensureZstd(): string { + if (this.zstdBin.length > 0) { + return this.zstdBin; + } + if (commandExists("zstd")) { + this.zstdBin = "zstd"; + return this.zstdBin; + } + const result = runCapture("nix", [ + "build", + "--no-link", + "--print-out-paths", + `${this.config.repoDir}#zstd`, + ], { allowFailure: true }); + if (result.status !== 0) { + throw new Error(`failed to build zstd from ${this.config.repoDir}/flake.nix:\n${result.stderr}`); + } + this.zstdBin = join(firstToken(result.stdout), "bin/zstd"); + return this.zstdBin; + } } export function parseRootcellImageManifest(raw: unknown): RootcellImageManifest { @@ -183,6 +206,14 @@ export function sha256File(path: string): string { } } +function firstToken(output: string): string { + const token = output.trim().split(/\s+/)[0]; + if (token === undefined || token.length === 0) { + throw new Error("command produced no output"); + } + return token; +} + function parseImageEntry(raw: unknown): RootcellImageEntry { if (typeof raw !== "object" || raw === null) { throw new Error("invalid rootcell image manifest: image entries must be objects"); diff --git a/src/rootcell/instance.ts b/src/rootcell/instance.ts index 22f67cc..459f8a2 100644 --- a/src/rootcell/instance.ts +++ b/src/rootcell/instance.ts @@ -9,7 +9,6 @@ import { writeFileSync, } from "node:fs"; import { join } from "node:path"; -import { randomUUID } from "node:crypto"; import type { InstanceState, RootcellInstance } from "./types.ts"; const STATE_SCHEMA_VERSION = 1; @@ -17,7 +16,6 @@ const DEFAULT_INSTANCE = "default"; const DEFAULT_POOL_START = "192.168.100.0"; const DEFAULT_POOL_END = "192.168.254.0"; const INSTANCE_NAME_RE = /^[a-z](?:[a-z0-9-]{0,30}[a-z0-9])?$/; -const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; interface InstancePaths { readonly name: string; @@ -28,8 +26,6 @@ interface InstancePaths { readonly pkiDir: string; readonly generatedDir: string; readonly statePath: string; - readonly socketPath: string; - readonly pidPath: string; } interface StateEntry { @@ -109,7 +105,6 @@ export function loadRootcellInstance(repoDir: string, instanceName: string, env: export function instancePaths(repoDir: string, instanceName: string): InstancePaths { const name = validateInstanceName(instanceName); const dir = join(repoDir, ".rootcell", "instances", name); - const runtimeDir = `/private/var/run/rootcell/${currentUid()}`; return { name, dir, @@ -119,8 +114,6 @@ export function instancePaths(repoDir: string, instanceName: string): InstancePa pkiDir: join(dir, "pki"), generatedDir: join(dir, "generated"), statePath: join(dir, "state.json"), - socketPath: `${runtimeDir}/${name}.sock`, - pidPath: `${runtimeDir}/${name}.pid`, }; } @@ -129,7 +122,7 @@ function ensureInstanceState(repoDir: string, paths: InstancePaths, env: NodeJS. assertNoSubnetCollisions(existingEntries); if (existsSync(paths.statePath)) { - const state = normalizeState(readState(paths.name, paths.statePath), paths); + const state = readState(paths.name, paths.statePath); writeStateIfChanged(paths.statePath, state); assertNoSubnetCollisions([ ...existingEntries.filter((entry) => entry.name !== paths.name), @@ -140,7 +133,7 @@ function ensureInstanceState(repoDir: string, paths: InstancePaths, env: NodeJS. const requested = stateFromEnv(paths, env); const used = new Set(existingEntries.map((entry) => entry.state.subnet)); - const state = requested ?? allocateState(paths, env, used); + const state = requested ?? allocateState(env, used); if (used.has(state.subnet)) { throw new Error(`subnet ${state.subnet}/24 is already allocated to another rootcell instance`); } @@ -170,7 +163,7 @@ function readAllInstanceStates(repoDir: string): StateEntry[] { if (!existsSync(paths.statePath)) { continue; } - entries.push({ name, state: normalizeState(readState(name, paths.statePath), paths) }); + entries.push({ name, state: readState(name, paths.statePath) }); } return entries; } @@ -192,42 +185,26 @@ function validateState(name: string, raw: unknown): InstanceState { } const record = raw as Record; const schemaVersion = record.schemaVersion; - const vmnetUuid = stringField(record, "vmnetUuid", name); const subnet = stringField(record, "subnet", name); const networkPrefix = record.networkPrefix; const firewallIp = stringField(record, "firewallIp", name); const agentIp = stringField(record, "agentIp", name); - const socketPath = stringField(record, "socketPath", name); - const pidPath = stringField(record, "pidPath", name); if (schemaVersion !== STATE_SCHEMA_VERSION) { throw new Error(`invalid rootcell instance state for ${name}: unsupported schemaVersion`); } - if (!UUID_RE.test(vmnetUuid)) { - throw new Error(`invalid rootcell instance state for ${name}: vmnetUuid is not a UUID`); - } if (networkPrefix !== 24) { throw new Error(`invalid rootcell instance state for ${name}: networkPrefix must be 24`); } validateSubnetAndHosts(subnet, firewallIp, agentIp, name); return { schemaVersion: STATE_SCHEMA_VERSION, - vmnetUuid, subnet, networkPrefix: 24, firewallIp, agentIp, - socketPath, - pidPath, }; } -function normalizeState(state: InstanceState, paths: InstancePaths): InstanceState { - if (state.socketPath === paths.socketPath && state.pidPath === paths.pidPath) { - return state; - } - return { ...state, socketPath: paths.socketPath, pidPath: paths.pidPath }; -} - function stringField(record: Record, field: string, name: string): string { const value = record[field]; if (typeof value !== "string" || value.length === 0) { @@ -251,10 +228,10 @@ function stateFromEnv(paths: InstancePaths, env: NodeJS.ProcessEnv): InstanceSta } const subnet = formatIpv4(subnet24(parseIpv4(firewallIp))); validateSubnetAndHosts(subnet, firewallIp, agentIp, paths.name); - return baseState(paths, subnet, firewallIp, agentIp); + return baseState(subnet, firewallIp, agentIp); } -function allocateState(paths: InstancePaths, env: NodeJS.ProcessEnv, used: ReadonlySet): InstanceState { +function allocateState(env: NodeJS.ProcessEnv, used: ReadonlySet): InstanceState { const { start, end } = poolFromEnv(env); for (let network = start; network <= end; network += 256) { const subnet = formatIpv4(network); @@ -262,21 +239,18 @@ function allocateState(paths: InstancePaths, env: NodeJS.ProcessEnv, used: Reado continue; } const prefix = subnet.slice(0, subnet.lastIndexOf(".")); - return baseState(paths, subnet, `${prefix}.2`, `${prefix}.3`); + return baseState(subnet, `${prefix}.2`, `${prefix}.3`); } throw new Error(`rootcell subnet pool is exhausted (${formatIpv4(start)}/24 through ${formatIpv4(end)}/24)`); } -function baseState(paths: InstancePaths, subnet: string, firewallIp: string, agentIp: string): InstanceState { +function baseState(subnet: string, firewallIp: string, agentIp: string): InstanceState { return { schemaVersion: STATE_SCHEMA_VERSION, - vmnetUuid: randomUUID(), subnet, networkPrefix: 24, firewallIp, agentIp, - socketPath: paths.socketPath, - pidPath: paths.pidPath, }; } @@ -372,7 +346,3 @@ function writeStateIfChanged(path: string, state: InstanceState): void { } writeFileSync(path, content, { encoding: "utf8", mode: 0o600 }); } - -function currentUid(): string { - return String(process.getuid?.() ?? 0); -} diff --git a/src/rootcell/providers/factory.ts b/src/rootcell/providers/factory.ts index bb0b17f..aebc4a4 100644 --- a/src/rootcell/providers/factory.ts +++ b/src/rootcell/providers/factory.ts @@ -1,24 +1,12 @@ import type { RootcellConfig } from "../types.ts"; import type { ProviderBundle } from "./types.ts"; -import { LimaVmProvider } from "./lima.ts"; -import { MacOsSocketVmnetNetworkProvider, type LimaSocketNetworkAttachment } from "./macos-socket-vmnet.ts"; import { MacOsVfkitNetworkProvider, type VfkitNetworkAttachment } from "./macos-vfkit-network.ts"; import { VfkitVmProvider } from "./vfkit.ts"; export function createProviderBundle( config: RootcellConfig, log: (message: string) => void, -): ProviderBundle { - const provider = process.env.ROOTCELL_VM_PROVIDER ?? "vfkit"; - if (provider === "lima") { - return { - network: new MacOsSocketVmnetNetworkProvider(config, log), - vm: new LimaVmProvider(config, log), - }; - } - if (provider !== "vfkit") { - throw new Error(`unsupported ROOTCELL_VM_PROVIDER '${provider}' (expected vfkit or lima)`); - } +): ProviderBundle { return { network: new MacOsVfkitNetworkProvider(config, log), vm: new VfkitVmProvider(config, log), diff --git a/src/rootcell/providers/lima.ts b/src/rootcell/providers/lima.ts deleted file mode 100644 index 1d43bfc..0000000 --- a/src/rootcell/providers/lima.ts +++ /dev/null @@ -1,255 +0,0 @@ -import { existsSync, readFileSync } from "node:fs"; -import { homedir } from "node:os"; -import { join } from "node:path"; -import { nixString } from "../env.ts"; -import { runAsyncInherited, runCapture, runInherited } from "../process.ts"; -import type { RootcellConfig } from "../types.ts"; -import type { CommandResult, InheritedCommandResult } from "../types.ts"; -import type { CopyToGuestOptions, ExecOptions, VmProvider, VmRole, VmStatus } from "./types.ts"; -import type { LimaSocketNetworkAttachment } from "./macos-socket-vmnet.ts"; - -export class LimaVmProvider implements VmProvider { - readonly id = "lima"; - private limaBin: string; - - constructor( - private readonly config: RootcellConfig, - private readonly log: (message: string) => void, - ) { - this.limaBin = process.env.LIMACTL ?? ""; - } - - status(name: string): Promise { - return Promise.resolve(limaStatusFromOutput(this.limactlCapture(["list", "--format", "{{.Status}}", name], true).stdout)); - } - - async forceStopIfRunning(name: string): Promise { - if ((await this.status(name)).state === "running") { - this.log(`force-stopping ${name} VM to repair stale ${this.config.instanceName} vmnet daemon...`); - this.limactlInherited(["stop", "--force", name]); - } - } - - async assertCompatible(name: string, network: LimaSocketNetworkAttachment): Promise { - const status = await this.status(name); - if (status.state === "missing") { - return; - } - if (this.vmUsesInstanceSocket(name, network.socketPath)) { - return; - } - this.log(`${name} exists but was not created for rootcell instance '${this.config.instanceName}'.`); - this.log(`Delete and recreate it to migrate to the isolated socket vmnet network: limactl delete ${name} --force`); - process.exit(1); - } - - async ensureRunning(input: { - readonly role: VmRole; - readonly name: string; - readonly network: LimaSocketNetworkAttachment; - }): Promise<{ readonly created: boolean }> { - const configPath = join(this.config.repoDir, input.role === "agent" ? "nixos.yaml" : "firewall.yaml"); - const status = await this.status(input.name); - switch (status.state) { - case "running": - return { created: false }; - case "stopped": - this.log(`starting ${input.name} VM...`); - this.startVm(input.name); - return { created: false }; - case "missing": - this.log(`${input.name} VM not found; creating (~3-5 min for image + boot)...`); - { - const result = this.limactlInherited([ - "start", - "--timeout", - this.config.vmStartTimeout, - "--tty=false", - `--name=${input.name}`, - "--set", - `.user.name = "${this.config.guestUser}"`, - "--set", - `.networks[0].socket = ${nixString(input.network.socketPath)}`, - "--set", - ".ssh.overVsock = true", - configPath, - ], { allowFailure: true }); - if (result.status !== 0) { - this.diagnoseStartFailure(input.name); - this.log(`limactl start ${input.name} failed; aborting.`); - process.exit(1); - } - } - return { created: true }; - case "unexpected": - this.log(`${input.name} VM in unexpected state: '${status.detail}'. Aborting.`); - process.exit(1); - } - } - - exec(name: string, command: readonly string[], options: ExecOptions = {}): Promise { - return Promise.resolve(this.limactlInherited(["shell", name, "--", ...guestCommand(command, options)], options)); - } - - execCapture(name: string, command: readonly string[], options: ExecOptions = {}): Promise { - return Promise.resolve(this.limactlCapture(["shell", name, "--", ...guestCommand(command, options)], options.allowFailure ?? false)); - } - - async execInteractive(name: string, command: readonly string[], options: ExecOptions = {}): Promise { - return await this.limactlAsyncInherited(["shell", name, "--", ...guestCommand(command, options)]); - } - - copyToGuest(name: string, hostPath: string, guestPath: string, options: CopyToGuestOptions = {}): Promise { - this.limactlInherited([ - "cp", - ...(options.recursive === true ? ["-r"] : []), - hostPath, - `${name}:${guestPath}`, - ]); - return Promise.resolve(); - } - - private ensureLima(): void { - if (this.limaBin.length > 0) { - return; - } - const result = runCapture("nix", [ - "build", - "--no-link", - "--print-out-paths", - `${this.config.repoDir}#lima`, - ], { allowFailure: true }); - if (result.status !== 0) { - this.log(`failed to build repo-patched Lima from ${this.config.repoDir}/flake.nix:`); - process.stderr.write(prefixLines(result.stderr, "rootcell: ")); - process.exit(1); - } - if (result.stderr.length > 0) { - process.stderr.write(result.stderr); - } - this.limaBin = join(firstToken(result.stdout), "bin/limactl"); - } - - private limaEnv(): NodeJS.ProcessEnv { - return { - ...process.env, - LIMA_DISABLE_DEFAULT_USERNET_FOR_VSOCK: this.config.agentVm, - }; - } - - private limactlCapture(args: readonly string[], allowFailure = false): ReturnType { - this.ensureLima(); - return runCapture(this.limaBin, args, { - env: this.limaEnv(), - allowFailure, - }); - } - - private limactlInherited(args: readonly string[], options: { readonly allowFailure?: boolean; readonly ignoredOutput?: boolean } = {}): ReturnType { - this.ensureLima(); - return runInherited(this.limaBin, args, { - env: this.limaEnv(), - ...(options.allowFailure === undefined ? {} : { allowFailure: options.allowFailure }), - ...(options.ignoredOutput === undefined ? {} : { ignoredOutput: options.ignoredOutput }), - }); - } - - private async limactlAsyncInherited(args: readonly string[]): Promise { - this.ensureLima(); - return await runAsyncInherited(this.limaBin, args, { env: this.limaEnv() }); - } - - private vmUsesInstanceSocket(name: string, socketPath: string): boolean { - const result = this.limactlCapture(["list", name, "--json"], true); - if (result.status !== 0) { - return false; - } - return limaListJsonContainsSocket(result.stdout, socketPath); - } - - private startVm(name: string): void { - const result = this.limactlInherited(["start", "--timeout", this.config.vmStartTimeout, name], { - allowFailure: true, - }); - if (result.status === 0) { - return; - } - this.diagnoseStartFailure(name); - process.exit(1); - } - - private diagnoseStartFailure(name: string): void { - const logFile = join(homedir(), ".lima", name, "ha.stderr.log"); - if (!existsSync(logFile)) { - return; - } - const tail = readFileSync(logFile, "utf8").split(/\r?\n/).slice(-80).join("\n"); - if (/Waiting for port to become available on .*:22/.test(tail) && !tail.includes("Started vsock forwarder")) { - this.log(`${name} VM did not establish Lima SSH over VSOCK.`); - this.log("Lima is waiting for guest TCP/22 on its default usernet path."); - this.log("The agent VM should be started with this repo's patched Lima, which skips that usernet NIC and polls VSOCK directly for SSH."); - } - } -} - -export function limaStatusFromOutput(output: string): VmStatus { - const status = output.trim(); - switch (status) { - case "": - return { state: "missing" }; - case "Running": - return { state: "running" }; - case "Stopped": - return { state: "stopped" }; - default: - return { state: "unexpected", detail: status }; - } -} - -export function limaListJsonContainsSocket(output: string, socketPath: string): boolean { - if (output.trim().length === 0) { - return false; - } - try { - return jsonContainsSocket(JSON.parse(output), socketPath); - } catch { - return output.includes(`"socket":${JSON.stringify(socketPath)}`); - } -} - -function guestCommand(command: readonly string[], options: ExecOptions): readonly string[] { - if (options.env === undefined || options.env.length === 0) { - return command; - } - return ["env", ...options.env, "--", ...command]; -} - -function firstToken(output: string): string { - const token = output.trim().split(/\s+/)[0]; - if (token === undefined || token.length === 0) { - throw new Error("command produced no output"); - } - return token; -} - -function prefixLines(text: string, prefix: string): string { - return text.split(/\r?\n/).filter((line) => line.length > 0).map((line) => `${prefix}${line}`).join("\n") + "\n"; -} - -function jsonContainsSocket(value: unknown, socketPath: string): boolean { - if (Array.isArray(value)) { - return value.some((item) => jsonContainsSocket(item, socketPath)); - } - if (typeof value !== "object" || value === null) { - return false; - } - for (const [key, child] of Object.entries(value)) { - if (key === "socket" && child === socketPath) { - return true; - } - if (jsonContainsSocket(child, socketPath)) { - return true; - } - } - return false; -} diff --git a/src/rootcell/providers/macos-socket-vmnet.ts b/src/rootcell/providers/macos-socket-vmnet.ts deleted file mode 100644 index a3cd974..0000000 --- a/src/rootcell/providers/macos-socket-vmnet.ts +++ /dev/null @@ -1,204 +0,0 @@ -import { existsSync, readFileSync } from "node:fs"; -import { join } from "node:path"; -import { runCapture, runInherited } from "../process.ts"; -import type { RootcellConfig } from "../types.ts"; -import type { NetworkPlan, NetworkProvider, VmNetworkAttachment } from "./types.ts"; - -const SOCKET_VMNET_DST = "/opt/socket_vmnet/bin/socket_vmnet"; -const ROOTCELL_VMNET_HELPER_DST = "/opt/rootcell/bin/rootcell-vmnet"; -const ROOTCELL_VMNET_SUDOERS = "/private/etc/sudoers.d/rootcell-vmnet"; - -export interface LimaSocketNetworkAttachment extends VmNetworkAttachment { - readonly kind: "lima-socket"; - readonly socketPath: string; - readonly sshOverVsock: true; - readonly disableDefaultUsernet: boolean; - readonly useDefaultNat: boolean; -} - -export class MacOsSocketVmnetNetworkProvider implements NetworkProvider { - readonly id = "macos-socket-vmnet"; - - constructor( - private readonly config: RootcellConfig, - private readonly log: (message: string) => void, - ) {} - - plan(): NetworkPlan { - return { - provider: this.id, - guest: { - firewallIp: this.config.firewallIp, - agentIp: this.config.agentIp, - networkPrefix: 24, - agentPrivateInterface: "enp0s1", - firewallPrivateInterface: "enp0s2", - firewallEgressInterface: "enp0s1", - }, - vms: { - agent: { - kind: "lima-socket", - socketPath: this.config.vmnetSocketPath, - sshOverVsock: true, - disableDefaultUsernet: true, - useDefaultNat: false, - }, - firewall: { - kind: "lima-socket", - socketPath: this.config.vmnetSocketPath, - sshOverVsock: true, - disableDefaultUsernet: false, - useDefaultNat: true, - }, - }, - }; - } - - preflight(): Promise { - this.ensureSocketVmnet(); - this.ensureRootcellVmnetHelper(); - return Promise.resolve(); - } - - async ensureReady(input: { - readonly affectedVms: readonly string[]; - readonly stopVmIfRunning: (name: string) => Promise; - }): Promise { - const status = runCapture("sudo", [ - "-n", - ROOTCELL_VMNET_HELPER_DST, - "status", - this.config.instanceName, - ], { allowFailure: true }); - if (status.status === 0) { - return; - } - if (status.status !== 1 || status.stderr.length > 0) { - process.stderr.write(status.stderr); - this.log("failed to check rootcell vmnet helper status."); - process.exit(1); - } - this.log(`starting isolated vmnet daemon for instance '${this.config.instanceName}' (${this.config.firewallIp}/24, ${this.config.agentIp}/24)...`); - for (const vm of input.affectedVms) { - await input.stopVmIfRunning(vm); - } - const start = runInherited("sudo", [ - "-n", - ROOTCELL_VMNET_HELPER_DST, - "start", - this.config.instanceName, - this.config.vmnetUuid, - ], { allowFailure: true }); - if (start.status !== 0) { - this.log("failed to start rootcell vmnet helper."); - process.exit(1); - } - } - - private ensureSocketVmnet(): void { - const result = runCapture("nix", [ - "build", - "--no-link", - "--print-out-paths", - `${this.config.repoDir}#socket_vmnet`, - ], { allowFailure: true }); - if (result.status !== 0) { - this.log(`failed to build socket_vmnet from ${this.config.repoDir}/pkgs/socket_vmnet.nix:`); - process.stderr.write(result.stderr); - process.exit(1); - } - if (result.stderr.length > 0) { - process.stderr.write(result.stderr); - } - - const out = firstToken(result.stdout); - const nixBin = join(out, "bin/socket_vmnet"); - if (existsSync(SOCKET_VMNET_DST)) { - const cmp = runInherited("cmp", ["-s", nixBin, SOCKET_VMNET_DST], { - allowFailure: true, - ignoredOutput: true, - }); - if (cmp.status === 0) { - return; - } - } - - process.stderr.write(`rootcell: socket_vmnet not installed (or out of date) at /opt/socket_vmnet. - -Why this needs sudo: macOS vmnet.framework requires socket_vmnet to run as -root. rootcell's one-time helper grant references the binary by this stable -root-owned path, while the binary itself is built declaratively from this -repo's flake (see pkgs/socket_vmnet.nix). - -Run: - - sudo install -m 0755 -d /opt/socket_vmnet/bin - sudo install -m 0755 \\ - ${out}/bin/socket_vmnet \\ - ${out}/bin/socket_vmnet_client \\ - /opt/socket_vmnet/bin/ - -Then re-run ./rootcell. -`); - process.exit(1); - } - - private ensureRootcellVmnetHelper(): void { - const helperSrc = join(this.config.repoDir, "src/bin/rootcell-vmnet-helper.sh"); - const helperOk = existsSync(ROOTCELL_VMNET_HELPER_DST) - && runInherited("cmp", ["-s", helperSrc, ROOTCELL_VMNET_HELPER_DST], { - allowFailure: true, - ignoredOutput: true, - }).status === 0; - const sudoersOk = this.rootcellVmnetSudoersLooksInstalled(); - if (helperOk && sudoersOk) { - return; - } - process.stderr.write(`rootcell: one-time rootcell vmnet helper setup needed. - -The new per-instance networks use a small root-owned helper with one stable -sudoers rule. This avoids editing Lima managed networks or regenerating Lima -sudoers for every instance. - -Run: - - sudo install -m 0755 -d /opt/rootcell/bin - sudo install -m 0755 \\ - ${shellQuote(helperSrc)} \\ - ${shellQuote(ROOTCELL_VMNET_HELPER_DST)} - sudo chown root:wheel ${shellQuote(ROOTCELL_VMNET_HELPER_DST)} - printf '%s\\n' '%staff ALL=(root:wheel) NOPASSWD:NOSETENV: ${ROOTCELL_VMNET_HELPER_DST} *' \\ - | sudo tee ${ROOTCELL_VMNET_SUDOERS} >/dev/null - sudo chmod 0440 ${ROOTCELL_VMNET_SUDOERS} - -Then re-run ./rootcell. -`); - process.exit(1); - } - - private rootcellVmnetSudoersLooksInstalled(): boolean { - if (!existsSync(ROOTCELL_VMNET_SUDOERS)) { - return false; - } - try { - return readFileSync(ROOTCELL_VMNET_SUDOERS, "utf8").includes(ROOTCELL_VMNET_HELPER_DST); - } catch { - return true; - } - } -} - -function firstToken(output: string): string { - const token = output.trim().split(/\s+/)[0]; - if (token === undefined || token.length === 0) { - throw new Error("command produced no output"); - } - return token; -} - -function shellQuote(value: string): string { - if (/^[A-Za-z0-9_./:=@%+-]+$/.test(value)) { - return value; - } - return `'${value.replaceAll("'", "'\\''")}'`; -} diff --git a/src/rootcell/rootcell.test.ts b/src/rootcell/rootcell.test.ts index c8e10d7..f6d6601 100644 --- a/src/rootcell/rootcell.test.ts +++ b/src/rootcell/rootcell.test.ts @@ -6,8 +6,6 @@ import { buildConfig } from "./rootcell.ts"; import { deriveVmNames, loadRootcellInstance, seedRootcellInstanceFiles } from "./instance.ts"; import { runCapture } from "./process.ts"; import { createProviderBundle } from "./providers/factory.ts"; -import { limaListJsonContainsSocket, limaStatusFromOutput } from "./providers/lima.ts"; -import { MacOsSocketVmnetNetworkProvider } from "./providers/macos-socket-vmnet.ts"; import { macFor, MacOsVfkitNetworkProvider } from "./providers/macos-vfkit-network.ts"; import { vfkitArgs, parseVfkitVmState, lookupDhcpLease, vfkitCloudInitUserData } from "./providers/vfkit.ts"; import { @@ -142,13 +140,11 @@ describe("environment parsing", () => { }); test("builds config from instance state", () => { - const config = buildConfig("/repo", { VM_START_TIMEOUT: "5s" }, fakeInstance("dev")); + const config = buildConfig("/repo", {}, fakeInstance("dev")); expect(config.agentVm).toBe("agent-dev"); expect(config.firewallVm).toBe("firewall-dev"); expect(config.firewallIp).toBe("192.168.109.2"); expect(config.agentIp).toBe("192.168.109.3"); - expect(config.vmnetSocketPath).toBe("/private/var/run/rootcell/501/dev.sock"); - expect(config.vmStartTimeout).toBe("5s"); expect(config.imageManifestUrl).toBe("https://github.com/rootcell-ai/rootcell/releases/latest/download/manifest.json"); }); }); @@ -160,54 +156,6 @@ describe("VM and network providers", () => { expect(providers.vm.id).toBe("vfkit"); }); - test("factory keeps Lima providers behind rollback env var", () => { - const old = process.env.ROOTCELL_VM_PROVIDER; - process.env.ROOTCELL_VM_PROVIDER = "lima"; - try { - const providers = createProviderBundle(buildConfig("/repo", {}, fakeInstance("dev")), ignoreLog); - expect(providers.network.id).toBe("macos-socket-vmnet"); - expect(providers.vm.id).toBe("lima"); - } finally { - if (old === undefined) { - delete process.env.ROOTCELL_VM_PROVIDER; - } else { - process.env.ROOTCELL_VM_PROVIDER = old; - } - } - }); - - test("macOS socket vmnet provider exposes guest config and Lima attachments", () => { - const config = buildConfig("/repo", {}, fakeInstance("dev")); - const plan = new MacOsSocketVmnetNetworkProvider(config, ignoreLog).plan(); - expect(plan).toEqual({ - provider: "macos-socket-vmnet", - guest: { - firewallIp: "192.168.109.2", - agentIp: "192.168.109.3", - networkPrefix: 24, - agentPrivateInterface: "enp0s1", - firewallPrivateInterface: "enp0s2", - firewallEgressInterface: "enp0s1", - }, - vms: { - agent: { - kind: "lima-socket", - socketPath: "/private/var/run/rootcell/501/dev.sock", - sshOverVsock: true, - disableDefaultUsernet: true, - useDefaultNat: false, - }, - firewall: { - kind: "lima-socket", - socketPath: "/private/var/run/rootcell/501/dev.sock", - sshOverVsock: true, - disableDefaultUsernet: false, - useDefaultNat: true, - }, - }, - }); - }); - test("macOS vfkit provider exposes host-control and hostless-private attachments", () => { const config = buildConfig("/repo", {}, fakeInstance("dev")); const plan = new MacOsVfkitNetworkProvider(config, ignoreLog).plan(); @@ -331,30 +279,6 @@ describe("VM and network providers", () => { } }); - test("Lima status output maps to provider-neutral VM states", () => { - expect(limaStatusFromOutput("")).toEqual({ state: "missing" }); - expect(limaStatusFromOutput("Running\n")).toEqual({ state: "running" }); - expect(limaStatusFromOutput("Stopped")).toEqual({ state: "stopped" }); - expect(limaStatusFromOutput("Broken")).toEqual({ state: "unexpected", detail: "Broken" }); - }); - - test("Lima socket compatibility parser finds nested socket attachments", () => { - const socketPath = "/private/var/run/rootcell/501/dev.sock"; - const output = JSON.stringify([ - { - name: "agent-dev", - config: { - networks: [ - { socket: socketPath }, - ], - }, - }, - ]); - expect(limaListJsonContainsSocket(output, socketPath)).toBe(true); - expect(limaListJsonContainsSocket(output, "/private/var/run/rootcell/501/other.sock")).toBe(false); - expect(limaListJsonContainsSocket(`{"socket":${JSON.stringify(socketPath)}`, socketPath)).toBe(true); - }); - test("vfkit state parser validates running state shape", () => { expect(parseVfkitVmState({ provider: "vfkit", @@ -369,7 +293,7 @@ describe("VM and network providers", () => { controlMac: "52:54:00:00:00:02", firewallControlIp: "192.168.64.2", }).firewallControlIp).toBe("192.168.64.2"); - expect(() => parseVfkitVmState({ provider: "lima" })).toThrow("provider mismatch"); + expect(() => parseVfkitVmState({ provider: "unknown" })).toThrow("provider mismatch"); }); test("macOS DHCP lease parser finds vfkit NAT IP by MAC", () => { @@ -541,15 +465,6 @@ function stripTrailingBlankLine(text: string): string { return text.endsWith("\n\n") ? text.slice(0, -1) : text; } -describe("Lima templates", () => { - test("checked-in Lima YAMLs use unmanaged socket networks", () => { - expect(readFileSync("nixos.yaml", "utf8")).toContain("socket:"); - expect(readFileSync("firewall.yaml", "utf8")).toContain("socket:"); - expect(readFileSync("nixos.yaml", "utf8")).not.toContain("lima: host"); - expect(readFileSync("firewall.yaml", "utf8")).not.toContain("lima: host"); - }); -}); - function fakeInstance(name: string): RootcellInstance { return { name, @@ -562,13 +477,10 @@ function fakeInstance(name: string): RootcellInstance { statePath: `/repo/.rootcell/instances/${name}/state.json`, state: { schemaVersion: 1, - vmnetUuid: "00000000-0000-4000-8000-000000000001", subnet: "192.168.109.0", networkPrefix: 24, firewallIp: "192.168.109.2", agentIp: "192.168.109.3", - socketPath: `/private/var/run/rootcell/501/${name}.sock`, - pidPath: `/private/var/run/rootcell/501/${name}.pid`, }, }; } @@ -587,13 +499,10 @@ function makeInstanceRepo(): string { function stateJson(name: string, prefix: string): string { return `${JSON.stringify({ schemaVersion: 1, - vmnetUuid: "00000000-0000-4000-8000-000000000001", subnet: `${prefix}.0`, networkPrefix: 24, firewallIp: `${prefix}.2`, agentIp: `${prefix}.3`, - socketPath: `/private/var/run/rootcell/501/${name}.sock`, - pidPath: `/private/var/run/rootcell/501/${name}.pid`, }, null, 2)}\n`; } diff --git a/src/rootcell/rootcell.ts b/src/rootcell/rootcell.ts index d53cd43..cbf1c30 100644 --- a/src/rootcell/rootcell.ts +++ b/src/rootcell/rootcell.ts @@ -80,10 +80,6 @@ export function buildConfig(repoDir: string, env: NodeJS.ProcessEnv, instance: R firewallIp: instance.state.firewallIp, agentIp: instance.state.agentIp, networkPrefix: String(instance.state.networkPrefix), - vmnetUuid: instance.state.vmnetUuid, - vmnetSocketPath: instance.state.socketPath, - vmnetPidPath: instance.state.pidPath, - vmStartTimeout: env.VM_START_TIMEOUT ?? "180s", imageManifestUrl: env.ROOTCELL_IMAGE_MANIFEST_URL ?? DEFAULT_IMAGE_MANIFEST_URL, ...(env.ROOTCELL_IMAGE_DIR === undefined || env.ROOTCELL_IMAGE_DIR.length === 0 ? {} : { imageDir: env.ROOTCELL_IMAGE_DIR }), }; @@ -394,8 +390,7 @@ exit 1 } private nixosConfiguration(role: "agent" | "firewall"): string { - const base = role === "agent" ? "agent-vm" : "firewall-vm"; - return this.providers.vm.id === "vfkit" ? `${base}-vfkit` : base; + return role === "agent" ? "agent-vm" : "firewall-vm"; } private hostTimeZone(): string { diff --git a/src/rootcell/types.ts b/src/rootcell/types.ts index bc34c37..ecd53f8 100644 --- a/src/rootcell/types.ts +++ b/src/rootcell/types.ts @@ -27,10 +27,6 @@ export interface RootcellConfig { readonly firewallIp: string; readonly agentIp: string; readonly networkPrefix: string; - readonly vmnetUuid: string; - readonly vmnetSocketPath: string; - readonly vmnetPidPath: string; - readonly vmStartTimeout: string; readonly imageManifestUrl: string; readonly imageDir?: string; } @@ -52,13 +48,10 @@ export type ParsedRootcellArgs = ParsedRootcellRunArgs | ParsedRootcellHandledAr export interface InstanceState { readonly schemaVersion: 1; - readonly vmnetUuid: string; readonly subnet: string; readonly networkPrefix: 24; readonly firewallIp: string; readonly agentIp: string; - readonly socketPath: string; - readonly pidPath: string; } export interface RootcellInstance {