From 8d2dcc757efc8928a9540b68fbc535f021fa5bb6 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 2 Jul 2026 20:33:33 -0700 Subject: [PATCH] Add CoCo guest components to dstack rootfs --- README.md | 6 + docs/coco-k8s-testing.md | 287 ++++++++++++++++++ dstack | 2 +- .../coco-guest-components.bb | 167 ++++++++++ .../files/attestation-agent.conf | 10 + .../files/coco-api-server-rest.service | 14 + .../files/coco-attestation-agent.service | 15 + .../files/coco-confidential-data-hub.service | 15 + .../coco-guest-components/files/coco-pause.c | 29 ++ .../files/confidential-data-hub.conf | 15 + .../files/ocicrypt_config.json | 7 + .../files/pause-config.json | 63 ++++ .../images/dstack-rootfs-base.inc | 2 + ...0001-kata-agent-build-with-rust-1.92.patch | 54 ++++ .../kata-agent/kata-agent-coco_git.bb | 113 +++++++ 15 files changed, 798 insertions(+), 1 deletion(-) create mode 100644 docs/coco-k8s-testing.md create mode 100644 meta-dstack/recipes-core/coco-guest-components/coco-guest-components.bb create mode 100644 meta-dstack/recipes-core/coco-guest-components/files/attestation-agent.conf create mode 100644 meta-dstack/recipes-core/coco-guest-components/files/coco-api-server-rest.service create mode 100644 meta-dstack/recipes-core/coco-guest-components/files/coco-attestation-agent.service create mode 100644 meta-dstack/recipes-core/coco-guest-components/files/coco-confidential-data-hub.service create mode 100644 meta-dstack/recipes-core/coco-guest-components/files/coco-pause.c create mode 100644 meta-dstack/recipes-core/coco-guest-components/files/confidential-data-hub.conf create mode 100644 meta-dstack/recipes-core/coco-guest-components/files/ocicrypt_config.json create mode 100644 meta-dstack/recipes-core/coco-guest-components/files/pause-config.json create mode 100644 meta-dstack/recipes-core/kata-agent/files/0001-kata-agent-build-with-rust-1.92.patch create mode 100644 meta-dstack/recipes-core/kata-agent/kata-agent-coco_git.bb diff --git a/README.md b/README.md index 7ddcbb3..bad706a 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,12 @@ This project implements Yocto layer and the overall build scripts for dstack Bas See https://github.com/Phala-Network/dstack-cloud for more details. +## CoCo/Kata Kubernetes smoke test + +After building the dstack guest rootfs with CoCo guest components, see +[`docs/coco-k8s-testing.md`](docs/coco-k8s-testing.md) for a Kata TDX/Kubernetes +smoke-test workflow. + ## Reproducible Build The Guest Image ### Pre-requisites diff --git a/docs/coco-k8s-testing.md b/docs/coco-k8s-testing.md new file mode 100644 index 0000000..e508a41 --- /dev/null +++ b/docs/coco-k8s-testing.md @@ -0,0 +1,287 @@ +# Testing the dstack CoCo guest image on Kubernetes + +This guide smoke-tests the dstack rootfs as a Kata/CoCo CVM guest image on a +Kubernetes node. It is intended for the first-cut CoCo integration where the +native dstack services and the CoCo guest components run independently in the +same image. + +The commands below assume a local Yocto build tree and a single-node test host. +Adjust paths and the `RuntimeClass` name if your Kata installation differs. + +## Prerequisites + +- A host with Kubernetes and Kata Containers installed. +- A confidential Kata runtime class, for example `kata-qemu-tdx`. +- The node is labelled for Kata scheduling, for example: + + ```bash + kubectl get runtimeclass + kubectl get nodes --show-labels | grep katacontainers.io/kata-runtime=true + ``` + +- The host can write to the Kata image/config directories, typically: + + ```text + /opt/kata/share/kata-containers/ + /opt/kata/share/defaults/kata-containers/configuration-qemu-tdx.toml + ``` + +> **Warning:** the test edits the host Kata runtime config. Use a dedicated test +> machine or keep a backup and restore it when finished. + +## Build the guest rootfs + +```bash +source openembedded-core/oe-init-build-env bb-build >/dev/null +bitbake dstack-rootfs +``` + +The build should produce: + +```text +bb-build/tmp/deploy/images/dstack/dstack-rootfs-dstack.cpio +bb-build/tmp/deploy/images/dstack/bzImage +``` + +Before building a disk image, verify that the CoCo/Kata pieces are present in +the rootfs work directory: + +```bash +ROOT=bb-build/tmp/work/dstack-poky-linux/dstack-rootfs/1.0/rootfs + +test -x "$ROOT/usr/bin/kata-agent" +test -f "$ROOT/etc/kata-opa/default-policy.rego" +test -f "$ROOT/etc/ocicrypt_config.json" +test -x "$ROOT/pause_bundle/rootfs/pause" +``` + +The default policy file is required because the `agent-policy` feature +initializes OPA before initdata is parsed. The pause bundle is required by +Kata's confidential/guest-pull sandbox path. + +## Create a Kata disk image from the built cpio + +```bash +IMG=/opt/kata/share/kata-containers/dstack-coco-mvp.ext4 +KERNEL=/opt/kata/share/kata-containers/vmlinuz-dstack-coco-mvp +CPIO=$(readlink -f bb-build/tmp/deploy/images/dstack/dstack-rootfs-dstack.cpio) +BZIMAGE=$(readlink -f bb-build/tmp/deploy/images/dstack/bzImage) +MNT=/tmp/dstack-coco-mvp-root + +sudo rm -f "${IMG}.tmp" +sudo truncate -s 3G "${IMG}.tmp" +printf 'label: dos\nunit: sectors\n\nstart=6144, type=83, bootable\n' | sudo sfdisk "${IMG}.tmp" + +LOOP=$(sudo losetup --find --show --partscan "${IMG}.tmp") +sleep 1 +sudo mkfs.ext4 -F "${LOOP}p1" +sudo mkdir -p "$MNT" +sudo mount "${LOOP}p1" "$MNT" + +sudo bash -c "cd '$MNT' && cpio -idmu --no-absolute-filenames < '$CPIO'" +sudo sync +sudo umount "$MNT" +sudo losetup -d "$LOOP" + +sudo mv -f "${IMG}.tmp" "$IMG" +sudo cp -a "$BZIMAGE" "$KERNEL" +``` + +If your system does not create `${LOOP}p1`, detach the loop device and use an +explicit offset mount/mkfs flow instead. + +## Point Kata TDX at the dstack image + +Back up the current config first: + +```bash +KATA_CFG=/opt/kata/share/defaults/kata-containers/configuration-qemu-tdx.toml +sudo cp -a "$KATA_CFG" "${KATA_CFG}.bak.$(date +%Y%m%d%H%M%S)" +``` + +Set the kernel/image/rootfs and make sure the agent starts the CoCo guest +components: + +```bash +sudo sed -i \ + -e 's#^kernel = ".*"#kernel = "/opt/kata/share/kata-containers/vmlinuz-dstack-coco-mvp"#' \ + -e 's#^image = ".*"#image = "/opt/kata/share/kata-containers/dstack-coco-mvp.ext4"#' \ + -e 's#^rootfs_type=.*#rootfs_type="ext4"#' \ + -e 's#^default_memory = .*#default_memory = 4096#' \ + "$KATA_CFG" + +# Add these to kernel_params if they are not already present: +# cgroup_no_v1=all systemd.unified_cgroup_hierarchy=1 +# systemd.unit=kata-containers.target +# agent.log=debug +# agent.guest_components_procs=api-server-rest +# agent.guest_components_rest_api=all +``` + +Kata normally reads this config when a new sandbox starts. If the runtime has a +stale config, restart containerd/kubelet on the test node. + +## Create initdata + +For a smoke test, use an allow-all policy and offline CDH config: + +```bash +cat >/tmp/dstack-coco-mvp-initdata.toml <<'EOF_INITDATA' +version = "0.1.0" +algorithm = "sha256" + +[data] +"aa.toml" = ''' +[eventlog_config] +init_pcr = 17 +enable_eventlog = false + +[log] +level = "debug" +''' + +"cdh.toml" = ''' +socket = "unix:///run/confidential-containers/cdh.sock" + +[kbc] +name = "offline_fs_kbc" +url = "" + +[log] +level = "debug" +''' + +"policy.rego" = ''' +package agent_policy + +default AddARPNeighborsRequest := true +default AddSwapRequest := true +default CloseStdinRequest := true +default CopyFileRequest := true +default CreateContainerRequest := true +default CreateSandboxRequest := true +default DestroySandboxRequest := true +default ExecProcessRequest := true +default GetDiagnosticDataRequest := true +default GetMetricsRequest := true +default GetOOMEventRequest := true +default GuestDetailsRequest := true +default ListInterfacesRequest := true +default ListRoutesRequest := true +default MemAgentCompactConfig := true +default MemAgentMemcgConfig := true +default MemHotplugByProbeRequest := true +default OnlineCPUMemRequest := true +default PauseContainerRequest := true +default PullImageRequest := true +default ReadStreamRequest := true +default RemoveContainerRequest := true +default RemoveStaleVirtiofsShareMountsRequest := true +default ReseedRandomDevRequest := true +default ResumeContainerRequest := true +default SetGuestDateTimeRequest := true +default SetPolicyRequest := true +default SignalProcessRequest := true +default StartContainerRequest := true +default StartTracingRequest := true +default StatsContainerRequest := true +default StopTracingRequest := true +default TtyWinResizeRequest := true +default UpdateContainerRequest := true +default UpdateEphemeralMountsRequest := true +default UpdateInterfaceRequest := true +default UpdateRoutesRequest := true +default WaitProcessRequest := true +default WriteStreamRequest := true +''' +EOF_INITDATA + +INITDATA_B64=$(gzip -c /tmp/dstack-coco-mvp-initdata.toml | base64 -w0) +``` + +For a KBS-backed run, change `cdh.toml` to select `cc_kbc` and set the KBS URL, +then regenerate `INITDATA_B64`. + +## Deploy the test Pod + +```bash +cat >/tmp/dstack-coco-mvp-test.yaml <- + cgroup_no_v1=all + systemd.unified_cgroup_hierarchy=1 + systemd.unit=kata-containers.target + agent.log=debug + agent.guest_components_procs=api-server-rest + agent.guest_components_rest_api=all +spec: + runtimeClassName: kata-qemu-tdx + restartPolicy: Never + containers: + - name: test + image: docker.io/library/busybox:latest + imagePullPolicy: IfNotPresent + command: ["sh", "-c", "echo hello-from-dstack-coco; uname -a; sleep 300"] +EOF_POD + +kubectl apply -f /tmp/dstack-coco-mvp-test.yaml +kubectl get pod dstack-coco-mvp-test -w +``` + +A successful run reaches `Running`, and the container log shows the dstack guest +kernel: + +```bash +kubectl logs dstack-coco-mvp-test +# hello-from-dstack-coco +# Linux dstack-coco-mvp-test 6.18.24-dstack ... x86_64 GNU/Linux +``` + +You can also confirm the QEMU command line uses the dstack image: + +```bash +pgrep -af 'qemu-system.*sandbox' | grep dstack-coco-mvp +``` + +## Troubleshooting + +- `timed out connecting to vsock ...:1024`: check the guest console and make + sure `/etc/kata-opa/default-policy.rego` exists in the image. +- `Pause image not present in rootfs`: check that `/pause_bundle/config.json` + and `/pause_bundle/rootfs/pause` exist in the image. +- `Creating watcher returned error too many open files`: the test host may have + too many stale shims or a low inotify limit. On a dedicated test node: + + ```bash + sudo sysctl -w fs.inotify.max_user_instances=1024 + sudo sysctl -w fs.inotify.max_user_watches=1048576 + ``` + +- To remove a stuck test sandbox, first delete the Pod, then check for stale + Kata shims/QEMU processes before killing anything: + + ```bash + kubectl delete pod dstack-coco-mvp-test --force --grace-period=0 --ignore-not-found + pgrep -af 'containerd-shim-kata|qemu-system.*sandbox' + ``` + +## Cleanup + +```bash +kubectl delete pod dstack-coco-mvp-test --force --grace-period=0 --ignore-not-found +``` + +Restore the backed-up Kata config when done: + +```bash +sudo cp -a /path/to/configuration-qemu-tdx.toml.bak.YYYYmmddHHMMSS \ + /opt/kata/share/defaults/kata-containers/configuration-qemu-tdx.toml +sudo systemctl restart containerd +sudo systemctl restart kubelet +``` diff --git a/dstack b/dstack index 439e3d5..4380a54 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 439e3d51ff353f03d03a6069e7c7dafc49a93677 +Subproject commit 4380a54a2831d27b4a7dbfc238e1667963ef2497 diff --git a/meta-dstack/recipes-core/coco-guest-components/coco-guest-components.bb b/meta-dstack/recipes-core/coco-guest-components/coco-guest-components.bb new file mode 100644 index 0000000..b0506d8 --- /dev/null +++ b/meta-dstack/recipes-core/coco-guest-components/coco-guest-components.bb @@ -0,0 +1,167 @@ +SUMMARY = "Confidential Containers guest components for dstack guest images" +DESCRIPTION = "Initial, independent integration of selected Confidential Containers guest-components: attestation-agent, confidential-data-hub, and REST API server." +HOMEPAGE = "https://github.com/confidential-containers/guest-components" +LICENSE = "Apache-2.0" +LIC_FILES_CHKSUM = "file://LICENSE;md5=86d3f3a95c324c9479bd8986968f4327" + +SRCREV = "3f931d10197d242675fa558d263aa740cc196c2f" +SRC_URI = "git://github.com/confidential-containers/guest-components.git;protocol=https;branch=main \ + file://coco-attestation-agent.service \ + file://coco-confidential-data-hub.service \ + file://coco-api-server-rest.service \ + file://attestation-agent.conf \ + file://confidential-data-hub.conf \ + file://ocicrypt_config.json \ + file://pause-config.json \ + file://coco-pause.c \ +" + +PV = "0.1.0+git" + +inherit cargo_bin systemd + +# Keep this first cut TDX-focused while enabling the CoCo KBS resource path: +# - attestation-agent: ttRPC server plus the Linux TSM_REPORTS based TDX attester. +# - attestation-agent/kbs: serve KBS attestation tokens to CDH's cc_kbc path. +# - confidential-data-hub/kbs: enable the KBS KBC plugin in addition to +# offline_fs_kbc. Runtime still selects the backend through cdh.toml or +# agent.aa_kbc_params, so offline_fs_kbc remains usable. +# - api-server-rest: REST bridge exposing both AA and CDH APIs on loopback. +CARGO_FEATURES = " \ + attestation-agent/bin \ + attestation-agent/ttrpc \ + attestation-agent/tdx-attester \ + attestation-agent/rust-crypto \ + attestation-agent/kbs \ + confidential-data-hub/bin \ + confidential-data-hub/ttrpc \ + confidential-data-hub/kbs \ +" + +EXTRA_CARGO_FLAGS = " \ + --no-default-features \ + -p attestation-agent --bin ttrpc-aa \ + -p confidential-data-hub --bin ttrpc-cdh \ + -p api-server-rest --bin api-server-rest \ +" + +# This layer's cargo_bin class intentionally lets cargo resolve crates directly. +# guest-components also has git dependencies, so network access is required here. +do_compile[network] = "1" + +SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" +SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','coco-attestation-agent.service coco-confidential-data-hub.service coco-api-server-rest.service','',d)}" +# In the Kata/CoCo MVP the modern kata-agent is responsible for launching +# AA/CDH/api-server-rest after it has parsed initdata. Keep the standalone +# systemd units installed for manual/non-Kata testing, but do not auto-start +# them to avoid racing/double-starting the agent-owned processes. +SYSTEMD_AUTO_ENABLE:${PN} = "disable" + +do_configure() { + cargo_bin_do_configure +} + +do_compile() { + # Kata's confidential/guest-pull path asks the guest kata-agent to synthesize + # the Kubernetes sandbox ("pause") container from a pre-baked bundle at + # /pause_bundle. The agent only copies args[0] from that bundle into the + # generated rootfs, so keep this executable static and self-contained. + ${CC} ${CFLAGS} -static -fno-pie -no-pie ${UNPACKDIR}/coco-pause.c \ + -o ${WORKDIR}/coco-pause ${LDFLAGS} + + export TARGET_CC="${WRAPPER_DIR}/cc-wrapper.sh" + export TARGET_CXX="${WRAPPER_DIR}/cxx-wrapper.sh" + export CC="${WRAPPER_DIR}/cc-wrapper.sh" + export CXX="${WRAPPER_DIR}/cxx-wrapper.sh" + export BUILD_CC="${WRAPPER_DIR}/cc-native-wrapper.sh" + export BUILD_CXX="${WRAPPER_DIR}/cxx-native-wrapper.sh" + export TARGET_LD="${WRAPPER_DIR}/linker-wrapper.sh" + export LD="${WRAPPER_DIR}/linker-wrapper.sh" + export PKG_CONFIG_ALLOW_CROSS="1" + + # cargo_bin.bbclass normally clears LDFLAGS. Some rustls/aws-lc-sys + # dependencies compile and execute tiny C feature-test programs when + # BUILD_SYS and TARGET_SYS are both x86_64. The target GCC's default ELF + # interpreter path is not runnable on the build host, so use the build-host + # dynamic linker only for those build-script probes. The final Rust target + # link still goes through linker-wrapper.sh, generated during do_configure + # with the normal target LDFLAGS. + export LDFLAGS="${BUILD_LDFLAGS}" + + export RUSTFLAGS="${RUSTFLAGS}" + export SSH_AUTH_SOCK="${SSH_AUTH_SOCK}" + + # This "DO_NOT_USE_THIS" option of cargo is currently the only way to + # configure a different linker for host and target builds when RUST_BUILD == + # RUST_TARGET. + export __CARGO_TEST_CHANNEL_OVERRIDE_DO_NOT_USE_THIS="nightly" + export CARGO_UNSTABLE_TARGET_APPLIES_TO_HOST="true" + export CARGO_UNSTABLE_HOST_CONFIG="true" + export CARGO_TARGET_APPLIES_TO_HOST="false" + export CARGO_TARGET_${CARGO_TARGET_LINKER_NAME}_LINKER="${WRAPPER_DIR}/linker-wrapper.sh" + export CARGO_HOST_LINKER="${WRAPPER_DIR}/linker-native-wrapper.sh" + export CARGO_BUILD_FLAGS="-C rpath" + export CARGO_PROFILE_RELEASE_DEBUG="true" + + # The CC crate defaults to using CFLAGS when compiling everything. We can + # give it custom flags for compiling on the host. + export HOST_CXXFLAGS="" + export HOST_CFLAGS="" + + bbnote "which rustc:" `which rustc` + bbnote "rustc --version" `rustc --version` + bbnote "which cargo:" `which cargo` + bbnote "cargo --version" `cargo --version` + bbnote cargo build ${CARGO_BUILD_FLAGS} + cargo build ${CARGO_BUILD_FLAGS} +} + +do_install() { + install -d ${D}${bindir} + install -m 0755 ${CARGO_BINDIR}/ttrpc-aa ${D}${bindir}/ttrpc-aa + install -m 0755 ${CARGO_BINDIR}/ttrpc-cdh ${D}${bindir}/ttrpc-cdh + install -m 0755 ${CARGO_BINDIR}/api-server-rest ${D}${bindir}/api-server-rest + + # Modern kata-agent's built-in CoCo launch plan still looks for the + # historical CoCo rootfs paths under /usr/local/bin. Provide compatibility + # symlinks while keeping the real binaries in ${bindir}. + install -d ${D}${prefix}/local/bin + ln -sf ${bindir}/ttrpc-aa ${D}${prefix}/local/bin/attestation-agent + ln -sf ${bindir}/ttrpc-cdh ${D}${prefix}/local/bin/confidential-data-hub + ln -sf ${bindir}/api-server-rest ${D}${prefix}/local/bin/api-server-rest + + install -d ${D}${sysconfdir} + install -m 0644 ${UNPACKDIR}/attestation-agent.conf ${D}${sysconfdir}/attestation-agent.conf + install -m 0644 ${UNPACKDIR}/confidential-data-hub.conf ${D}${sysconfdir}/confidential-data-hub.conf + install -m 0644 ${UNPACKDIR}/ocicrypt_config.json ${D}${sysconfdir}/ocicrypt_config.json + + # Legacy monolithic-rootfs locations consumed by kata-agent when no CoCo + # extension image is mounted. + install -d ${D}/pause_bundle/rootfs + install -m 0644 ${UNPACKDIR}/pause-config.json ${D}/pause_bundle/config.json + install -m 0755 ${WORKDIR}/coco-pause ${D}/pause_bundle/rootfs/pause + + if ${@bb.utils.contains('DISTRO_FEATURES', 'systemd', 'true', 'false', d)}; then + install -d ${D}${systemd_system_unitdir} + install -m 0644 ${UNPACKDIR}/coco-attestation-agent.service ${D}${systemd_system_unitdir}/coco-attestation-agent.service + install -m 0644 ${UNPACKDIR}/coco-confidential-data-hub.service ${D}${systemd_system_unitdir}/coco-confidential-data-hub.service + install -m 0644 ${UNPACKDIR}/coco-api-server-rest.service ${D}${systemd_system_unitdir}/coco-api-server-rest.service + fi +} + +FILES:${PN} += " \ + ${systemd_system_unitdir}/coco-attestation-agent.service \ + ${systemd_system_unitdir}/coco-confidential-data-hub.service \ + ${systemd_system_unitdir}/coco-api-server-rest.service \ + ${sysconfdir}/attestation-agent.conf \ + ${sysconfdir}/confidential-data-hub.conf \ + ${sysconfdir}/ocicrypt_config.json \ + ${prefix}/local/bin/attestation-agent \ + ${prefix}/local/bin/confidential-data-hub \ + ${prefix}/local/bin/api-server-rest \ + /pause_bundle \ +" + +# Cargo embeds build paths into binaries; allow TMPDIR references. +INSANE_SKIP:${PN} += "buildpaths" +INSANE_SKIP:${PN}-dbg += "buildpaths" diff --git a/meta-dstack/recipes-core/coco-guest-components/files/attestation-agent.conf b/meta-dstack/recipes-core/coco-guest-components/files/attestation-agent.conf new file mode 100644 index 0000000..6b91d69 --- /dev/null +++ b/meta-dstack/recipes-core/coco-guest-components/files/attestation-agent.conf @@ -0,0 +1,10 @@ +# Minimal CoCo Attestation Agent config for dstack guest images. +# This first integration only enables local evidence generation; token/KBS +# semantics are intentionally left independent from dstack KMS. + +[eventlog_config] +init_pcr = 17 +enable_eventlog = false + +[log] +level = "info" diff --git a/meta-dstack/recipes-core/coco-guest-components/files/coco-api-server-rest.service b/meta-dstack/recipes-core/coco-guest-components/files/coco-api-server-rest.service new file mode 100644 index 0000000..c808752 --- /dev/null +++ b/meta-dstack/recipes-core/coco-guest-components/files/coco-api-server-rest.service @@ -0,0 +1,14 @@ +[Unit] +Description=Confidential Containers REST API Server +Documentation=https://github.com/confidential-containers/guest-components +After=network.target coco-attestation-agent.service coco-confidential-data-hub.service +Wants=coco-attestation-agent.service coco-confidential-data-hub.service + +[Service] +Type=simple +ExecStart=/usr/bin/api-server-rest --features all --bind 127.0.0.1:8006 --aa_addr unix:///run/confidential-containers/attestation-agent/attestation-agent.sock --cdh_addr unix:///run/confidential-containers/cdh.sock +Restart=on-failure +RestartSec=2s + +[Install] +WantedBy=multi-user.target diff --git a/meta-dstack/recipes-core/coco-guest-components/files/coco-attestation-agent.service b/meta-dstack/recipes-core/coco-guest-components/files/coco-attestation-agent.service new file mode 100644 index 0000000..139fce6 --- /dev/null +++ b/meta-dstack/recipes-core/coco-guest-components/files/coco-attestation-agent.service @@ -0,0 +1,15 @@ +[Unit] +Description=Confidential Containers Attestation Agent (ttRPC) +Documentation=https://github.com/confidential-containers/guest-components +After=local-fs.target dstack-prepare.service +Wants=dstack-prepare.service + +[Service] +Type=simple +ExecStartPre=/bin/mkdir -p /run/confidential-containers/attestation-agent +ExecStart=/usr/bin/ttrpc-aa -c /etc/attestation-agent.conf --attestation_sock unix:///run/confidential-containers/attestation-agent/attestation-agent.sock +Restart=on-failure +RestartSec=2s + +[Install] +WantedBy=multi-user.target diff --git a/meta-dstack/recipes-core/coco-guest-components/files/coco-confidential-data-hub.service b/meta-dstack/recipes-core/coco-guest-components/files/coco-confidential-data-hub.service new file mode 100644 index 0000000..1d14854 --- /dev/null +++ b/meta-dstack/recipes-core/coco-guest-components/files/coco-confidential-data-hub.service @@ -0,0 +1,15 @@ +[Unit] +Description=Confidential Containers Confidential Data Hub (ttRPC) +Documentation=https://github.com/confidential-containers/guest-components +After=local-fs.target network.target coco-attestation-agent.service +Wants=coco-attestation-agent.service + +[Service] +Type=simple +ExecStartPre=/bin/mkdir -p /run/confidential-containers +ExecStart=/usr/bin/ttrpc-cdh -c /etc/confidential-data-hub.conf +Restart=on-failure +RestartSec=2s + +[Install] +WantedBy=multi-user.target diff --git a/meta-dstack/recipes-core/coco-guest-components/files/coco-pause.c b/meta-dstack/recipes-core/coco-guest-components/files/coco-pause.c new file mode 100644 index 0000000..003457e --- /dev/null +++ b/meta-dstack/recipes-core/coco-guest-components/files/coco-pause.c @@ -0,0 +1,29 @@ +// Minimal Kubernetes pause process for Kata/CoCo guest-pull sandbox containers. +// It is installed into /pause_bundle/rootfs/pause and linked statically because +// kata-agent copies only this executable into the synthesized pause rootfs. + +#include +#include +#include + +static void exit_cleanly(int signo) { + (void)signo; + _exit(0); +} + +int main(void) { + struct sigaction sa; + sa.sa_handler = exit_cleanly; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + + sigaction(SIGINT, &sa, NULL); + sigaction(SIGTERM, &sa, NULL); + + for (;;) { + pause(); + if (errno == EINTR) { + continue; + } + } +} diff --git a/meta-dstack/recipes-core/coco-guest-components/files/confidential-data-hub.conf b/meta-dstack/recipes-core/coco-guest-components/files/confidential-data-hub.conf new file mode 100644 index 0000000..a26dbf6 --- /dev/null +++ b/meta-dstack/recipes-core/coco-guest-components/files/confidential-data-hub.conf @@ -0,0 +1,15 @@ +# Minimal CoCo Confidential Data Hub config for dstack guest images. +# KBS support is compiled in, but keep the baked-in default as offline_fs_kbc +# so the image boots without a KBS URL. For real CoCo KBS, override this file +# via initdata cdh.toml with [kbc].name = "cc_kbc" and the KBS URL. +# Optional offline resources can be supplied through +# /etc/aa-offline_fs_kbc-resources.json. + +socket = "unix:///run/confidential-containers/cdh.sock" + +[kbc] +name = "offline_fs_kbc" +url = "" + +[log] +level = "info" diff --git a/meta-dstack/recipes-core/coco-guest-components/files/ocicrypt_config.json b/meta-dstack/recipes-core/coco-guest-components/files/ocicrypt_config.json new file mode 100644 index 0000000..75df218 --- /dev/null +++ b/meta-dstack/recipes-core/coco-guest-components/files/ocicrypt_config.json @@ -0,0 +1,7 @@ +{ + "key-providers": { + "attestation-agent": { + "ttrpc": "unix:///run/confidential-containers/cdh.sock" + } + } +} diff --git a/meta-dstack/recipes-core/coco-guest-components/files/pause-config.json b/meta-dstack/recipes-core/coco-guest-components/files/pause-config.json new file mode 100644 index 0000000..3b23598 --- /dev/null +++ b/meta-dstack/recipes-core/coco-guest-components/files/pause-config.json @@ -0,0 +1,63 @@ +{ + "ociVersion": "1.0.0", + "process": { + "terminal": false, + "user": { + "uid": 65535, + "gid": 65535 + }, + "args": [ + "/pause" + ], + "env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ], + "cwd": "/", + "capabilities": { + "bounding": [ + "CAP_AUDIT_WRITE", + "CAP_KILL", + "CAP_NET_BIND_SERVICE" + ], + "effective": [ + "CAP_AUDIT_WRITE", + "CAP_KILL", + "CAP_NET_BIND_SERVICE" + ], + "inheritable": [ + "CAP_AUDIT_WRITE", + "CAP_KILL", + "CAP_NET_BIND_SERVICE" + ], + "permitted": [ + "CAP_AUDIT_WRITE", + "CAP_KILL", + "CAP_NET_BIND_SERVICE" + ], + "ambient": [ + "CAP_AUDIT_WRITE", + "CAP_KILL", + "CAP_NET_BIND_SERVICE" + ] + }, + "rlimits": [ + { + "type": "RLIMIT_NOFILE", + "hard": 1024, + "soft": 1024 + } + ], + "noNewPrivileges": true + }, + "root": { + "path": "rootfs" + }, + "hostname": "kata-pause", + "mounts": [ + { + "destination": "/proc", + "type": "proc", + "source": "proc" + } + ] +} diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 4862014..a16aa55 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -14,6 +14,8 @@ IMAGE_INSTALL = "\ docker-moby \ docker-compose \ dstack-guest \ + kata-agent-coco \ + coco-guest-components \ wireguard-tools \ cryptsetup \ curl \ diff --git a/meta-dstack/recipes-core/kata-agent/files/0001-kata-agent-build-with-rust-1.92.patch b/meta-dstack/recipes-core/kata-agent/files/0001-kata-agent-build-with-rust-1.92.patch new file mode 100644 index 0000000..77c1b55 --- /dev/null +++ b/meta-dstack/recipes-core/kata-agent/files/0001-kata-agent-build-with-rust-1.92.patch @@ -0,0 +1,54 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: dstack +Date: Thu, 2 Jul 2026 09:00:00 -0700 +Subject: [PATCH] kata-agent: build with Rust 1.92 + +meta-dstack currently builds Rust binaries with cargo-bin 1.92. The +upstream Kata main branch requires Rust 1.94 in Cargo.toml, but the agent +code used by this MVP builds with 1.92 once the newly-unsafe x86 cpuid +intrinsic calls are wrapped in unsafe blocks. + +Upstream-Status: Inappropriate [meta-dstack currently builds this MVP with cargo-bin 1.92] +Signed-off-by: dstack +--- + Cargo.toml | 2 +- + src/libs/kata-sys-util/src/protection.rs | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/Cargo.toml b/Cargo.toml +index 46a796b..51a43aa 100644 +--- a/Cargo.toml ++++ b/Cargo.toml +@@ -2,7 +2,7 @@ + authors = ["The Kata Containers community "] + edition = "2018" + license = "Apache-2.0" +-rust-version = "1.94" ++rust-version = "1.92" + + [workspace] + members = [ +diff --git a/src/libs/kata-sys-util/src/protection.rs b/src/libs/kata-sys-util/src/protection.rs +index 93e11fe..d19fe44 100644 +--- a/src/libs/kata-sys-util/src/protection.rs ++++ b/src/libs/kata-sys-util/src/protection.rs +@@ -126,7 +126,7 @@ pub fn available_guest_protection() -> Result + // shouldn't hurt to double-check and have better logging if anything + // goes wrong. + +- let fn0 = x86_64::__cpuid(0); ++ let fn0 = unsafe { x86_64::__cpuid(0) }; + // The values in [ ebx, edx, ecx ] spell out "AuthenticAMD" when + // interpreted byte-wise as ASCII. No need to bother here with an + // actual conversion to string though. +@@ -140,7 +140,7 @@ pub fn available_guest_protection() -> Result + } + + // AMD64 Architecture Prgrammer's Manual Fn8000_001f docs on pg. 640 +- let fn8000_001f = x86_64::__cpuid(0x8000_001f); ++ let fn8000_001f = unsafe { x86_64::__cpuid(0x8000_001f) }; + if fn8000_001f.eax & 0x10 == 0 { + return Err(ProtectionError::CheckFailed("SEV not supported".to_owned())); + } +-- +2.43.0 diff --git a/meta-dstack/recipes-core/kata-agent/kata-agent-coco_git.bb b/meta-dstack/recipes-core/kata-agent/kata-agent-coco_git.bb new file mode 100644 index 0000000..bab1f8a --- /dev/null +++ b/meta-dstack/recipes-core/kata-agent/kata-agent-coco_git.bb @@ -0,0 +1,113 @@ +SUMMARY = "Kata agent with CoCo initdata support for dstack guest images" +DESCRIPTION = "Modern Rust Kata agent built with init-data and agent-policy features, used to boot dstack rootfs as a Kata/CoCo guest image." +HOMEPAGE = "https://github.com/kata-containers/kata-containers" +LICENSE = "Apache-2.0" +LIC_FILES_CHKSUM = "file://LICENSE;md5=86d3f3a95c324c9479bd8986968f4327" + +SRCREV = "d7be140eee9f96452a6120c65320cef8be1c7ecc" +SRC_URI = "git://github.com/kata-containers/kata-containers.git;protocol=https;branch=main \ + file://0001-kata-agent-build-with-rust-1.92.patch \ +" + +PV = "0.1.0+git" + +inherit cargo_bin systemd + +# Build only the in-guest agent. Enable the two features required by the +# CoCo/Kata initdata flow: +# - init-data: read the initdata block device and extract aa.toml/cdh.toml/policy.rego +# - agent-policy: initialize the Kata agent policy engine from policy.rego +# Seccomp/devicemapper are intentionally left out for this first MVP to keep +# the dependency surface small. +CARGO_FEATURES = "kata-agent/init-data kata-agent/agent-policy" +EXTRA_CARGO_FLAGS = "--no-default-features -p kata-agent" + +do_compile[network] = "1" + +SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" +SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','kata-agent.service','',d)}" +# Do not start kata-agent in a normal dstack boot. Kata runtime should boot the +# image with systemd.unit=kata-containers.target (or otherwise explicitly start +# kata-agent.service) when this rootfs is used as a Kata guest image. +SYSTEMD_AUTO_ENABLE:${PN} = "disable" + +PROVIDES += "kata-agent" +RPROVIDES:${PN} += "kata-agent" +RDEPENDS:${PN} += "bash systemd" + +KATA_AGENT_VERSION ?= "${PV}" +KATA_AGENT_API_VERSION ?= "0.0.1" + +kata_agent_generate_file() { + src="$1" + dst="$2" + install -d "$(dirname "$dst")" + sed \ + -e 's|@AGENT_NAME@|kata-agent|g' \ + -e 's|@AGENT_VERSION@|${KATA_AGENT_VERSION}|g' \ + -e 's|@API_VERSION@|${KATA_AGENT_API_VERSION}|g' \ + -e 's|@BINDIR@|${bindir}|g' \ + -e 's|@COMMIT@|${SRCREV}|g' \ + -e 's|@VERSION_COMMIT@|${KATA_AGENT_VERSION}-${SRCREV}|g' \ + "$src" > "$dst" +} + +do_configure() { + cargo_bin_do_configure + + # Kata normally creates this file from src/agent/Makefile. We build via + # cargo_bin from the workspace root, so generate the small version module + # here instead of invoking Kata's Makefile. + kata_agent_generate_file \ + ${S}/src/agent/src/version.rs.in \ + ${S}/src/agent/src/version.rs +} + +do_compile() { + cargo_bin_do_compile +} + +do_install() { + install -d ${D}${bindir} + install -m 0755 ${CARGO_BINDIR}/kata-agent ${D}${bindir}/kata-agent + + # The agent-policy feature initializes OPA before initdata is parsed. + # Ship Kata's allow-all policy as the default baseline; initdata + # policy.rego can still replace it after initdata extraction. + install -d ${D}${sysconfdir}/kata-opa + install -m 0644 ${S}/src/kata-opa/allow-all.rego \ + ${D}${sysconfdir}/kata-opa/default-policy.rego + + if ${@bb.utils.contains('DISTRO_FEATURES', 'systemd', 'true', 'false', d)}; then + install -d ${D}${systemd_system_unitdir} + kata_agent_generate_file \ + ${S}/src/agent/kata-agent.service.in \ + ${D}${systemd_system_unitdir}/kata-agent.service + install -m 0644 ${S}/src/agent/kata-containers.target \ + ${D}${systemd_system_unitdir}/kata-containers.target + install -m 0644 ${S}/src/agent/kata-extension-mount@.service \ + ${D}${systemd_system_unitdir}/kata-extension-mount@.service + + install -d ${D}${libexecdir} ${D}${systemd_unitdir}/system-generators + install -m 0755 ${S}/src/agent/kata-extension-mount.sh \ + ${D}${libexecdir}/kata-extension-mount.sh + install -m 0755 ${S}/src/agent/kata-extension-umount.sh \ + ${D}${libexecdir}/kata-extension-umount.sh + install -m 0755 ${S}/src/agent/kata-extension-mount-generator.sh \ + ${D}${systemd_unitdir}/system-generators/kata-extension-mount-generator + fi +} + +FILES:${PN} += " \ + ${sysconfdir}/kata-opa/default-policy.rego \ + ${systemd_system_unitdir}/kata-agent.service \ + ${systemd_system_unitdir}/kata-containers.target \ + ${systemd_system_unitdir}/kata-extension-mount@.service \ + ${libexecdir}/kata-extension-mount.sh \ + ${libexecdir}/kata-extension-umount.sh \ + ${systemd_unitdir}/system-generators/kata-extension-mount-generator \ +" + +# Cargo embeds build paths into binaries; allow TMPDIR references. +INSANE_SKIP:${PN} += "buildpaths" +INSANE_SKIP:${PN}-dbg += "buildpaths"